Merged in feature/splittest (pull request #137)

Split Slow DB Test

* split

deployments/compose.generate.yaml

@@ -12,7 +12,7 @@ services:
       - 5431
     healthcheck:
       test: ["CMD", "sh", "-c", "pg_isready -U ${PGUSER} -h localhost -p 5432 && psql -U ${PGUSER} -c 'SELECT 1'"]
-      interval: 500ms
+      interval: 50ms
       timeout: 5s
       retries: 50
     networks:

internal/database/repository/result_test.go

@@ -28,13 +28,13 @@ func TestResults(t *testing.T) {
 
 	queries := cfg.GetDBQueries()
 
-	contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
+	queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-	clientId := createClientWithCollector(t, t.Context(), queries)
+	clientId := createClientWithCollector(t, queries)
-	documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+	documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
 
 	qv := int32(1)
 	res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
-		Queryid:      &jsonQueryID,
+		Queryid:      &queryId,
 		Queryversion: &qv,
 		Documentid:   &documentID,
 	})
@@ -271,7 +271,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
 
 		err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
@@ -387,9 +387,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-		documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
 
 		documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
 			Clientid: clientId,
@@ -484,9 +484,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-		documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
 
 		_, err := queries.AddLatestQueryVersion(ctx, jsonQueryID)
 		require.NoError(t, err)
@@ -512,7 +512,7 @@ func BenchmarkListUnsynced(b *testing.B) {
 
 	queries := cfg.GetDBQueries()
 
-	clientId := createClientWithCollector(b, ctx, queries)
+	clientId := createClientWithCollector(b, queries)
 	_, jsonQueryID := createDependentQueries(b, ctx, queries)
 
 	err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{

internal/database/repository/sync_test.go

@@ -172,17 +172,17 @@ func createDependentQueries(t testing.TB, ctx context.Context, queries *reposito
 	return contextQueryID, jsonQueryID
 }
 
-func createClientWithCollector(t testing.TB, ctx context.Context, queries *repository.Queries) string {
+func createClientWithCollector(t testing.TB, queries *repository.Queries) string {
 	clientId := "EXAMPLE"
-	err := queries.CreateClient(ctx, &repository.CreateClientParams{
+	err := queries.CreateClient(t.Context(), &repository.CreateClientParams{
 		Name:     "example_client",
 		Clientid: clientId,
 	})
 	require.NoError(t, err)
 
-	collectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
+	collectorVersion, err := queries.AddLatestCollectorVersion(t.Context(), clientId)
 	require.NoError(t, err)
-	err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
+	err = queries.SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
 		Versionid: collectorVersion,
 		Clientid:  clientId,
 	})
@@ -229,7 +229,7 @@ func TestClientSync(t *testing.T) {
 		queries := cfg.GetDBQueries()
 
 		_, jsonQueryID := createDependentQueries(t, ctx, queries)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
 
 		isSynced, err := queries.IsClientSynced(ctx, &clientId)
 		require.NoError(t, err)
@@ -248,7 +248,7 @@ func TestClientSync(t *testing.T) {
 		assert.True(t, isSynced)
 	})
 
-	t.Run("document fail clean", func(t *testing.T) {
+	t.Run("create document", func(t *testing.T) {
 		t.Parallel()
 		ctx := context.Background()
 
@@ -258,12 +258,12 @@ func TestClientSync(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		_, jsonQueryID := createDependentQueries(t, ctx, queries)
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
 
 		err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
 			Clientid:     clientId,
-			Queryid:      jsonQueryID,
+			Queryid:      queryId,
 			Addedversion: 1,
 			Name:         "first_key",
 		})
@@ -282,6 +282,21 @@ func TestClientSync(t *testing.T) {
 			isSynced:   false,
 			fields:     `{"first_key": null}`,
 		})
+	})
+
+	t.Run("document fail clean", func(t *testing.T) {
+		t.Parallel()
+		ctx := context.Background()
+
+		cfg := &serviceconfig.BaseConfig{}
+		net := test.GetNetwork(t)
+		test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
+		queries := cfg.GetDBQueries()
+
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
+		clientId := createClientWithCollector(t, queries)
+		documentID := createDocumentWithCollector(t, queries, clientId, queryId)
 
 		cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
 			Documentid: documentID,
@@ -300,13 +315,13 @@ func TestClientSync(t *testing.T) {
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
-			hash:       "example_noclean",
+			hash:       "example_hash",
 			isSynced:   true,
 			fields:     `{"first_key": null}`,
 		})
 	})
 
-	t.Run("single query", func(t *testing.T) {
+	t.Run("clean document", func(t *testing.T) {
 		t.Parallel()
 		ctx := context.Background()
 
@@ -317,24 +332,10 @@ func TestClientSync(t *testing.T) {
 		queries := cfg.GetDBQueries()
 
 		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-
+		documentID := createDocumentWithCollector(t, queries, clientId, queryId)
-		err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
-			Clientid:     clientId,
-			Queryid:      queryId,
-			Addedversion: 1,
-			Name:         "first_key",
-		})
-		require.NoError(t, err)
-
-		bucket := "example_bucket"
-
-		documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
-			Clientid: clientId,
-			Hash:     "example_hash",
-		})
-		require.NoError(t, err)
 
+		bucket := "buck"
 		key := "example_key"
 		hash := "hahssh"
 		cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
@@ -361,6 +362,21 @@ func TestClientSync(t *testing.T) {
 			isSynced:   false,
 			fields:     `{"first_key": null}`,
 		})
+	})
+
+	t.Run("extracted text", func(t *testing.T) {
+		t.Parallel()
+		ctx := context.Background()
+
+		cfg := &serviceconfig.BaseConfig{}
+		net := test.GetNetwork(t)
+		test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
+		queries := cfg.GetDBQueries()
+
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
+		clientId := createClientWithCollector(t, queries)
+		documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, queryId)
 
 		textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 			Cleanid: cleanId,
@@ -387,8 +403,23 @@ func TestClientSync(t *testing.T) {
 			isSynced:   false,
 			fields:     `{"first_key": null}`,
 		})
+	})
 
-		_, err = queries.AddResult(ctx, &repository.AddResultParams{
+	t.Run("single query", func(t *testing.T) {
+		t.Parallel()
+		ctx := context.Background()
+
+		cfg := &serviceconfig.BaseConfig{}
+		net := test.GetNetwork(t)
+		test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
+		queries := cfg.GetDBQueries()
+
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
+		clientId := createClientWithCollector(t, queries)
+		documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, queryId)
+
+		_, err := queries.AddResult(ctx, &repository.AddResultParams{
 			Queryid:      queryId,
 			Value:        "json_value",
 			Textentryid:  textId,
@@ -416,60 +447,8 @@ func TestClientSync(t *testing.T) {
 		queries := cfg.GetDBQueries()
 
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-
+		documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
-		err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
-			Clientid:     clientId,
-			Queryid:      jsonQueryID,
-			Addedversion: 1,
-			Name:         "first_key",
-		})
-		require.NoError(t, err)
-
-		bucket := "example_bucket"
-
-		documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
-			Clientid: clientId,
-			Hash:     "example_hash",
-		})
-		require.NoError(t, err)
-
-		key := "example_key"
-		hash := "hahssh"
-		cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
-			Documentid: documentID,
-			Bucket:     &bucket,
-			Key:        &key,
-			Hash:       &hash,
-			Mimetype: repository.NullCleanmimetype{
-				Valid:         true,
-				Cleanmimetype: repository.CleanmimetypeApplicationPdf,
-			},
-		})
-		require.NoError(t, err)
-		err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
-			Cleanid: cleanId,
-			Version: 1,
-		})
-		require.NoError(t, err)
-
-		textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-			Cleanid: cleanId,
-			Part:    0,
-			Createdat: pgtype.Timestamp{
-				Time:  time.Now().UTC(),
-				Valid: true,
-			},
-			Bucket: "hi",
-			Key:    "hello",
-			Hash:   "example",
-		})
-		require.NoError(t, err)
-		err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-			Version: 1,
-			Textid:  textId,
-		})
-		require.NoError(t, err)
 
 		contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
 			Queryid:      contextQueryID,
@@ -521,8 +500,8 @@ func TestClientSync(t *testing.T) {
 		queries := cfg.GetDBQueries()
 
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, textId := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
 
 		contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID)
 		require.NoError(t, err)
@@ -581,9 +560,9 @@ func TestClientSync(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, cleanId, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, cleanId, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
 
 		textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 			Bucket: "hi",
@@ -610,28 +589,14 @@ func TestClientSync(t *testing.T) {
 			fields:     `{"first_key": null}`,
 		})
 
-		contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
+		_, err = queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      contextQueryID,
+			Queryid:      queryId,
-			Value:        "updated_text_context",
-			Textentryid:  textId,
-			Queryversion: 1,
-		})
-		require.NoError(t, err)
-
-		jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      jsonQueryID,
 			Value:        "updated_text_json",
 			Textentryid:  textId,
 			Queryversion: 1,
 		})
 		require.NoError(t, err)
 
-		err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
-			Resultid:         jsonResultID,
-			Requiredresultid: contextResultID,
-		})
-		require.NoError(t, err)
-
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
@@ -651,9 +616,9 @@ func TestClientSync(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
 
 		bucket := "cu"
 		key := "kk"
@@ -700,28 +665,14 @@ func TestClientSync(t *testing.T) {
 		})
 		require.NoError(t, err)
 
-		contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
+		_, err = queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      contextQueryID,
+			Queryid:      queryId,
-			Value:        "update_clean_context",
-			Textentryid:  textThreeId,
-			Queryversion: 1,
-		})
-		require.NoError(t, err)
-
-		jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      jsonQueryID,
 			Value:        "update_clean_json",
 			Textentryid:  textThreeId,
 			Queryversion: 1,
 		})
 		require.NoError(t, err)
 
-		err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
-			Resultid:         jsonResultID,
-			Requiredresultid: contextResultId,
-		})
-		require.NoError(t, err)
-
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
@@ -731,7 +682,7 @@ func TestClientSync(t *testing.T) {
 		})
 	})
 
-	t.Run("change collector name", func(t *testing.T) {
+	t.Run("add new collector version", func(t *testing.T) {
 		t.Parallel()
 		ctx := context.Background()
 
@@ -741,13 +692,78 @@ func TestClientSync(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
+
+		_, err := queries.AddLatestCollectorVersion(ctx, clientId)
+		require.NoError(t, err)
+
+		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
+			clientId:   clientId,
+			documentId: documentID,
+			hash:       "example_hash",
+			isSynced:   true,
+			fields:     `{"first_key": "json_value"}`,
+		})
+	})
+
+	t.Run("add new active collector version", func(t *testing.T) {
+		t.Parallel()
+		ctx := context.Background()
+
+		cfg := &serviceconfig.BaseConfig{}
+		net := test.GetNetwork(t)
+		test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
+		queries := cfg.GetDBQueries()
+
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
+		clientId := createClientWithCollector(t, queries)
+		documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
 
 		latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
 		require.NoError(t, err)
 
+		err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
+			Versionid: latestCollectorVersion,
+			Clientid:  clientId,
+		})
+		require.NoError(t, err)
+
+		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
+			clientId:   clientId,
+			documentId: documentID,
+			hash:       "example_hash",
+			isSynced:   true,
+			fields:     `{"first_key": "json_value"}`,
+		})
+	})
+
+	t.Run("remove query from new collector version", func(t *testing.T) {
+		t.Parallel()
+		ctx := context.Background()
+
+		cfg := &serviceconfig.BaseConfig{}
+		net := test.GetNetwork(t)
+		test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
+
+		queries := cfg.GetDBQueries()
+
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
+		clientId := createClientWithCollector(t, queries)
+		documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
+
+		latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
+		require.NoError(t, err)
+
+		err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
+			Clientid:       clientId,
+			Queryid:        queryId,
+			Removedversion: &latestCollectorVersion,
+		})
+		require.NoError(t, err)
+
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
@@ -762,21 +778,6 @@ func TestClientSync(t *testing.T) {
 		})
 		require.NoError(t, err)
 
-		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
-			clientId:   clientId,
-			documentId: documentID,
-			hash:       "example_hash",
-			isSynced:   true,
-			fields:     `{"first_key": "json_value"}`,
-		})
-
-		err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
-			Clientid:       clientId,
-			Queryid:        jsonQueryID,
-			Removedversion: &latestCollectorVersion,
-		})
-		require.NoError(t, err)
-
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
@@ -784,22 +785,6 @@ func TestClientSync(t *testing.T) {
 			isSynced:   true,
 			fields:     `{}`,
 		})
-
-		err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
-			Clientid:     clientId,
-			Queryid:      jsonQueryID,
-			Addedversion: latestCollectorVersion,
-			Name:         "second_key",
-		})
-		require.NoError(t, err)
-
-		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
-			clientId:   clientId,
-			documentId: documentID,
-			hash:       "example_hash",
-			isSynced:   true,
-			fields:     `{"second_key": "json_value"}`,
-		})
 	})
 
 	t.Run("add existing query", func(t *testing.T) {
@@ -813,8 +798,8 @@ func TestClientSync(t *testing.T) {
 		queries := cfg.GetDBQueries()
 
 		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
 
 		err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
 			Clientid:     clientId,
@@ -843,13 +828,13 @@ func TestClientSync(t *testing.T) {
 
 		queries := cfg.GetDBQueries()
 
-		contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
+		queryId := createQuery(t, queries, repository.QuerytypeContextFull)
-		clientId := createClientWithCollector(t, ctx, queries)
+		clientId := createClientWithCollector(t, queries)
-		documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
+		documentID, _, textId := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
 
 		superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor)
 		err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
-			Queryid:         contextQueryID,
+			Queryid:         queryId,
 			Requiredqueryid: superQueryID,
 			Addedversion:    1,
 		})
@@ -871,16 +856,8 @@ func TestClientSync(t *testing.T) {
 		})
 		require.NoError(t, err)
 
-		contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
+		resultId, err := queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      contextQueryID,
+			Queryid:      queryId,
-			Value:        "context_with_super_value",
-			Textentryid:  textId,
-			Queryversion: 1,
-		})
-		require.NoError(t, err)
-
-		jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
-			Queryid:      jsonQueryID,
 			Value:        "json_with_super",
 			Textentryid:  textId,
 			Queryversion: 1,
@@ -888,25 +865,11 @@ func TestClientSync(t *testing.T) {
 		require.NoError(t, err)
 
 		err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
-			Resultid:         contextResultId,
+			Resultid:         resultId,
 			Requiredresultid: superResultId,
 		})
 		require.NoError(t, err)
 
-		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
-			clientId:   clientId,
-			documentId: documentID,
-			hash:       "example_hash",
-			isSynced:   false,
-			fields:     `{"first_key": null}`,
-		})
-
-		err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
-			Resultid:         jsonResultId,
-			Requiredresultid: contextResultId,
-		})
-		require.NoError(t, err)
-
 		getDocumentSyncState(t, ctx, queries, docSyncStateParams{
 			clientId:   clientId,
 			documentId: documentID,
@@ -917,25 +880,31 @@ func TestClientSync(t *testing.T) {
 	})
 }
 
-func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
+func createDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) uuid.UUID {
-	err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
+	err := queries.AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
 		Clientid:     clientId,
 		Queryid:      jsonQueryID,
 		Addedversion: 1,
 		Name:         "first_key",
 	})
 	require.NoError(t, err)
-	bucket := "example_bucket"
 
-	documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
+	documentID, err := queries.CreateDocument(t.Context(), &repository.CreateDocumentParams{
 		Clientid: clientId,
 		Hash:     "example_hash",
 	})
 	require.NoError(t, err)
 
+	return documentID
+}
+
+func createCleanDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID) {
+	documentID := createDocumentWithCollector(t, queries, clientId, jsonQueryID)
+
+	bucket := "example_bucket"
 	key := "example_key"
 	hash := "hahssh"
-	cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
+	cleanId, err := queries.AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
 		Documentid: documentID,
 		Bucket:     &bucket,
 		Key:        &key,
@@ -946,13 +915,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
 		},
 	})
 	require.NoError(t, err)
-	err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
+	err = queries.AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
 		Cleanid: cleanId,
 		Version: 1,
 	})
 	require.NoError(t, err)
 
-	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
+	return documentID, cleanId
+}
+
+func createDocumentWithCollectorAndText(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
+	documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, jsonQueryID)
+
+	textId, err := queries.AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
 		Cleanid: cleanId,
 		Part:    0,
 		Createdat: pgtype.Timestamp{
@@ -964,21 +939,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
 		Hash:   "example",
 	})
 	require.NoError(t, err)
-	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
+	err = queries.AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
 		Version: 1,
 		Textid:  textId,
 	})
 	require.NoError(t, err)
 
-	contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
+	return documentID, cleanId, textId
-		Queryid:      contextQueryID,
+}
-		Value:        "example_context",
-		Textentryid:  textId,
-		Queryversion: 1,
-	})
-	require.NoError(t, err)
 
-	jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
+func createDocumentWithCollectorAndResult(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
+	documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
+
+	_, err := queries.AddResult(t.Context(), &repository.AddResultParams{
 		Queryid:      jsonQueryID,
 		Value:        "json_value",
 		Textentryid:  textId,
@@ -986,7 +959,29 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
 	})
 	require.NoError(t, err)
 
-	err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
+	return documentID, cleanId, textId
+}
+
+func createDocumentWithCollectorAndResults(t testing.TB, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
+	documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
+
+	contextResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
+		Queryid:      contextQueryID,
+		Value:        "example_context",
+		Textentryid:  textId,
+		Queryversion: 1,
+	})
+	require.NoError(t, err)
+
+	jsonResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
+		Queryid:      jsonQueryID,
+		Value:        "json_value",
+		Textentryid:  textId,
+		Queryversion: 1,
+	})
+	require.NoError(t, err)
+
+	err = queries.AddResultDependency(t.Context(), &repository.AddResultDependencyParams{
 		Resultid:         jsonResultId,
 		Requiredresultid: contextResultId,
 	})
@@ -1005,8 +1000,8 @@ func BenchmarkIsClientSynced(b *testing.B) {
 	queries := cfg.GetDBQueries()
 
 	contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
-	clientId := createClientWithCollector(b, ctx, queries)
+	clientId := createClientWithCollector(b, queries)
-	_, _, _ = createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID)
+	_, _, _ = createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
 
 	b.ResetTimer()
 
@@ -1025,8 +1020,8 @@ func BenchmarkGetDocumentExternal(b *testing.B) {
 	queries := cfg.GetDBQueries()
 
 	contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
-	clientId := createClientWithCollector(b, ctx, queries)
+	clientId := createClientWithCollector(b, queries)
-	documentID, _, _ := createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID)
+	documentID, _, _ := createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
 
 	b.ResetTimer()
 

scripts/tests.yml

@@ -6,6 +6,7 @@ version: "3"
 vars:
   COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out"
   BENCH_FILE: "{{.OUT_DIR}}/bench.out"
+  TEST_FILE: "{{.OUT_DIR}}/test.out"
   INTERNAL: "./internal/..."
   API: "./api/..."
   PKG: "./pkg/..."
@@ -44,6 +45,33 @@ tasks:
         -coverprofile={{.COVERAGE_FILE}} \
         ./...
       - task: coverage
+  perf:
+    vars:
+      NUM_RESULTS: 5
+    cmds:
+      - GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -json ./...
+      - |
+        echo -e "🐢 TOP {{.NUM_RESULTS}} SLOWEST INDIVIDUAL PACKAGE RUNS:"
+        jq -r 'select(.Action == "pass" and .Test == null) | "\(.Elapsed)s \(.Package)"' "{{.TEST_FILE}}" | \
+          sort -rn | head -n {{.NUM_RESULTS}}
+
+        echo -e "\n📉 TOP {{.NUM_RESULTS}} SLOWEST TESTS:"
+        jq -r 'select(.Action == "pass" and .Test != null) | "\(.Elapsed)s \(.Test) (\(.Package))"' "{{.TEST_FILE}}" | \
+          sort -rn | head -n {{.NUM_RESULTS}}
+
+        echo -e "\n📦 TOP {{.NUM_RESULTS}} SLOWEST PACKAGES (TOTAL TIME):"
+        jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" |
+          awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' |
+          sort -rn | head -n {{.NUM_RESULTS}}
+
+        SLOWEST_PKG=$(jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" |
+        awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' |
+        sort -rn | head -n 1 | awk '{print $2}')
+
+        echo -e "\n🔎 TOP {{.NUM_RESULTS}} SLOWEST TESTS IN SLOWEST PACKAGE ($SLOWEST_PKG):"
+        jq -r --arg pkg "$SLOWEST_PKG" \
+          'select(.Action == "pass" and .Test != null and .Package == $pkg) | "\(.Elapsed)s \(.Test)"' "{{.TEST_FILE}}" |
+          sort -rn | head -n {{.NUM_RESULTS}}
   bench:
     vars:
       TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
@@ -65,38 +93,45 @@ tasks:
       TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
       FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out"
     cmds:
-      - |
+      - grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} > {{.FILTER_COVERAGE_FILE}}
-        grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} \
-        > {{.FILTER_COVERAGE_FILE}}
       - go tool cover -func={{.FILTER_COVERAGE_FILE}} > {{.TMP_FILE}}
       - |
-        COVERAGE=$(grep total: {{.TMP_FILE}} \
+        COVERAGE=$(grep total: {{.TMP_FILE}} | awk '{print $NF}' | sed 's/%//')
-          | awk '{print $3}' | sed 's/%//' | bc)
         echo "Coverage Threshold: {{.COVERAGE_THRESHOLD}}%"
         echo "Total Coverage: $COVERAGE%"
 
+        FAIL=false
         if (( $(echo "$COVERAGE < {{.COVERAGE_THRESHOLD}}" | bc -l) )); then
           echo "❌ Error: Total Coverage below Coverage Threshold"
           FAIL=true
         fi
 
         echo "Function Coverage Threshold: {{.FUNCTION_COVERAGE_THRESHOLD}}%"
-        FAIL_FUNC=false
+
-        grep -v total: {{.TMP_FILE}} | while read -r line; do
+        awk -v threshold="{{.FUNCTION_COVERAGE_THRESHOLD}}" '
-          FUNC_COV=$(echo "$line" | awk '{print $3}' | sed 's/%//' | bc)
+        !/total:/ {
-          if (( $(echo "$FUNC_COV < \
+          line = $0;
-            {{.FUNCTION_COVERAGE_THRESHOLD}}" | bc -l) )); then
+          coverage = $NF;
-            if [ "$FAIL_FUNC" = false ]; then
+          gsub(/%/, "", coverage);
-              echo "❌ Error: The following functions have coverage below "\
+
-                "{{.FUNCTION_COVERAGE_THRESHOLD}}%:"
+          if (coverage + 0 < threshold + 0) {
-              FAIL_FUNC=true
+            if (!printed_header) {
+              print "❌ Error: The following functions have coverage below " threshold "%:";
+              printed_header = 1;
+            }
+            print line;
+            exit_code = 1;
+          }
+        }
+        END {
+          exit exit_code;
+        }
+        ' {{.TMP_FILE}}
+
+        if [ $? -eq 1 ]; then
           FAIL=true
         fi
 
-            echo "$line"
-          fi
-        done
-
         if [ "$FAIL" = true ]; then
           exit 1
         fi