diff --git a/deployments/compose.generate.yaml b/deployments/compose.generate.yaml index 1429bd16..ac270476 100644 --- a/deployments/compose.generate.yaml +++ b/deployments/compose.generate.yaml @@ -12,7 +12,7 @@ services: - 5431 healthcheck: test: ["CMD", "sh", "-c", "pg_isready -U ${PGUSER} -h localhost -p 5432 && psql -U ${PGUSER} -c 'SELECT 1'"] - interval: 500ms + interval: 50ms timeout: 5s retries: 50 networks: diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index 0c61e8f8..c1bda7a6 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -28,13 +28,13 @@ func TestResults(t *testing.T) { queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, t.Context(), queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) qv := int32(1) res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ - Queryid: &jsonQueryID, + Queryid: &queryId, Queryversion: &qv, Documentid: &documentID, }) @@ -271,7 +271,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { queries := cfg.GetDBQueries() - clientId := createClientWithCollector(t, ctx, queries) + clientId := createClientWithCollector(t, queries) contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ @@ -387,9 +387,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { queries := cfg.GetDBQueries() - clientId := createClientWithCollector(t, ctx, queries) + clientId := createClientWithCollector(t, queries) contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID) documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Clientid: clientId, @@ -484,9 +484,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { queries := cfg.GetDBQueries() - clientId := createClientWithCollector(t, ctx, queries) + clientId := createClientWithCollector(t, queries) contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID) _, err := queries.AddLatestQueryVersion(ctx, jsonQueryID) require.NoError(t, err) @@ -512,7 +512,7 @@ func BenchmarkListUnsynced(b *testing.B) { queries := cfg.GetDBQueries() - clientId := createClientWithCollector(b, ctx, queries) + clientId := createClientWithCollector(b, queries) _, jsonQueryID := createDependentQueries(b, ctx, queries) err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go index 8176f0aa..dbe51bd3 100644 --- a/internal/database/repository/sync_test.go +++ b/internal/database/repository/sync_test.go @@ -172,17 +172,17 @@ func createDependentQueries(t testing.TB, ctx context.Context, queries *reposito return contextQueryID, jsonQueryID } -func createClientWithCollector(t testing.TB, ctx context.Context, queries *repository.Queries) string { +func createClientWithCollector(t testing.TB, queries *repository.Queries) string { clientId := "EXAMPLE" - err := queries.CreateClient(ctx, &repository.CreateClientParams{ + err := queries.CreateClient(t.Context(), &repository.CreateClientParams{ Name: "example_client", Clientid: clientId, }) require.NoError(t, err) - collectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) + collectorVersion, err := queries.AddLatestCollectorVersion(t.Context(), clientId) require.NoError(t, err) - err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + err = queries.SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{ Versionid: collectorVersion, Clientid: clientId, }) @@ -229,7 +229,7 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() _, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) + clientId := createClientWithCollector(t, queries) isSynced, err := queries.IsClientSynced(ctx, &clientId) require.NoError(t, err) @@ -248,7 +248,7 @@ func TestClientSync(t *testing.T) { assert.True(t, isSynced) }) - t.Run("document fail clean", func(t *testing.T) { + t.Run("create document", func(t *testing.T) { t.Parallel() ctx := context.Background() @@ -258,12 +258,12 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() - _, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Clientid: clientId, - Queryid: jsonQueryID, + Queryid: queryId, Addedversion: 1, Name: "first_key", }) @@ -282,6 +282,21 @@ func TestClientSync(t *testing.T) { isSynced: false, fields: `{"first_key": null}`, }) + }) + + t.Run("document fail clean", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID := createDocumentWithCollector(t, queries, clientId, queryId) cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ Documentid: documentID, @@ -300,13 +315,13 @@ func TestClientSync(t *testing.T) { getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, - hash: "example_noclean", + hash: "example_hash", isSynced: true, fields: `{"first_key": null}`, }) }) - t.Run("single query", func(t *testing.T) { + t.Run("clean document", func(t *testing.T) { t.Parallel() ctx := context.Background() @@ -317,24 +332,10 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() queryId := createQuery(t, queries, repository.QuerytypeContextFull) - clientId := createClientWithCollector(t, ctx, queries) - - err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: queryId, - Addedversion: 1, - Name: "first_key", - }) - require.NoError(t, err) - - bucket := "example_bucket" - - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: clientId, - Hash: "example_hash", - }) - require.NoError(t, err) + clientId := createClientWithCollector(t, queries) + documentID := createDocumentWithCollector(t, queries, clientId, queryId) + bucket := "buck" key := "example_key" hash := "hahssh" cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ @@ -361,6 +362,21 @@ func TestClientSync(t *testing.T) { isSynced: false, fields: `{"first_key": null}`, }) + }) + + t.Run("extracted text", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, queryId) textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ Cleanid: cleanId, @@ -387,8 +403,23 @@ func TestClientSync(t *testing.T) { isSynced: false, fields: `{"first_key": null}`, }) + }) - _, err = queries.AddResult(ctx, &repository.AddResultParams{ + t.Run("single query", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, queryId) + + _, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: queryId, Value: "json_value", Textentryid: textId, @@ -416,60 +447,8 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - - err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Addedversion: 1, - Name: "first_key", - }) - require.NoError(t, err) - - bucket := "example_bucket" - - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: clientId, - Hash: "example_hash", - }) - require.NoError(t, err) - - key := "example_key" - hash := "hahssh" - cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ - Documentid: documentID, - Bucket: &bucket, - Key: &key, - Hash: &hash, - Mimetype: repository.NullCleanmimetype{ - Valid: true, - Cleanmimetype: repository.CleanmimetypeApplicationPdf, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ - Cleanid: cleanId, - Version: 1, - }) - require.NoError(t, err) - - textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Cleanid: cleanId, - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - Bucket: "hi", - Key: "hello", - Hash: "example", - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textId, - }) - require.NoError(t, err) + clientId := createClientWithCollector(t, queries) + documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID) contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, @@ -521,8 +500,8 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + clientId := createClientWithCollector(t, queries) + documentID, _, textId := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID) contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID) require.NoError(t, err) @@ -581,9 +560,9 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, cleanId, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, cleanId, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ Bucket: "hi", @@ -610,28 +589,14 @@ func TestClientSync(t *testing.T) { fields: `{"first_key": null}`, }) - contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "updated_text_context", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: queryId, Value: "updated_text_json", Textentryid: textId, Queryversion: 1, }) require.NoError(t, err) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultID, - Requiredresultid: contextResultID, - }) - require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, @@ -651,9 +616,9 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) bucket := "cu" key := "kk" @@ -700,28 +665,14 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "update_clean_context", - Textentryid: textThreeId, - Queryversion: 1, - }) - require.NoError(t, err) - - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: queryId, Value: "update_clean_json", Textentryid: textThreeId, Queryversion: 1, }) require.NoError(t, err) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultID, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, @@ -731,7 +682,7 @@ func TestClientSync(t *testing.T) { }) }) - t.Run("change collector name", func(t *testing.T) { + t.Run("add new collector version", func(t *testing.T) { t.Parallel() ctx := context.Background() @@ -741,13 +692,78 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) + + _, err := queries.AddLatestCollectorVersion(ctx, clientId) + require.NoError(t, err) + + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) + }) + + t.Run("add new active collector version", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) require.NoError(t, err) + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: latestCollectorVersion, + Clientid: clientId, + }) + require.NoError(t, err) + + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) + }) + + t.Run("remove query from new collector version", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) + + latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) + require.NoError(t, err) + + err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ + Clientid: clientId, + Queryid: queryId, + Removedversion: &latestCollectorVersion, + }) + require.NoError(t, err) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, @@ -762,21 +778,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ - clientId: clientId, - documentId: documentID, - hash: "example_hash", - isSynced: true, - fields: `{"first_key": "json_value"}`, - }) - - err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Removedversion: &latestCollectorVersion, - }) - require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, @@ -784,22 +785,6 @@ func TestClientSync(t *testing.T) { isSynced: true, fields: `{}`, }) - - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Addedversion: latestCollectorVersion, - Name: "second_key", - }) - require.NoError(t, err) - - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ - clientId: clientId, - documentId: documentID, - hash: "example_hash", - isSynced: true, - fields: `{"second_key": "json_value"}`, - }) }) t.Run("add existing query", func(t *testing.T) { @@ -813,8 +798,8 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + clientId := createClientWithCollector(t, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID) err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Clientid: clientId, @@ -843,13 +828,13 @@ func TestClientSync(t *testing.T) { queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) - clientId := createClientWithCollector(t, ctx, queries) - documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) + clientId := createClientWithCollector(t, queries) + documentID, _, textId := createDocumentWithCollectorAndResult(t, queries, clientId, queryId) superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor) err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ - Queryid: contextQueryID, + Queryid: queryId, Requiredqueryid: superQueryID, Addedversion: 1, }) @@ -871,16 +856,8 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "context_with_super_value", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, + resultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: queryId, Value: "json_with_super", Textentryid: textId, Queryversion: 1, @@ -888,25 +865,11 @@ func TestClientSync(t *testing.T) { require.NoError(t, err) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: contextResultId, + Resultid: resultId, Requiredresultid: superResultId, }) require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ - clientId: clientId, - documentId: documentID, - hash: "example_hash", - isSynced: false, - fields: `{"first_key": null}`, - }) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultId, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - getDocumentSyncState(t, ctx, queries, docSyncStateParams{ clientId: clientId, documentId: documentID, @@ -917,25 +880,31 @@ func TestClientSync(t *testing.T) { }) } -func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) { - err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ +func createDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) uuid.UUID { + err := queries.AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{ Clientid: clientId, Queryid: jsonQueryID, Addedversion: 1, Name: "first_key", }) require.NoError(t, err) - bucket := "example_bucket" - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + documentID, err := queries.CreateDocument(t.Context(), &repository.CreateDocumentParams{ Clientid: clientId, Hash: "example_hash", }) require.NoError(t, err) + return documentID +} + +func createCleanDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID) { + documentID := createDocumentWithCollector(t, queries, clientId, jsonQueryID) + + bucket := "example_bucket" key := "example_key" hash := "hahssh" - cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + cleanId, err := queries.AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{ Documentid: documentID, Bucket: &bucket, Key: &key, @@ -946,13 +915,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu }, }) require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + err = queries.AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{ Cleanid: cleanId, Version: 1, }) require.NoError(t, err) - textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + return documentID, cleanId +} + +func createDocumentWithCollectorAndText(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) { + documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, jsonQueryID) + + textId, err := queries.AddDocumentText(t.Context(), &repository.AddDocumentTextParams{ Cleanid: cleanId, Part: 0, Createdat: pgtype.Timestamp{ @@ -964,21 +939,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu Hash: "example", }) require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + err = queries.AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{ Version: 1, Textid: textId, }) require.NoError(t, err) - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "example_context", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) + return documentID, cleanId, textId +} - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ +func createDocumentWithCollectorAndResult(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) { + documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID) + + _, err := queries.AddResult(t.Context(), &repository.AddResultParams{ Queryid: jsonQueryID, Value: "json_value", Textentryid: textId, @@ -986,7 +959,29 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu }) require.NoError(t, err) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + return documentID, cleanId, textId +} + +func createDocumentWithCollectorAndResults(t testing.TB, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) { + documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID) + + contextResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_context", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + jsonResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "json_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + err = queries.AddResultDependency(t.Context(), &repository.AddResultDependencyParams{ Resultid: jsonResultId, Requiredresultid: contextResultId, }) @@ -1005,8 +1000,8 @@ func BenchmarkIsClientSynced(b *testing.B) { queries := cfg.GetDBQueries() contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) - clientId := createClientWithCollector(b, ctx, queries) - _, _, _ = createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + clientId := createClientWithCollector(b, queries) + _, _, _ = createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID) b.ResetTimer() @@ -1025,8 +1020,8 @@ func BenchmarkGetDocumentExternal(b *testing.B) { queries := cfg.GetDBQueries() contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) - clientId := createClientWithCollector(b, ctx, queries) - documentID, _, _ := createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + clientId := createClientWithCollector(b, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID) b.ResetTimer() diff --git a/scripts/tests.yml b/scripts/tests.yml index c687be4c..8a42bcfd 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -6,6 +6,7 @@ version: "3" vars: COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out" BENCH_FILE: "{{.OUT_DIR}}/bench.out" + TEST_FILE: "{{.OUT_DIR}}/test.out" INTERNAL: "./internal/..." API: "./api/..." PKG: "./pkg/..." @@ -44,6 +45,33 @@ tasks: -coverprofile={{.COVERAGE_FILE}} \ ./... - task: coverage + perf: + vars: + NUM_RESULTS: 5 + cmds: + - GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -json ./... + - | + echo -e "🐢 TOP {{.NUM_RESULTS}} SLOWEST INDIVIDUAL PACKAGE RUNS:" + jq -r 'select(.Action == "pass" and .Test == null) | "\(.Elapsed)s \(.Package)"' "{{.TEST_FILE}}" | \ + sort -rn | head -n {{.NUM_RESULTS}} + + echo -e "\nšŸ“‰ TOP {{.NUM_RESULTS}} SLOWEST TESTS:" + jq -r 'select(.Action == "pass" and .Test != null) | "\(.Elapsed)s \(.Test) (\(.Package))"' "{{.TEST_FILE}}" | \ + sort -rn | head -n {{.NUM_RESULTS}} + + echo -e "\nšŸ“¦ TOP {{.NUM_RESULTS}} SLOWEST PACKAGES (TOTAL TIME):" + jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" | + awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' | + sort -rn | head -n {{.NUM_RESULTS}} + + SLOWEST_PKG=$(jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" | + awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' | + sort -rn | head -n 1 | awk '{print $2}') + + echo -e "\nšŸ”Ž TOP {{.NUM_RESULTS}} SLOWEST TESTS IN SLOWEST PACKAGE ($SLOWEST_PKG):" + jq -r --arg pkg "$SLOWEST_PKG" \ + 'select(.Action == "pass" and .Test != null and .Package == $pkg) | "\(.Elapsed)s \(.Test)"' "{{.TEST_FILE}}" | + sort -rn | head -n {{.NUM_RESULTS}} bench: vars: TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" @@ -65,37 +93,44 @@ tasks: TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out" cmds: - - | - grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} \ - > {{.FILTER_COVERAGE_FILE}} + - grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} > {{.FILTER_COVERAGE_FILE}} - go tool cover -func={{.FILTER_COVERAGE_FILE}} > {{.TMP_FILE}} - | - COVERAGE=$(grep total: {{.TMP_FILE}} \ - | awk '{print $3}' | sed 's/%//' | bc) + COVERAGE=$(grep total: {{.TMP_FILE}} | awk '{print $NF}' | sed 's/%//') echo "Coverage Threshold: {{.COVERAGE_THRESHOLD}}%" echo "Total Coverage: $COVERAGE%" + FAIL=false if (( $(echo "$COVERAGE < {{.COVERAGE_THRESHOLD}}" | bc -l) )); then echo "āŒ Error: Total Coverage below Coverage Threshold" FAIL=true fi echo "Function Coverage Threshold: {{.FUNCTION_COVERAGE_THRESHOLD}}%" - FAIL_FUNC=false - grep -v total: {{.TMP_FILE}} | while read -r line; do - FUNC_COV=$(echo "$line" | awk '{print $3}' | sed 's/%//' | bc) - if (( $(echo "$FUNC_COV < \ - {{.FUNCTION_COVERAGE_THRESHOLD}}" | bc -l) )); then - if [ "$FAIL_FUNC" = false ]; then - echo "āŒ Error: The following functions have coverage below "\ - "{{.FUNCTION_COVERAGE_THRESHOLD}}%:" - FAIL_FUNC=true - FAIL=true - fi - echo "$line" - fi - done + awk -v threshold="{{.FUNCTION_COVERAGE_THRESHOLD}}" ' + !/total:/ { + line = $0; + coverage = $NF; + gsub(/%/, "", coverage); + + if (coverage + 0 < threshold + 0) { + if (!printed_header) { + print "āŒ Error: The following functions have coverage below " threshold "%:"; + printed_header = 1; + } + print line; + exit_code = 1; + } + } + END { + exit exit_code; + } + ' {{.TMP_FILE}} + + if [ $? -eq 1 ]; then + FAIL=true + fi if [ "$FAIL" = true ]; then exit 1