jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splittest (pull request #137)

Browse Source 
Split Slow DB Test

* split
This commit is contained in:
Michael McGuinness
2025-05-09 23:49:02 +00:00
parent c4c0b9bd32
commit b5c30b4af7
4 changed files with 270 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deployments/compose.generate.yaml
+1 -1
View File
@@ -12,7 +12,7 @@ services:
- 5431
healthcheck:
test: ["CMD", "sh", "-c", "pg_isready -U ${PGUSER} -h localhost -p 5432 && psql -U ${PGUSER} -c 'SELECT 1'"]
interval: 500ms
interval: 50ms
timeout: 5s
retries: 50
networks:
internal/database/repository/result_test.go
+10 -10
View File
@@ -28,13 +28,13 @@ func TestResults(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, t.Context(), queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
qv := int32(1)
res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: &jsonQueryID,
Queryid: &queryId,
Queryversion: &qv,
Documentid: &documentID,
})
@@ -271,7 +271,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
queries := cfg.GetDBQueries()
clientId := createClientWithCollector(t, ctx, queries)
clientId := createClientWithCollector(t, queries)
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
@@ -387,9 +387,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
queries := cfg.GetDBQueries()
clientId := createClientWithCollector(t, ctx, queries)
clientId := createClientWithCollector(t, queries)
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientId,
@@ -484,9 +484,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
queries := cfg.GetDBQueries()
clientId := createClientWithCollector(t, ctx, queries)
clientId := createClientWithCollector(t, queries)
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
_, err := queries.AddLatestQueryVersion(ctx, jsonQueryID)
require.NoError(t, err)
@@ -512,7 +512,7 @@ func BenchmarkListUnsynced(b *testing.B) {
queries := cfg.GetDBQueries()
clientId := createClientWithCollector(b, ctx, queries)
clientId := createClientWithCollector(b, queries)
_, jsonQueryID := createDependentQueries(b, ctx, queries)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
internal/database/repository/sync_test.go
+205 -210
View File
@@ -172,17 +172,17 @@ func createDependentQueries(t testing.TB, ctx context.Context, queries *reposito
return contextQueryID, jsonQueryID
}
func createClientWithCollector(t testing.TB, ctx context.Context, queries *repository.Queries) string {
func createClientWithCollector(t testing.TB, queries *repository.Queries) string {
clientId := "EXAMPLE"
err := queries.CreateClient(ctx, &repository.CreateClientParams{
err := queries.CreateClient(t.Context(), &repository.CreateClientParams{
Name: "example_client",
Clientid: clientId,
})
require.NoError(t, err)
collectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
collectorVersion, err := queries.AddLatestCollectorVersion(t.Context(), clientId)
require.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
err = queries.SetActiveCollectorVersion(t.Context(), &repository.SetActiveCollectorVersionParams{
Versionid: collectorVersion,
Clientid: clientId,
})
@@ -229,7 +229,7 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
_, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
clientId := createClientWithCollector(t, queries)
isSynced, err := queries.IsClientSynced(ctx, &clientId)
require.NoError(t, err)
@@ -248,7 +248,7 @@ func TestClientSync(t *testing.T) {
assert.True(t, isSynced)
})
t.Run("document fail clean", func(t *testing.T) {
t.Run("create document", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
@@ -258,12 +258,12 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
_, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Clientid: clientId,
Queryid: jsonQueryID,
Queryid: queryId,
Addedversion: 1,
Name: "first_key",
})
@@ -282,6 +282,21 @@ func TestClientSync(t *testing.T) {
isSynced: false,
fields: `{"first_key": null}`,
})
})
t.Run("document fail clean", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
net := test.GetNetwork(t)
test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID := createDocumentWithCollector(t, queries, clientId, queryId)
cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: documentID,
@@ -300,13 +315,13 @@ func TestClientSync(t *testing.T) {
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_noclean",
hash: "example_hash",
isSynced: true,
fields: `{"first_key": null}`,
})
})
t.Run("single query", func(t *testing.T) {
t.Run("clean document", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
@@ -317,24 +332,10 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, ctx, queries)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Clientid: clientId,
Queryid: queryId,
Addedversion: 1,
Name: "first_key",
})
require.NoError(t, err)
bucket := "example_bucket"
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientId,
Hash: "example_hash",
})
require.NoError(t, err)
clientId := createClientWithCollector(t, queries)
documentID := createDocumentWithCollector(t, queries, clientId, queryId)
bucket := "buck"
key := "example_key"
hash := "hahssh"
cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
@@ -361,6 +362,21 @@ func TestClientSync(t *testing.T) {
isSynced: false,
fields: `{"first_key": null}`,
})
})
t.Run("extracted text", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
net := test.GetNetwork(t)
test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, queryId)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanId,
@@ -387,8 +403,23 @@ func TestClientSync(t *testing.T) {
isSynced: false,
fields: `{"first_key": null}`,
})
})
_, err = queries.AddResult(ctx, &repository.AddResultParams{
t.Run("single query", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
net := test.GetNetwork(t)
test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, queryId)
_, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: queryId,
Value: "json_value",
Textentryid: textId,
@@ -416,60 +447,8 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Clientid: clientId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "first_key",
})
require.NoError(t, err)
bucket := "example_bucket"
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientId,
Hash: "example_hash",
})
require.NoError(t, err)
key := "example_key"
hash := "hahssh"
cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
Documentid: documentID,
Bucket: &bucket,
Key: &key,
Hash: &hash,
Mimetype: repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.CleanmimetypeApplicationPdf,
},
})
require.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Cleanid: cleanId,
Version: 1,
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Cleanid: cleanId,
Part: 0,
Createdat: pgtype.Timestamp{
Time: time.Now().UTC(),
Valid: true,
},
Bucket: "hi",
Key: "hello",
Hash: "example",
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
})
require.NoError(t, err)
clientId := createClientWithCollector(t, queries)
documentID, _, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
@@ -521,8 +500,8 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
clientId := createClientWithCollector(t, queries)
documentID, _, textId := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID)
require.NoError(t, err)
@@ -581,9 +560,9 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, cleanId, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, cleanId, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: "hi",
@@ -610,28 +589,14 @@ func TestClientSync(t *testing.T) {
fields: `{"first_key": null}`,
})
contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "updated_text_context",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: queryId,
Value: "updated_text_json",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: jsonResultID,
Requiredresultid: contextResultID,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
@@ -651,9 +616,9 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
bucket := "cu"
key := "kk"
@@ -700,28 +665,14 @@ func TestClientSync(t *testing.T) {
})
require.NoError(t, err)
contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "update_clean_context",
Textentryid: textThreeId,
Queryversion: 1,
})
require.NoError(t, err)
jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: queryId,
Value: "update_clean_json",
Textentryid: textThreeId,
Queryversion: 1,
})
require.NoError(t, err)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: jsonResultID,
Requiredresultid: contextResultId,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
@@ -731,7 +682,7 @@ func TestClientSync(t *testing.T) {
})
})
t.Run("change collector name", func(t *testing.T) {
t.Run("add new collector version", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
@@ -741,13 +692,78 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
_, err := queries.AddLatestCollectorVersion(ctx, clientId)
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_hash",
isSynced: true,
fields: `{"first_key": "json_value"}`,
})
})
t.Run("add new active collector version", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
net := test.GetNetwork(t)
test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
require.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: latestCollectorVersion,
Clientid: clientId,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_hash",
isSynced: true,
fields: `{"first_key": "json_value"}`,
})
})
t.Run("remove query from new collector version", func(t *testing.T) {
t.Parallel()
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
net := test.GetNetwork(t)
test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{})
queries := cfg.GetDBQueries()
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId)
require.NoError(t, err)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Clientid: clientId,
Queryid: queryId,
Removedversion: &latestCollectorVersion,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
@@ -762,21 +778,6 @@ func TestClientSync(t *testing.T) {
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_hash",
isSynced: true,
fields: `{"first_key": "json_value"}`,
})
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Clientid: clientId,
Queryid: jsonQueryID,
Removedversion: &latestCollectorVersion,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
@@ -784,22 +785,6 @@ func TestClientSync(t *testing.T) {
isSynced: true,
fields: `{}`,
})
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Clientid: clientId,
Queryid: jsonQueryID,
Addedversion: latestCollectorVersion,
Name: "second_key",
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_hash",
isSynced: true,
fields: `{"second_key": "json_value"}`,
})
})
t.Run("add existing query", func(t *testing.T) {
@@ -813,8 +798,8 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
clientId := createClientWithCollector(t, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(t, queries, clientId, contextQueryID, jsonQueryID)
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Clientid: clientId,
@@ -843,13 +828,13 @@ func TestClientSync(t *testing.T) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries)
clientId := createClientWithCollector(t, ctx, queries)
documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID)
queryId := createQuery(t, queries, repository.QuerytypeContextFull)
clientId := createClientWithCollector(t, queries)
documentID, _, textId := createDocumentWithCollectorAndResult(t, queries, clientId, queryId)
superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor)
err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: contextQueryID,
Queryid: queryId,
Requiredqueryid: superQueryID,
Addedversion: 1,
})
@@ -871,16 +856,8 @@ func TestClientSync(t *testing.T) {
})
require.NoError(t, err)
contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "context_with_super_value",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
resultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: queryId,
Value: "json_with_super",
Textentryid: textId,
Queryversion: 1,
@@ -888,25 +865,11 @@ func TestClientSync(t *testing.T) {
require.NoError(t, err)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: contextResultId,
Resultid: resultId,
Requiredresultid: superResultId,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
hash: "example_hash",
isSynced: false,
fields: `{"first_key": null}`,
})
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: jsonResultId,
Requiredresultid: contextResultId,
})
require.NoError(t, err)
getDocumentSyncState(t, ctx, queries, docSyncStateParams{
clientId: clientId,
documentId: documentID,
@@ -917,25 +880,31 @@ func TestClientSync(t *testing.T) {
})
}
func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
func createDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) uuid.UUID {
err := queries.AddCollectorQuery(t.Context(), &repository.AddCollectorQueryParams{
Clientid: clientId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "first_key",
})
require.NoError(t, err)
bucket := "example_bucket"
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
documentID, err := queries.CreateDocument(t.Context(), &repository.CreateDocumentParams{
Clientid: clientId,
Hash: "example_hash",
})
require.NoError(t, err)
return documentID
}
func createCleanDocumentWithCollector(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID) {
documentID := createDocumentWithCollector(t, queries, clientId, jsonQueryID)
bucket := "example_bucket"
key := "example_key"
hash := "hahssh"
cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{
cleanId, err := queries.AddDocumentClean(t.Context(), &repository.AddDocumentCleanParams{
Documentid: documentID,
Bucket: &bucket,
Key: &key,
@@ -946,13 +915,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
},
})
require.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
err = queries.AddDocumentCleanEntry(t.Context(), &repository.AddDocumentCleanEntryParams{
Cleanid: cleanId,
Version: 1,
})
require.NoError(t, err)
textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
return documentID, cleanId
}
func createDocumentWithCollectorAndText(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
documentID, cleanId := createCleanDocumentWithCollector(t, queries, clientId, jsonQueryID)
textId, err := queries.AddDocumentText(t.Context(), &repository.AddDocumentTextParams{
Cleanid: cleanId,
Part: 0,
Createdat: pgtype.Timestamp{
@@ -964,21 +939,19 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
Hash: "example",
})
require.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
err = queries.AddDocumentTextEntry(t.Context(), &repository.AddDocumentTextEntryParams{
Version: 1,
Textid: textId,
})
require.NoError(t, err)
contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_context",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
return documentID, cleanId, textId
}
jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{
func createDocumentWithCollectorAndResult(t testing.TB, queries *repository.Queries, clientId string, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
_, err := queries.AddResult(t.Context(), &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "json_value",
Textentryid: textId,
@@ -986,7 +959,29 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu
})
require.NoError(t, err)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
return documentID, cleanId, textId
}
func createDocumentWithCollectorAndResults(t testing.TB, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) {
documentID, cleanId, textId := createDocumentWithCollectorAndText(t, queries, clientId, jsonQueryID)
contextResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_context",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
jsonResultId, err := queries.AddResult(t.Context(), &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "json_value",
Textentryid: textId,
Queryversion: 1,
})
require.NoError(t, err)
err = queries.AddResultDependency(t.Context(), &repository.AddResultDependencyParams{
Resultid: jsonResultId,
Requiredresultid: contextResultId,
})
@@ -1005,8 +1000,8 @@ func BenchmarkIsClientSynced(b *testing.B) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
clientId := createClientWithCollector(b, ctx, queries)
_, _, _ = createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID)
clientId := createClientWithCollector(b, queries)
_, _, _ = createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
b.ResetTimer()
@@ -1025,8 +1020,8 @@ func BenchmarkGetDocumentExternal(b *testing.B) {
queries := cfg.GetDBQueries()
contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries)
clientId := createClientWithCollector(b, ctx, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID)
clientId := createClientWithCollector(b, queries)
documentID, _, _ := createDocumentWithCollectorAndResults(b, queries, clientId, contextQueryID, jsonQueryID)
b.ResetTimer()
scripts/tests.yml
+54 -19
View File
@@ -6,6 +6,7 @@ version: "3"
vars:
COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out"
BENCH_FILE: "{{.OUT_DIR}}/bench.out"
TEST_FILE: "{{.OUT_DIR}}/test.out"
INTERNAL: "./internal/..."
API: "./api/..."
PKG: "./pkg/..."
@@ -44,6 +45,33 @@ tasks:
-coverprofile={{.COVERAGE_FILE}} \
./...
- task: coverage
perf:
vars:
NUM_RESULTS: 5
cmds:
- GOMAXPROCS={{.TEST_PARALLEL}} go test -parallel {{.CPU_COUNT}} -json ./...
- |
echo -e "🐢 TOP {{.NUM_RESULTS}} SLOWEST INDIVIDUAL PACKAGE RUNS:"
jq -r 'select(.Action == "pass" and .Test == null) | "\(.Elapsed)s \(.Package)"' "{{.TEST_FILE}}" | \
sort -rn | head -n {{.NUM_RESULTS}}
echo -e "\n📉 TOP {{.NUM_RESULTS}} SLOWEST TESTS:"
jq -r 'select(.Action == "pass" and .Test != null) | "\(.Elapsed)s \(.Test) (\(.Package))"' "{{.TEST_FILE}}" | \
sort -rn | head -n {{.NUM_RESULTS}}
echo -e "\n📦 TOP {{.NUM_RESULTS}} SLOWEST PACKAGES (TOTAL TIME):"
jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" |
awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' |
sort -rn | head -n {{.NUM_RESULTS}}
SLOWEST_PKG=$(jq -r 'select(.Action == "pass") | [.Package, .Elapsed] | @tsv' "{{.TEST_FILE}}" |
awk '{pkg[$1] += $2} END {for (p in pkg) print pkg[p] "s " p}' |
sort -rn | head -n 1 | awk '{print $2}')
echo -e "\n🔎 TOP {{.NUM_RESULTS}} SLOWEST TESTS IN SLOWEST PACKAGE ($SLOWEST_PKG):"
jq -r --arg pkg "$SLOWEST_PKG" \
'select(.Action == "pass" and .Test != null and .Package == $pkg) | "\(.Elapsed)s \(.Test)"' "{{.TEST_FILE}}" |
sort -rn | head -n {{.NUM_RESULTS}}
bench:
vars:
TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
@@ -65,37 +93,44 @@ tasks:
TMP_FILE: "{{.OUT_DIR}}/coverage.tmp"
FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out"
cmds:
- |
grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} \
> {{.FILTER_COVERAGE_FILE}}
- grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} > {{.FILTER_COVERAGE_FILE}}
- go tool cover -func={{.FILTER_COVERAGE_FILE}} > {{.TMP_FILE}}
- |
COVERAGE=$(grep total: {{.TMP_FILE}} \
| awk '{print $3}' | sed 's/%//' | bc)
COVERAGE=$(grep total: {{.TMP_FILE}} | awk '{print $NF}' | sed 's/%//')
echo "Coverage Threshold: {{.COVERAGE_THRESHOLD}}%"
echo "Total Coverage: $COVERAGE%"
FAIL=false
if (( $(echo "$COVERAGE < {{.COVERAGE_THRESHOLD}}" | bc -l) )); then
echo "❌ Error: Total Coverage below Coverage Threshold"
FAIL=true
fi
echo "Function Coverage Threshold: {{.FUNCTION_COVERAGE_THRESHOLD}}%"
FAIL_FUNC=false
grep -v total: {{.TMP_FILE}} | while read -r line; do
FUNC_COV=$(echo "$line" | awk '{print $3}' | sed 's/%//' | bc)
if (( $(echo "$FUNC_COV < \
{{.FUNCTION_COVERAGE_THRESHOLD}}" | bc -l) )); then
if [ "$FAIL_FUNC" = false ]; then
echo "❌ Error: The following functions have coverage below "\
"{{.FUNCTION_COVERAGE_THRESHOLD}}%:"
FAIL_FUNC=true
FAIL=true
fi
echo "$line"
fi
done
awk -v threshold="{{.FUNCTION_COVERAGE_THRESHOLD}}" '
!/total:/ {
line = $0;
coverage = $NF;
gsub(/%/, "", coverage);
if (coverage + 0 < threshold + 0) {
if (!printed_header) {
print "❌ Error: The following functions have coverage below " threshold "%:";
printed_header = 1;
}
print line;
exit_code = 1;
}
}
END {
exit exit_code;
}
' {{.TMP_FILE}}
if [ $? -eq 1 ]; then
FAIL=true
fi
if [ "$FAIL" = true ]; then
exit 1