diff --git a/api/queryAPI/api.gen.go b/api/queryAPI/api.gen.go index daebee3b..6c843e1e 100644 --- a/api/queryAPI/api.gen.go +++ b/api/queryAPI/api.gen.go @@ -872,13 +872,13 @@ var swaggerSpec = []string{ "mSPcnDX/Bze/d5ont7fNUfWPEDZKRwg6SUumohVwfnX5XilVKkTR61klq5BVp6U1wnY7vcHGwt5tikwZ", "I7Q8sbSEY2xlQ6RyuFrf2/aQ9R91mPX/CX1ygFJ0dKDBnJFcxbYSUFgsa4HkI5saOFCZHGKx2UAbxZmf", "8tDFIhBCGw0N6mKpGSeRqRIelCAviKaqf50pqDQ0Y8pJCvEy6w2dgG82u6eDB7OEGuc3iYn37obVSm5C", - "mL8Ulr2KrG//+pxI4ARXn4K5a2p7Pvlyw1TsNM6KM3jbXu/wzOs+WV9na5rKe/Z1aLbUa5BwzcWBjTm4", - "v9nUYB8nS9krY9swcO8zsv0FTpkMu/qNfPtK5Fu3+hUekUDvFqxb4yfLSbepU+wJbO1V3zobeyvB74H6", - "lSx8ju+Vx9mNSpfJxavNeLYZoF6DZrt3fF5OsjW0WPcbFMlPitm7YsnVvS14t8bS32i3r0C7XRtJ29Je", - "E36BI1SmKu/0TwxoIDYXi2NqDkfrspb8790USDEg5N7dIntvusY3ChelN+HFdPYhrL3bXCWtkVHE4RtV", - "bVdsGKXS5a6l++IP1VXGr5wFsZ/cANRXF+OKi5s+i9oPXe1adoxiT1+cc6gsOtRoqpJozWIQadEvT4so", - "lxNrunFM0GxPRXboup2lnNGkr9Kh97p9pTsm21Oi+XV7MCeamR7yR5nrdpO/x5r2li0WPN89/18AAAD/", - "/1+CAiY6ZwAA", + "mL8Ulr2KrG//+pxI4ARXn4K5a2p7Pvlyw1TsNM6KM3jbXu/wzOs+WV9na5rKm9tWr0HENZcHNubh/mbT", + "g32cLmWvjW3Dwr3PyPYXOGkyDOs3Au4rEXDd6td4RQLBW7Bvja8sJ9+mjrEn0LVXfuvs7K0UvwcKWLLw", + "Od5XGW83KmMml7A249xmAHsNyu3ecXo54dZQZN3vUSQ/L2bvjSXX+Lbg4Bprf6PgvgIFdyNEbUt7bfgF", + "zlCZtrzTPzmgAdlcNI6pOSyty2Dyv39TIMmAkHt3jew96hr/KFyc3oQn09mHsPauc5W0RkYRh2/UtV2x", + "Y5RKV7uX7o8/VFcev3IWxH5yK1BfZ4wrLnP6LGo/dLV72XGKPX1xDqKy6lCjqkqqNbNBpIXAPFWiXGKs", + "6caxQ7M9FRmj63aW8kiTvkoH4ev2le6ibE857a/biznpzPSSP+Jct5v8/da0t2wR4fnu+f8CAAD//wtA", + "8jpSZwAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/cmd/metricsExample_test/main.go b/cmd/metricsExample_test/main.go index 165b2e80..fc4ab402 100644 --- a/cmd/metricsExample_test/main.go +++ b/cmd/metricsExample_test/main.go @@ -109,7 +109,7 @@ func simulateMetrics(ctx context.Context, metrics *prometheus.Metrics) { eventTypes := []string{"scheduled", "triggered", "manual"} priorities := []string{"low", "medium", "high"} - ticker := time.NewTicker(2 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { diff --git a/deployments/compose.aws.yaml b/deployments/compose.aws.yaml index 765f4961..ed4ff130 100644 --- a/deployments/compose.aws.yaml +++ b/deployments/compose.aws.yaml @@ -29,6 +29,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -60,6 +65,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -91,6 +101,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -122,6 +137,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -158,6 +178,11 @@ services: AWS_ENDPOINT_URL_S3: "http://localstack:4566" AWS_ENDPOINT_URL_SQS: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -189,6 +214,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -220,6 +250,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -251,6 +286,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -282,6 +322,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network @@ -313,6 +358,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - aws-server-network diff --git a/deployments/compose.local.yaml b/deployments/compose.local.yaml index 9fb2f162..d6403fea 100644 --- a/deployments/compose.local.yaml +++ b/deployments/compose.local.yaml @@ -29,6 +29,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -60,6 +65,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -91,6 +101,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -122,6 +137,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -153,6 +173,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -184,6 +209,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -215,6 +245,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -246,6 +281,15 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" + # COGNITO_USER_POOL_ID: ${COGNITO_USER_POOL_ID} + # COGNITO_CLIENT_SECRET: ${COGNITO_CLIENT_SECRET} + # COGNITO_DOMAIN: ${COGNITO_DOMAIN} + # COGNITO_CLIENT_ID: ${COGNITO_CLIENT_ID} networks: - server-network @@ -277,6 +321,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network @@ -308,6 +357,11 @@ services: DB_NOSSL: ${DB_NOSSL} AWS_ENDPOINT_URL: "http://localstack:4566" AWS_S3_USE_PATH_STYLE: true + DISABLE_AUTH: true + COGNITO_USER_POOL_ID: "fillin" + COGNITO_CLIENT_SECRET: "fillin" + COGNITO_DOMAIN: "fillin" + COGNITO_CLIENT_ID: "fillin" networks: - server-network diff --git a/devbox.json b/devbox.json index f198d95d..e59d7a13 100644 --- a/devbox.json +++ b/devbox.json @@ -60,7 +60,8 @@ "go@1.24.0", "sqlcheck@1.3", "docker-client@27.3.1", - "gotestsum@latest" + "gotestsum@latest", + "goperf@latest" ], "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index d8bff3be..cec0778f 100644 --- a/devbox.lock +++ b/devbox.lock @@ -401,6 +401,54 @@ } } }, + "goperf@latest": { + "last_modified": "2025-04-10T20:20:34Z", + "resolved": "github:NixOS/nixpkgs/d19cf9dfc633816a437204555afeb9e722386b76#goperf", + "source": "devbox-search", + "version": "0-unstable-2025-04-07", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/vf411izylvc8fq1m91ri8303xv1mha5d-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/vf411izylvc8fq1m91ri8303xv1mha5d-goperf-0-unstable-2025-04-07" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/3f2ap46w6bshaf24s0d48rswm3rl12l6-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/3f2ap46w6bshaf24s0d48rswm3rl12l6-goperf-0-unstable-2025-04-07" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hp9gdz3y082isszsg6zcz8d6g0l1gk59-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/hp9gdz3y082isszsg6zcz8d6g0l1gk59-goperf-0-unstable-2025-04-07" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ardq6yb76al8kp45jlqa59by17l2a58y-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/ardq6yb76al8kp45jlqa59by17l2a58y-goperf-0-unstable-2025-04-07" + } + } + }, "gotestsum@latest": { "last_modified": "2025-04-05T00:48:53Z", "resolved": "github:NixOS/nixpkgs/250b695f41e0e2f5afbf15c6b12480de1fe0001b#gotestsum", diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go index e9c12ec4..456e824f 100644 --- a/internal/database/repository/sync_test.go +++ b/internal/database/repository/sync_test.go @@ -242,7 +242,7 @@ func TestClientSync(t *testing.T) { }, docExternal) }) - t.Run("valid extraction", func(t *testing.T) { + t.Run("standard extraction", func(t *testing.T) { t.Parallel() ctx := context.Background() @@ -349,72 +349,11 @@ func TestClientSync(t *testing.T) { Fields: []byte(`{"first_key": null}`), }, docExternal) - t.Run("document standard results", func(t *testing.T) { - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "example_context", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "json_value", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultId, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_value"}`), - }, docExternal) - }) - - contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: contextQueryID, - Versionid: contextLatestVersion, + contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_context", + Textentryid: textId, + Queryversion: 1, }) require.NoError(t, err) @@ -430,480 +369,739 @@ func TestClientSync(t *testing.T) { Fields: []byte(`{"first_key": null}`), }, docExternal) - t.Run("update upstream query", func(t *testing.T) { - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "context_version_2", - Textentryid: textId, - Queryversion: contextLatestVersion, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "updated_context_value", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultId, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "updated_context_value"}`), - }, docExternal) + jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "json_value", + Textentryid: textId, + Queryversion: 1, }) + require.NoError(t, err) - t.Run("update text entry", func(t *testing.T) { - textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Bucket: "hi", - Key: "hello", - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - Cleanid: cleanId, - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textId, - }) - require.NoError(t, err) + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "updated_text_context", - Textentryid: textId, - Queryversion: 2, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "updated_text_json", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultID, - Requiredresultid: contextResultID, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "updated_text_json"}`), - }, docExternal) + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultId, + Requiredresultid: contextResultId, }) - t.Run("updated clean entry", func(t *testing.T) { - cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ - Documentid: documentID, - Bucket: &bucket, - Key: &key, - Hash: &hash, - Mimetype: repository.NullCleanmimetype{ - Valid: true, - Cleanmimetype: repository.CleanmimetypeApplicationPdf, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ - Cleanid: cleanthreeid, - Version: 1, - }) - require.NoError(t, err) + require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) - textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Bucket: "hi", - Key: "hello", - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - Cleanid: cleanthreeid, - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textThreeId, - }) - require.NoError(t, err) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "json_value"}`), + }, docExternal) + }) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + t.Run("update upstream query", func(t *testing.T) { + t.Parallel() + ctx := context.Background() - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "update_clean_context", - Textentryid: textThreeId, - Queryversion: contextLatestVersion, - }) - require.NoError(t, err) + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + queries := cfg.GetDBQueries() - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "update_clean_json", - Textentryid: textThreeId, - Queryversion: 1, - }) - require.NoError(t, err) + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultID, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "update_clean_json"}`), - }, docExternal) - - latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "update_clean_json"}`), - }, docExternal) - - t.Run("update collector name", func(t *testing.T) { - err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Versionid: latestCollectorVersion, - Clientid: clientId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "update_clean_json"}`), - }, docExternal) - - err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Removedversion: &latestCollectorVersion, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{}`), - }, docExternal) - - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Addedversion: latestCollectorVersion, - Name: "second_key", - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": "update_clean_json"}`), - }, docExternal) - }) - - t.Run("add existing query", func(t *testing.T) { - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: contextQueryID, - Addedversion: latestCollectorVersion, - Name: "example_key", - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": "update_clean_json", "example_key": "update_clean_context"}`), - }, docExternal) - }) - - t.Run("add super query", func(t *testing.T) { - superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) - require.NoError(t, err) - superQueryVersion, err := queries.AddLatestQueryVersion(ctx, superQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: superQueryID, - Versionid: superQueryVersion, - }) - require.NoError(t, err) - err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ - Queryid: contextQueryID, - Requiredqueryid: superQueryID, - Addedversion: superQueryVersion, - }) - require.NoError(t, err) - - superResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: superQueryID, - Value: "super_value", - Textentryid: textThreeId, - Queryversion: superQueryVersion, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": null, "example_key": null}`), - }, docExternal) - - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "context_with_super_value", - Textentryid: textThreeId, - Queryversion: contextLatestVersion, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": null, "example_key": null}`), - }, docExternal) - - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "json_with_super", - Textentryid: textThreeId, - Queryversion: 1, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": null, "example_key": null}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: contextResultId, - Requiredresultid: superResultId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": null, "example_key": "context_with_super_value"}`), - }, docExternal) - - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ - Resultid: jsonResultId, - Requiredresultid: contextResultId, - }) - require.NoError(t, err) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": "json_with_super", "example_key": "context_with_super_value"}`), - }, docExternal) - }) + contextLatestVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID) + require.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: contextQueryID, + Versionid: contextLatestVersion, }) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "context_version_2", + Textentryid: textId, + Queryversion: contextLatestVersion, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "updated_context_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultId, + Requiredresultid: contextResultId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "updated_context_value"}`), + }, docExternal) + }) + + t.Run("update text extry", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, cleanId, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Bucket: "hi", + Key: "hello", + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + Cleanid: cleanId, + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textId, + }) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "updated_text_context", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "updated_text_json", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultID, + Requiredresultid: contextResultID, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "updated_text_json"}`), + }, docExternal) + }) + + t.Run("update clean entry", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + bucket := "cu" + key := "kk" + hash := "hash" + cleanthreeid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + Documentid: documentID, + Bucket: &bucket, + Key: &key, + Hash: &hash, + Mimetype: repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Cleanid: cleanthreeid, + Version: 1, + }) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Bucket: "hi", + Key: "hello", + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + Cleanid: cleanthreeid, + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textThreeId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "update_clean_context", + Textentryid: textThreeId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "update_clean_json", + Textentryid: textThreeId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultID, + Requiredresultid: contextResultId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "update_clean_json"}`), + }, docExternal) + }) + + t.Run("change collector name", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "json_value"}`), + }, docExternal) + + err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ + Versionid: latestCollectorVersion, + Clientid: clientId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "json_value"}`), + }, docExternal) + + err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ + Clientid: clientId, + Queryid: jsonQueryID, + Removedversion: &latestCollectorVersion, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{}`), + }, docExternal) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Queryid: jsonQueryID, + Addedversion: latestCollectorVersion, + Name: "second_key", + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"second_key": "json_value"}`), + }, docExternal) + }) + + t.Run("add existing query", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Queryid: contextQueryID, + Addedversion: 1, + Name: "example_key", + }) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "json_value", "example_key": "example_context"}`), + }, docExternal) + }) + + t.Run("add super query", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(t, ctx) + _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) + require.NoError(t, err) + superQueryVersion, err := queries.AddLatestQueryVersion(ctx, superQueryID) + require.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: superQueryID, + Versionid: superQueryVersion, + }) + require.NoError(t, err) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: contextQueryID, + Requiredqueryid: superQueryID, + Addedversion: superQueryVersion, + }) + require.NoError(t, err) + + superResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: superQueryID, + Value: "super_value", + Textentryid: textId, + Queryversion: superQueryVersion, + }) + require.NoError(t, err) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err := queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "context_with_super_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "json_with_super", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: contextResultId, + Requiredresultid: superResultId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.False(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": null}`), + }, docExternal) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultId, + Requiredresultid: contextResultId, + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + docExternal, err = queries.GetDocumentExternal(ctx, documentID) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: documentID, + Clientid: clientId, + Hash: "example_hash", + Fields: []byte(`{"first_key": "json_with_super"}`), + }, docExternal) }) } + +func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, queries *repository.Queries, clientId string, contextQueryID, jsonQueryID uuid.UUID) (uuid.UUID, uuid.UUID, uuid.UUID) { + err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Queryid: jsonQueryID, + Addedversion: 1, + Name: "first_key", + }) + require.NoError(t, err) + bucket := "example_bucket" + + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientId, + Hash: "example_hash", + }) + require.NoError(t, err) + + key := "example_key" + hash := "hahssh" + cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + Documentid: documentID, + Bucket: &bucket, + Key: &key, + Hash: &hash, + Mimetype: repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Cleanid: cleanId, + Version: 1, + }) + require.NoError(t, err) + + textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Cleanid: cleanId, + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + Bucket: "hi", + Key: "hello", + Hash: "example", + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textId, + }) + require.NoError(t, err) + + contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "example_context", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "json_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ + Resultid: jsonResultId, + Requiredresultid: contextResultId, + }) + require.NoError(t, err) + + return documentID, cleanId, textId +} + +func BenchmarkIsClientSynced(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(b, ctx) + _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) + clientId := createClientWithCollector(b, ctx, queries) + _, _, _ = createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + + b.ResetTimer() + + for b.Loop() { + _, _ = queries.IsClientSynced(ctx, &clientId) + } +} + +func BenchmarkGetDocumentExternal(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(b, ctx) + _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) + clientId := createClientWithCollector(b, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + + b.ResetTimer() + + for b.Loop() { + _, _ = queries.GetDocumentExternal(ctx, documentID) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 3a5de0de..6f91eb0e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -63,7 +63,7 @@ func New(ctx context.Context, cfg Config) (func() error, error) { return nil, err } - cfg.SetThreadPool(cfg.GetLogger()) + cfg.StartThreadPool(cfg.GetLogger()) return closeTracer, nil } diff --git a/internal/serviceconfig/build/build.go b/internal/serviceconfig/build/build.go index 5f036660..e9f5175a 100644 --- a/internal/serviceconfig/build/build.go +++ b/internal/serviceconfig/build/build.go @@ -25,16 +25,16 @@ var startupTime time.Time var startupTimeOnce sync.Once func GetVersionTimestamp() time.Time { - startupTimeOnce.Do(getStartTime(&startupTime, debug.ReadBuildInfo)) + startupTimeOnce.Do(getStartTime(&startupTime, debug.ReadBuildInfo, time.Now)) return startupTime } -func getStartTime(startupTime *time.Time, readBuildInfo func() (info *debug.BuildInfo, ok bool)) func() { +func getStartTime(startupTime *time.Time, readBuildInfo func() (info *debug.BuildInfo, ok bool), clock func() time.Time) func() { return func() { info, ok := readBuildInfo() if !ok { - *startupTime = time.Now().UTC() + *startupTime = clock() return } @@ -52,7 +52,7 @@ func getStartTime(startupTime *time.Time, readBuildInfo func() (info *debug.Buil } // Return current time if no build time found - *startupTime = time.Now().UTC() + *startupTime = clock() } } diff --git a/internal/serviceconfig/build/build_test.go b/internal/serviceconfig/build/build_test.go index 849c6b20..79eccf6b 100644 --- a/internal/serviceconfig/build/build_test.go +++ b/internal/serviceconfig/build/build_test.go @@ -51,27 +51,30 @@ func TestGetVersionTimestamp(t *testing.T) { } func TestGetStartTime(t *testing.T) { + injectedTime := time.Now().UTC() + injectTime := func() time.Time { + return injectedTime + } + vcsTime := time.Now().UTC() t.Run("vcs time", func(t *testing.T) { var tt time.Time - expectedTime := time.Now().UTC() f := getStartTime(&tt, func() (info *debug.BuildInfo, ok bool) { return &debug.BuildInfo{ Settings: []debug.BuildSetting{ { Key: "vcs.time", - Value: expectedTime.Format(time.RFC3339), + Value: vcsTime.Format(time.RFC3339), }, }, }, true - }) + }, injectTime) assert.NotNil(t, f) f() - assert.Equal(t, expectedTime.Truncate(time.Second), tt) + assert.Equal(t, vcsTime.Truncate(time.Second), tt) }) t.Run("vcs time - invalid format", func(t *testing.T) { var tt time.Time // to make the test less likely to fail due to non deterministic temporal issues. - expectedTime := time.Now().UTC().Add(-time.Millisecond) f := getStartTime(&tt, func() (info *debug.BuildInfo, ok bool) { return &debug.BuildInfo{ Settings: []debug.BuildSetting{ @@ -81,10 +84,27 @@ func TestGetStartTime(t *testing.T) { }, }, }, true - }) + }, injectTime) assert.NotNil(t, f) f() - assert.True(t, expectedTime.Before(tt)) + assert.Equal(t, injectedTime, tt) + }) + t.Run("vcs time - not available", func(t *testing.T) { + var tt time.Time + // to make the test less likely to fail due to non deterministic temporal issues. + f := getStartTime(&tt, func() (info *debug.BuildInfo, ok bool) { + return &debug.BuildInfo{ + Settings: []debug.BuildSetting{ + { + Key: "vcs.time", + Value: "invalid", + }, + }, + }, true + }, injectTime) + assert.NotNil(t, f) + f() + assert.Equal(t, injectedTime, tt) }) } diff --git a/internal/serviceconfig/database/pool.go b/internal/serviceconfig/database/pool.go index dbd8a1cf..53110c92 100644 --- a/internal/serviceconfig/database/pool.go +++ b/internal/serviceconfig/database/pool.go @@ -76,7 +76,7 @@ func (b *DBConfig) SetDBPool(ctx context.Context) error { func (b *DBConfig) DBPing(ctx context.Context) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() for { diff --git a/internal/serviceconfig/objectstore/config.go b/internal/serviceconfig/objectstore/config.go index 7d66c671..dfed5469 100644 --- a/internal/serviceconfig/objectstore/config.go +++ b/internal/serviceconfig/objectstore/config.go @@ -75,7 +75,7 @@ func (c *ObjectStoreConfig) GetStoreClient() S3Client { func (c *ObjectStoreConfig) PingStoreByName(ctx context.Context, name string) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() input := &s3.HeadBucketInput{ diff --git a/internal/serviceconfig/queue/config.go b/internal/serviceconfig/queue/config.go index 4533e44c..d9891089 100644 --- a/internal/serviceconfig/queue/config.go +++ b/internal/serviceconfig/queue/config.go @@ -42,7 +42,7 @@ func (c *QueueConfig) GetSQSEndpoint() string { func (c *QueueConfig) PingQueueByURL(ctx context.Context, url string) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() input := &sqs.GetQueueAttributesInput{ diff --git a/internal/serviceconfig/threadpool/config.go b/internal/serviceconfig/threadpool/config.go index 2c625a39..5e03cfb9 100644 --- a/internal/serviceconfig/threadpool/config.go +++ b/internal/serviceconfig/threadpool/config.go @@ -13,7 +13,7 @@ type ConfigProvider interface { GetThreadMonitorInterval() int GetThreadPool() pond.Pool ThreadPoolStopAndWait() - SetThreadPool(*slog.Logger) + StartThreadPool(*slog.Logger) } type ThreadPoolConfig struct { @@ -30,11 +30,11 @@ type PoolMetrics struct { ProcessingTime time.Duration } -func (dp *ThreadPoolConfig) SetThreadPool(logger *slog.Logger) { +func (dp *ThreadPoolConfig) StartThreadPool(logger *slog.Logger) { dp.logger = logger if dp.MaxWorkers <= 0 { - dp.MaxWorkers = runtime.NumCPU() * 10000 + dp.MaxWorkers = runtime.NumCPU() * 10 } if dp.MonitorInterval <= 0 { @@ -85,7 +85,7 @@ func (dp *ThreadPoolConfig) GetThreadMonitorInterval() int { func (dp *ThreadPoolConfig) GetThreadPool() pond.Pool { if dp.pool == nil { - dp.SetThreadPool(slog.Default()) + dp.StartThreadPool(slog.Default()) } return dp.pool diff --git a/internal/serviceconfig/threadpool/config_test.go b/internal/serviceconfig/threadpool/config_test.go index 13dd3d19..9103a0ec 100644 --- a/internal/serviceconfig/threadpool/config_test.go +++ b/internal/serviceconfig/threadpool/config_test.go @@ -16,9 +16,9 @@ func TestSetThreadPool(t *testing.T) { } sl := slog.New(l) - cfg.SetThreadPool(sl) + cfg.StartThreadPool(sl) assert.Equal(t, 120, cfg.MonitorInterval) - assert.LessOrEqual(t, 10000, cfg.MaxWorkers) + assert.LessOrEqual(t, 10, cfg.MaxWorkers) assert.NotNil(t, cfg.pool) } @@ -30,9 +30,9 @@ func TestStopAndWait(t *testing.T) { sl := slog.New(l) cfg.MonitorInterval = 1 - cfg.SetThreadPool(sl) + cfg.StartThreadPool(sl) assert.Equal(t, 1, cfg.MonitorInterval) - assert.LessOrEqual(t, 10000, cfg.MaxWorkers) + assert.LessOrEqual(t, 10, cfg.MaxWorkers) assert.NotNil(t, cfg.pool) for i := range 1000 { diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index 9c9b8ed7..97951efd 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -310,12 +310,13 @@ func WaitForMockEndpoint(t testing.TB, server MockServer, request MockRequest) M verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) + clientTimeout := 500 * time.Millisecond client := &http.Client{ - Timeout: 5 * time.Second, + Timeout: clientTimeout, } timeout := time.After(60 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) + ticker := time.NewTicker(clientTimeout) defer ticker.Stop() slog.Info("Attempting to process request", "body", jsonData) diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index 9916da74..aa7d6d34 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -8,100 +8,10 @@ import ( "strings" "testing" - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/aws" - "queryorchestration/internal/serviceconfig/objectstore" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestCreateAPINetwork(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - cfg := &FullDepsConfig{} - - conn, cleanup := CreateAPINetwork(t, ctx, cfg, QueryAPI) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -type FullDepsConfig struct { - aws.AWSConfig - serviceconfig.BaseConfig - objectstore.ObjectStoreConfig -} - -func TestCreateFullDependencies(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - cfg := &FullDepsConfig{} - - conn, cleanup := CreateFullDependencies(t, ctx, cfg) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateFullNetwork(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - cfg := &FullDepsConfig{} - - conn, cleanup := CreateFullNetwork(t, ctx, cfg) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateMockServer(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - net := DepNetwork.Get(t, ctx) - - server, cleanup := CreateMockServer(t, ctx, net) - - assert.NotNil(t, server) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateMockExpectation(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - server := MockServer{ - Client: ts.Client(), - External: Address(ts.URL), - } - expectation := MockExpectation{} - CreateMockExpectation(t, server, expectation) -} - func TestCreateDetectDocumentTextExpectation(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/internal/test/queue.go b/internal/test/queue.go index 2013265f..729b4263 100644 --- a/internal/test/queue.go +++ b/internal/test/queue.go @@ -68,7 +68,7 @@ func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProv func AssertMessage(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, params *queue.ReceiveParams) types.Message { t.Helper() timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() for { diff --git a/scripts/tests.yml b/scripts/tests.yml index 506df603..44b7fd27 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -5,11 +5,12 @@ version: "3" vars: COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out" + BENCH_FILE: "{{.OUT_DIR}}/bench.out" INTERNAL: "./internal/..." API: "./api/..." PKG: "./pkg/..." CMD: "./cmd/..." - UNIT_PARALLEL: 16 + UNIT_PARALLEL: 32 E2E_PARALLEL: 1 # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go" @@ -115,3 +116,10 @@ tasks: -coverprofile={{.COVERAGE_FILE}} \ ./... - task: coverage + bench: + vars: + TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" + FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out" + cmds: + - go test -run ^Benchmark -bench=. ./... > {{.BENCH_FILE}} + - benchstat {{.BENCH_FILE}} diff --git a/serviceAPIs/queryAPI.yaml b/serviceAPIs/queryAPI.yaml index 856f6e54..1582c3a7 100644 --- a/serviceAPIs/queryAPI.yaml +++ b/serviceAPIs/queryAPI.yaml @@ -21,7 +21,7 @@ tags: description: Operations related to collectors - name: DocumentsService description: Operations related to documents - - name: QueryAPI + - name: QueryService description: Operations related to queries - name: ExportService description: Operations related to exports @@ -127,7 +127,7 @@ paths: get: operationId: listQueries tags: - - QueryAPI + - QueryService summary: List queries description: Retrieves a list of queries based on filter criteria. security: @@ -153,7 +153,7 @@ paths: post: operationId: createQuery tags: - - QueryAPI + - QueryService summary: Create a new query description: Creates a new query with the provided details. security: @@ -190,7 +190,7 @@ paths: get: operationId: getQuery tags: - - QueryAPI + - QueryService summary: Get a query by ID description: Retrieves a specific query by its ID. security: @@ -216,7 +216,7 @@ paths: patch: operationId: updateQuery tags: - - QueryAPI + - QueryService summary: Update a query description: Updates an existing query with new details. security: @@ -249,7 +249,7 @@ paths: post: operationId: testQuery tags: - - QueryAPI + - QueryService summary: Test a query description: Executes a test run of a query with the provided parameters. security: @@ -496,7 +496,6 @@ paths: "500": $ref: "#/components/responses/InternalError" - /login: get: operationId: login @@ -629,9 +628,9 @@ paths: schema: type: string description: HTML content for the home page menu - maxLength: 1048576 # 1MB limit - pattern: "^.*$" # Any string content - format: html # Indicate format is HTML + maxLength: 1048576 + pattern: "^.*$" + format: html "400": $ref: "#/components/responses/InvalidRequest" "401": @@ -691,8 +690,8 @@ components: scheme: bearer bearerFormat: JWT description: >- - JWT token obtained after Cognito authentication. Implements RFC8725 JWT - best practices for security. + JWT token obtained after Cognito authentication. Implements RFC8725 JWT + best practices for security. cognitoAuth: type: oauth2 description: >- @@ -759,8 +758,6 @@ components: $ref: "#/components/schemas/ErrorMessage" schemas: - - # Query API schemas ClientID: type: string