From af361b4a67b3614af738a2e9a40431e0a094940d Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Tue, 29 Apr 2025 17:13:09 +0000 Subject: [PATCH] Merged in feature/testclean (pull request #126) Bit of test clean up * fixes --- cmd/metricsExample_test/main.go | 2 +- devbox.json | 3 +- devbox.lock | 48 +++++++++++ internal/database/repository/sync_test.go | 40 +++++++++ internal/serviceconfig/database/pool.go | 2 +- internal/serviceconfig/objectstore/config.go | 2 +- internal/serviceconfig/queue/config.go | 2 +- internal/test/ecosystem.go | 5 +- internal/test/ecosystem_test.go | 90 -------------------- internal/test/queue.go | 2 +- scripts/tests.yml | 10 ++- 11 files changed, 107 insertions(+), 99 deletions(-) diff --git a/cmd/metricsExample_test/main.go b/cmd/metricsExample_test/main.go index 165b2e80..fc4ab402 100644 --- a/cmd/metricsExample_test/main.go +++ b/cmd/metricsExample_test/main.go @@ -109,7 +109,7 @@ func simulateMetrics(ctx context.Context, metrics *prometheus.Metrics) { eventTypes := []string{"scheduled", "triggered", "manual"} priorities := []string{"low", "medium", "high"} - ticker := time.NewTicker(2 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) defer ticker.Stop() for { diff --git a/devbox.json b/devbox.json index f198d95d..e59d7a13 100644 --- a/devbox.json +++ b/devbox.json @@ -60,7 +60,8 @@ "go@1.24.0", "sqlcheck@1.3", "docker-client@27.3.1", - "gotestsum@latest" + "gotestsum@latest", + "goperf@latest" ], "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index d8bff3be..cec0778f 100644 --- a/devbox.lock +++ b/devbox.lock @@ -401,6 +401,54 @@ } } }, + "goperf@latest": { + "last_modified": "2025-04-10T20:20:34Z", + "resolved": "github:NixOS/nixpkgs/d19cf9dfc633816a437204555afeb9e722386b76#goperf", + "source": "devbox-search", + "version": "0-unstable-2025-04-07", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/vf411izylvc8fq1m91ri8303xv1mha5d-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/vf411izylvc8fq1m91ri8303xv1mha5d-goperf-0-unstable-2025-04-07" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/3f2ap46w6bshaf24s0d48rswm3rl12l6-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/3f2ap46w6bshaf24s0d48rswm3rl12l6-goperf-0-unstable-2025-04-07" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/hp9gdz3y082isszsg6zcz8d6g0l1gk59-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/hp9gdz3y082isszsg6zcz8d6g0l1gk59-goperf-0-unstable-2025-04-07" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/ardq6yb76al8kp45jlqa59by17l2a58y-goperf-0-unstable-2025-04-07", + "default": true + } + ], + "store_path": "/nix/store/ardq6yb76al8kp45jlqa59by17l2a58y-goperf-0-unstable-2025-04-07" + } + } + }, "gotestsum@latest": { "last_modified": "2025-04-05T00:48:53Z", "resolved": "github:NixOS/nixpkgs/250b695f41e0e2f5afbf15c6b12480de1fe0001b#gotestsum", diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go index ed045332..456e824f 100644 --- a/internal/database/repository/sync_test.go +++ b/internal/database/repository/sync_test.go @@ -1065,3 +1065,43 @@ func createDocumentWithCollectorAndResults(t testing.TB, ctx context.Context, qu return documentID, cleanId, textId } + +func BenchmarkIsClientSynced(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(b, ctx) + _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) + clientId := createClientWithCollector(b, ctx, queries) + _, _, _ = createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + + b.ResetTimer() + + for b.Loop() { + _, _ = queries.IsClientSynced(ctx, &clientId) + } +} + +func BenchmarkGetDocumentExternal(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.DepNetwork.Get(b, ctx) + _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(b, ctx, queries) + clientId := createClientWithCollector(b, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(b, ctx, queries, clientId, contextQueryID, jsonQueryID) + + b.ResetTimer() + + for b.Loop() { + _, _ = queries.GetDocumentExternal(ctx, documentID) + } +} diff --git a/internal/serviceconfig/database/pool.go b/internal/serviceconfig/database/pool.go index dbd8a1cf..53110c92 100644 --- a/internal/serviceconfig/database/pool.go +++ b/internal/serviceconfig/database/pool.go @@ -76,7 +76,7 @@ func (b *DBConfig) SetDBPool(ctx context.Context) error { func (b *DBConfig) DBPing(ctx context.Context) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() for { diff --git a/internal/serviceconfig/objectstore/config.go b/internal/serviceconfig/objectstore/config.go index 7d66c671..dfed5469 100644 --- a/internal/serviceconfig/objectstore/config.go +++ b/internal/serviceconfig/objectstore/config.go @@ -75,7 +75,7 @@ func (c *ObjectStoreConfig) GetStoreClient() S3Client { func (c *ObjectStoreConfig) PingStoreByName(ctx context.Context, name string) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() input := &s3.HeadBucketInput{ diff --git a/internal/serviceconfig/queue/config.go b/internal/serviceconfig/queue/config.go index 4533e44c..d9891089 100644 --- a/internal/serviceconfig/queue/config.go +++ b/internal/serviceconfig/queue/config.go @@ -42,7 +42,7 @@ func (c *QueueConfig) GetSQSEndpoint() string { func (c *QueueConfig) PingQueueByURL(ctx context.Context, url string) error { timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() input := &sqs.GetQueueAttributesInput{ diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index 9c9b8ed7..97951efd 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -310,12 +310,13 @@ func WaitForMockEndpoint(t testing.TB, server MockServer, request MockRequest) M verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) + clientTimeout := 500 * time.Millisecond client := &http.Client{ - Timeout: 5 * time.Second, + Timeout: clientTimeout, } timeout := time.After(60 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) + ticker := time.NewTicker(clientTimeout) defer ticker.Stop() slog.Info("Attempting to process request", "body", jsonData) diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index 9916da74..aa7d6d34 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -8,100 +8,10 @@ import ( "strings" "testing" - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/aws" - "queryorchestration/internal/serviceconfig/objectstore" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestCreateAPINetwork(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - cfg := &FullDepsConfig{} - - conn, cleanup := CreateAPINetwork(t, ctx, cfg, QueryAPI) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -type FullDepsConfig struct { - aws.AWSConfig - serviceconfig.BaseConfig - objectstore.ObjectStoreConfig -} - -func TestCreateFullDependencies(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - cfg := &FullDepsConfig{} - - conn, cleanup := CreateFullDependencies(t, ctx, cfg) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateFullNetwork(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - cfg := &FullDepsConfig{} - - conn, cleanup := CreateFullNetwork(t, ctx, cfg) - - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateMockServer(t *testing.T) { - t.Parallel() - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - net := DepNetwork.Get(t, ctx) - - server, cleanup := CreateMockServer(t, ctx, net) - - assert.NotNil(t, server) - assert.NotNil(t, cleanup) - - cleanup() -} - -func TestCreateMockExpectation(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - })) - defer ts.Close() - - server := MockServer{ - Client: ts.Client(), - External: Address(ts.URL), - } - expectation := MockExpectation{} - CreateMockExpectation(t, server, expectation) -} - func TestCreateDetectDocumentTextExpectation(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/internal/test/queue.go b/internal/test/queue.go index 2013265f..729b4263 100644 --- a/internal/test/queue.go +++ b/internal/test/queue.go @@ -68,7 +68,7 @@ func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProv func AssertMessage(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, params *queue.ReceiveParams) types.Message { t.Helper() timeout := time.After(30 * time.Second) - tick := time.NewTicker(2 * time.Second) + tick := time.NewTicker(500 * time.Millisecond) defer tick.Stop() for { diff --git a/scripts/tests.yml b/scripts/tests.yml index 506df603..44b7fd27 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -5,11 +5,12 @@ version: "3" vars: COVERAGE_FILE: "{{.OUT_DIR}}/coverage.out" + BENCH_FILE: "{{.OUT_DIR}}/bench.out" INTERNAL: "./internal/..." API: "./api/..." PKG: "./pkg/..." CMD: "./cmd/..." - UNIT_PARALLEL: 16 + UNIT_PARALLEL: 32 E2E_PARALLEL: 1 # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go" @@ -115,3 +116,10 @@ tasks: -coverprofile={{.COVERAGE_FILE}} \ ./... - task: coverage + bench: + vars: + TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" + FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out" + cmds: + - go test -run ^Benchmark -bench=. ./... > {{.BENCH_FILE}} + - benchstat {{.BENCH_FILE}}