From ee776d26814f6d0066ace355fb15d25c497ea096 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Tue, 6 May 2025 01:59:52 +0000 Subject: [PATCH] Merged in feature/mockserver (pull request #135) Single Mock Server * mockserver * mockserver * reqs * mockserver * slowrunner * someoptimisedqueries * passedfullsuite * passedfullsuite --- api/clientSyncRunner/runner_test.go | 1 - api/docCleanRunner/runner_test.go | 1 - api/docInitRunner/runner_test.go | 1 - api/docSyncRunner/runner_test.go | 1 - api/docTextRunner/runner_test.go | 1 - api/queryAPI/controllers_test.go | 14 - api/queryAPI/query_test.go | 1 - api/queryAPI/status_test.go | 2 + api/queryRunner/runner_test.go | 3 +- api/querySyncRunner/runner_test.go | 1 - api/queryVersionSyncRunner/runner_test.go | 1 - api/storeEventRunner/runner_test.go | 1 - internal/client/service_test.go | 24 -- internal/client/sync/service_test.go | 13 - internal/client/update/service_test.go | 24 -- internal/collector/get_test.go | 7 +- internal/collector/service_test.go | 24 -- internal/collector/set/service_test.go | 23 -- .../00000000000101_collector_views.up.sql | 64 +++- .../00000000000102_document_views.up.sql | 5 +- internal/database/queries/client.sql | 13 +- internal/database/queries/collector.sql | 3 +- internal/database/queries/query.sql | 9 +- internal/database/queries/result.sql | 12 +- internal/database/repository/client.sql.go | 26 +- internal/database/repository/collector.sql.go | 20 +- .../database/repository/collector_test.go | 71 ++++- internal/database/repository/query.sql.go | 22 +- internal/database/repository/query_test.go | 90 ++++-- internal/database/repository/result.sql.go | 24 +- internal/database/repository/sync_test.go | 36 +-- internal/document/service_test.go | 14 - internal/export/service_test.go | 14 - internal/query/result/processor/parse.go | 6 +- internal/query/result/processor/parse_test.go | 19 +- internal/query/result/service_test.go | 23 -- internal/query/result/set/set.go | 4 +- internal/query/result/set/set_test.go | 4 +- internal/query/service_test.go | 23 -- internal/query/test/service_test.go | 24 -- .../query/types/contextFull/creator_test.go | 2 - .../query/types/contextFull/updator_test.go | 2 - .../query/types/jsonExtractor/creator_test.go | 2 - .../query/types/jsonExtractor/updator_test.go | 2 - internal/query/update/service_test.go | 24 -- internal/query/versionsync/service_test.go | 16 - internal/server/runner/listener_test.go | 17 +- .../serviceconfig/threadpool/config_test.go | 2 +- internal/test/ecosystem.go | 278 ++---------------- internal/test/mockserver.go | 261 ++++++++++++++++ .../{ecosystem_test.go => mockserver_test.go} | 3 +- internal/test/network.go | 1 - internal/test/queryAPI/service.go | 39 +-- internal/test/queryAPI/service_test.go | 80 ----- scripts/Taskfile.yml | 2 +- test/process_test.go | 78 ++--- 56 files changed, 656 insertions(+), 822 deletions(-) delete mode 100644 api/queryAPI/controllers_test.go delete mode 100644 internal/client/service_test.go delete mode 100644 internal/client/sync/service_test.go delete mode 100644 internal/client/update/service_test.go delete mode 100644 internal/collector/service_test.go delete mode 100644 internal/collector/set/service_test.go delete mode 100644 internal/document/service_test.go delete mode 100644 internal/export/service_test.go delete mode 100644 internal/query/result/service_test.go delete mode 100644 internal/query/service_test.go delete mode 100644 internal/query/test/service_test.go delete mode 100644 internal/query/update/service_test.go delete mode 100644 internal/query/versionsync/service_test.go create mode 100644 internal/test/mockserver.go rename internal/test/{ecosystem_test.go => mockserver_test.go} (96%) delete mode 100644 internal/test/queryAPI/service_test.go diff --git a/api/clientSyncRunner/runner_test.go b/api/clientSyncRunner/runner_test.go index 2a4adb27..7d0fdb36 100644 --- a/api/clientSyncRunner/runner_test.go +++ b/api/clientSyncRunner/runner_test.go @@ -43,7 +43,6 @@ func TestQueryRunner(t *testing.T) { runner := clientsyncrunner.New(&clientsyncrunner.Services{ ClientSync: svc, }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { bod := clientsyncrunner.Body{ diff --git a/api/docCleanRunner/runner_test.go b/api/docCleanRunner/runner_test.go index 9f492f1c..f6e3f496 100644 --- a/api/docCleanRunner/runner_test.go +++ b/api/docCleanRunner/runner_test.go @@ -51,7 +51,6 @@ func TestDocCleanRunner(t *testing.T) { runner := doccleanrunner.New(&doccleanrunner.Services{ Clean: documentclean.New(cfg), }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { bod := doccleanrunner.Body{ diff --git a/api/docInitRunner/runner_test.go b/api/docInitRunner/runner_test.go index a0745a0a..22ddc454 100644 --- a/api/docInitRunner/runner_test.go +++ b/api/docInitRunner/runner_test.go @@ -43,7 +43,6 @@ func TestDocInitRunner(t *testing.T) { runner := docinitrunner.New(&docinitrunner.Services{ Document: documentinit.New(cfg), }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { clientId := "clientid" diff --git a/api/docSyncRunner/runner_test.go b/api/docSyncRunner/runner_test.go index 11a4add7..1fff9fd6 100644 --- a/api/docSyncRunner/runner_test.go +++ b/api/docSyncRunner/runner_test.go @@ -45,7 +45,6 @@ func TestDocInitRunner(t *testing.T) { Client: client.New(cfg), }), }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { j := client.Client{ diff --git a/api/docTextRunner/runner_test.go b/api/docTextRunner/runner_test.go index e5fba5cd..db72fdf3 100644 --- a/api/docTextRunner/runner_test.go +++ b/api/docTextRunner/runner_test.go @@ -58,7 +58,6 @@ func TestDocCleanRunner(t *testing.T) { runner := doctextrunner.New(&doctextrunner.Services{ Text: documenttext.New(cfg), }) - assert.NotNil(t, runner) doc := doctextrunner.Body{ DocumentID: uuid.New(), } diff --git a/api/queryAPI/controllers_test.go b/api/queryAPI/controllers_test.go deleted file mode 100644 index c189b211..00000000 --- a/api/queryAPI/controllers_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package queryapi_test - -import ( - "testing" - - queryapi "queryorchestration/api/queryAPI" - - "github.com/stretchr/testify/assert" -) - -func TestNewControllers(t *testing.T) { - cons := queryapi.NewControllers(&queryapi.Services{}) - assert.NotNil(t, cons) -} diff --git a/api/queryAPI/query_test.go b/api/queryAPI/query_test.go index 72bf0873..2382497e 100644 --- a/api/queryAPI/query_test.go +++ b/api/queryAPI/query_test.go @@ -105,7 +105,6 @@ func TestListQueries(t *testing.T) { var res queryapi.ListQueries err = json.Unmarshal(rec.Body.Bytes(), &res) require.NoError(t, err) - assert.NotNil(t, res.Queries) assert.ElementsMatch(t, res.Queries, []queryapi.Query{ { Id: id, diff --git a/api/queryAPI/status_test.go b/api/queryAPI/status_test.go index b32e7314..d30d57d9 100644 --- a/api/queryAPI/status_test.go +++ b/api/queryAPI/status_test.go @@ -10,6 +10,7 @@ import ( queryapi "queryorchestration/api/queryAPI" "queryorchestration/internal/client" "queryorchestration/internal/database/repository" + "queryorchestration/internal/export" "queryorchestration/internal/serviceconfig" "github.com/labstack/echo/v4" @@ -27,6 +28,7 @@ func TestGetClientStatus(t *testing.T) { cons := queryapi.NewControllers(&queryapi.Services{ Client: client.New(cfg), + Export: export.New(), }) e := echo.New() diff --git a/api/queryRunner/runner_test.go b/api/queryRunner/runner_test.go index 6b026aee..5356ed10 100644 --- a/api/queryRunner/runner_test.go +++ b/api/queryRunner/runner_test.go @@ -52,7 +52,6 @@ func TestQueryRunner(t *testing.T) { Sync: resultsync.New(cfg), }), }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { doc := queryrunner.Body{ @@ -112,7 +111,7 @@ func TestQueryRunner(t *testing.T) { pool.ExpectExec("name: AddResultDependency :exec").WithArgs(resultId, requiredResultId). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() - pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(query.ID, doc.DocumentID). + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(&query.ID, &doc.DocumentID). WillReturnRows( pgxmock.NewRows([]string{"queryId"}). AddRow(&reqQuery), diff --git a/api/querySyncRunner/runner_test.go b/api/querySyncRunner/runner_test.go index 77385146..134d8903 100644 --- a/api/querySyncRunner/runner_test.go +++ b/api/querySyncRunner/runner_test.go @@ -46,7 +46,6 @@ func TestQueryRunner(t *testing.T) { runner := querysyncrunner.New(&querysyncrunner.Services{ QuerySync: svc, }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { doc := querysyncrunner.Body{ diff --git a/api/queryVersionSyncRunner/runner_test.go b/api/queryVersionSyncRunner/runner_test.go index 9e857d52..4b4590d5 100644 --- a/api/queryVersionSyncRunner/runner_test.go +++ b/api/queryVersionSyncRunner/runner_test.go @@ -43,7 +43,6 @@ func TestQueryRunner(t *testing.T) { runner := queryversionsyncrunner.New(&queryversionsyncrunner.Services{ Sync: svc, }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { doc := queryversionsyncrunner.Body{ diff --git a/api/storeEventRunner/runner_test.go b/api/storeEventRunner/runner_test.go index dd0e586f..13b9bb88 100644 --- a/api/storeEventRunner/runner_test.go +++ b/api/storeEventRunner/runner_test.go @@ -42,7 +42,6 @@ func TestDocInitRunner(t *testing.T) { runner := New(&Services{ documentstore.New(cfg), }) - assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { clientId := "hi" diff --git a/internal/client/service_test.go b/internal/client/service_test.go deleted file mode 100644 index b5d049f2..00000000 --- a/internal/client/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package client_test - -import ( - "testing" - - "queryorchestration/internal/client" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := client.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/client/sync/service_test.go b/internal/client/sync/service_test.go deleted file mode 100644 index 6943132c..00000000 --- a/internal/client/sync/service_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package clientsync - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - svc := New(nil) - assert.NotNil(t, svc) - assert.Equal(t, svc.batchSize, int32(100)) -} diff --git a/internal/client/update/service_test.go b/internal/client/update/service_test.go deleted file mode 100644 index e836bc94..00000000 --- a/internal/client/update/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package clientupdate_test - -import ( - "testing" - - "queryorchestration/internal/client" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := client.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/collector/get_test.go b/internal/collector/get_test.go index 5fc08439..05a3702b 100644 --- a/internal/collector/get_test.go +++ b/internal/collector/get_test.go @@ -69,10 +69,15 @@ func TestListQueries(t *testing.T) { }, } + v := int32(2) + ty := repository.NullQuerytype{ + Querytype: repository.QuerytypeContextFull, + Valid: true, + } pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(clientId). WillReturnRows( pgxmock.NewRows([]string{"clientId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(clientId, &ogc[0].ID, repository.QuerytypeContextFull, int32(2), nil), + AddRow(&clientId, &ogc[0].ID, ty, &v, nil), ) qs, err := svc.ListQueries(ctx, clientId) diff --git a/internal/collector/service_test.go b/internal/collector/service_test.go deleted file mode 100644 index ece80fc5..00000000 --- a/internal/collector/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package collector_test - -import ( - "testing" - - "queryorchestration/internal/collector" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := collector.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/collector/set/service_test.go b/internal/collector/set/service_test.go deleted file mode 100644 index bc12c7ef..00000000 --- a/internal/collector/set/service_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package collectorset_test - -import ( - "testing" - - collectorset "queryorchestration/internal/collector/set" - "queryorchestration/internal/database/repository" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &CollectorSetConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := collectorset.New(cfg, &collectorset.Services{}) - assert.NotNil(t, svc) -} diff --git a/internal/database/migrations/00000000000101_collector_views.up.sql b/internal/database/migrations/00000000000101_collector_views.up.sql index 43e69534..adb3b265 100644 --- a/internal/database/migrations/00000000000101_collector_views.up.sql +++ b/internal/database/migrations/00000000000101_collector_views.up.sql @@ -1,12 +1,17 @@ -CREATE VIEW collectorCurrentActiveVersions as -SELECT DISTINCT - j.clientId, - coalesce( - (FIRST_VALUE(v.versionId) OVER (PARTITION BY j.clientId ORDER BY v.id DESC)), - 0 - )::int as activeVersion - FROM clients as j - LEFT JOIN collectorActiveVersions as v on v.clientId = j.clientId; +CREATE VIEW collectorCurrentActiveVersions AS +SELECT + c.clientId, + COALESCE(v.versionId, 0)::int AS activeVersion +FROM + clients c +LEFT JOIN ( + SELECT DISTINCT ON (clientId) + clientId, + versionId + FROM + collectorActiveVersions + ORDER BY clientId, id DESC +) v ON c.clientId = v.clientId; CREATE VIEW collectorLatestVersions as SELECT @@ -75,3 +80,44 @@ CREATE VIEW collectorQueryDependencyTree AS FROM collectorQueryDependencyTree as ct JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId JOIN queries as q on q.queryId = ct.queryId; + + +CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient( + _clientId varchar(255) +) +RETURNS TABLE ( + clientId varchar(255), + queryId uuid, + queryType queryType, + queryVersion int, + requiredIds uuid[] +) AS $$ +BEGIN + RETURN QUERY + WITH clientQueries as ( + SELECT q.clientId, q.queryId + FROM currentCollectorQueries as q + WHERE q.clientId = _clientId + ), + dependencyTree as ( + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.clientId, cq.queryId, ri.requiredIds + FROM clientQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId + + UNION ALL + + SELECT acq.clientId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) + ) + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.queryId = ct.queryId + ) + SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds + FROM dependencyTree as t + WHERE t.clientID is not null; +END; +$$ LANGUAGE plpgsql; diff --git a/internal/database/migrations/00000000000102_document_views.up.sql b/internal/database/migrations/00000000000102_document_views.up.sql index 75e89ee7..59b627f8 100644 --- a/internal/database/migrations/00000000000102_document_views.up.sql +++ b/internal/database/migrations/00000000000102_document_views.up.sql @@ -116,8 +116,8 @@ BEGIN ), tree as ( select q.queryId, q.requiredIds, q.clientId, q.queryVersion - from collectorQueryDependencyTree as q - join docs as d on q.clientId = d.clientId + from docs, + LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q ), query_dependency_tree AS ( WITH RECURSIVE query_deps AS ( @@ -135,7 +135,6 @@ BEGIN FROM query_deps qd JOIN tree q ON qd.requiredId = q.queryId WHERE array_length(q.requiredIds, 1) > 0 - ) SELECT query_deps.queryId, diff --git a/internal/database/queries/client.sql b/internal/database/queries/client.sql index 85256d55..f95287f9 100644 --- a/internal/database/queries/client.sql +++ b/internal/database/queries/client.sql @@ -33,7 +33,8 @@ doc_text_entries AS ( d.document_id, cte.id AS text_entry_id, d.clean_entry_id, - d.clean_fail + d.clean_fail, + d.client_id FROM doc_clean_entries d LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id @@ -41,16 +42,14 @@ doc_text_entries AS ( required_results AS ( -- All required document-query-version combinations SELECT - d.document_id, + dte.document_id, cqdt.queryId, cqdt.queryVersion, dte.text_entry_id FROM - doc_clean_entries d - JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id - JOIN doc_text_entries dte ON dte.document_id = d.document_id - and dte.text_entry_id IS NOT NULL - where d.clean_fail is null + doc_text_entries dte + JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id + where dte.clean_fail is null and dte.text_entry_id IS NOT NULL ), existing_results AS ( -- Valid results that exist diff --git a/internal/database/queries/collector.sql b/internal/database/queries/collector.sql index 6e946f16..226cad1e 100644 --- a/internal/database/queries/collector.sql +++ b/internal/database/queries/collector.sql @@ -1,5 +1,6 @@ -- name: ListCollectorQueries :many -SELECT * FROM collectorQueryDependencyTree WHERE clientId = @clientId; +SELECT clientId, queryId, queryType, queryVersion, requiredIds + FROM collectorQueryDependencyTreeByClient(@clientId); -- name: GetCollectorByClientID :one SELECT * FROM fullActiveCollectors WHERE clientId = @clientId LIMIT 1; diff --git a/internal/database/queries/query.sql b/internal/database/queries/query.sql index ef5ddc0e..cc2bc306 100644 --- a/internal/database/queries/query.sql +++ b/internal/database/queries/query.sql @@ -75,13 +75,12 @@ SELECT EXISTS ( -- name: ListQueryDirectDependentsByDocumentID :many WITH doc AS ( - SELECT id, clientId FROM documents where id = @documentId + SELECT clientId FROM documents WHERE id = @documentId ) SELECT dt.queryId - FROM doc as d - JOIN collectorQueryDependencyTree as dt - on d.clientId = dt.clientId - and @queryId = any(dt.requiredIds); +FROM doc, + LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt +WHERE @queryId = ANY(dt.requiredIds); -- name: ListQueryClientIDs :many SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1; diff --git a/internal/database/queries/result.sql b/internal/database/queries/result.sql index e1c718fa..a0313b4f 100644 --- a/internal/database/queries/result.sql +++ b/internal/database/queries/result.sql @@ -67,11 +67,19 @@ SELECT r.id, r.value WITH docs as ( SELECT id, clientId from documents where id = $1 ), +dependency_tree as ( + select q.queryId, q.requiredIds, q.clientId, q.queryVersion + from docs, + LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q +), +text_entries as ( + select id, documentId from currentTextEntries where documentId = $1 +), unsyncedQueries AS ( SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d - JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId - JOIN currentTextEntries as cte on cte.documentId = d.id + JOIN dependency_tree as dt on d.clientId = dt.clientId + JOIN text_entries as cte on cte.documentId = d.id LEFT JOIN results as r on r.queryId = dt.queryId and r.queryVersion = dt.queryVersion diff --git a/internal/database/repository/client.sql.go b/internal/database/repository/client.sql.go index 8a7d7477..5b58b81e 100644 --- a/internal/database/repository/client.sql.go +++ b/internal/database/repository/client.sql.go @@ -80,7 +80,8 @@ doc_text_entries AS ( d.document_id, cte.id AS text_entry_id, d.clean_entry_id, - d.clean_fail + d.clean_fail, + d.client_id FROM doc_clean_entries d LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id @@ -88,16 +89,14 @@ doc_text_entries AS ( required_results AS ( -- All required document-query-version combinations SELECT - d.document_id, + dte.document_id, cqdt.queryId, cqdt.queryVersion, dte.text_entry_id FROM - doc_clean_entries d - JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id - JOIN doc_text_entries dte ON dte.document_id = d.document_id - and dte.text_entry_id IS NOT NULL - where d.clean_fail is null + doc_text_entries dte + JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id + where dte.clean_fail is null and dte.text_entry_id IS NOT NULL ), existing_results AS ( -- Valid results that exist @@ -178,7 +177,8 @@ SELECT ( // d.document_id, // cte.id AS text_entry_id, // d.clean_entry_id, -// d.clean_fail +// d.clean_fail, +// d.client_id // FROM // doc_clean_entries d // LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id @@ -186,16 +186,14 @@ SELECT ( // required_results AS ( // -- All required document-query-version combinations // SELECT -// d.document_id, +// dte.document_id, // cqdt.queryId, // cqdt.queryVersion, // dte.text_entry_id // FROM -// doc_clean_entries d -// JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id -// JOIN doc_text_entries dte ON dte.document_id = d.document_id -// and dte.text_entry_id IS NOT NULL -// where d.clean_fail is null +// doc_text_entries dte +// JOIN collectorQueryDependencyTreeByClient($1) cqdt ON cqdt.clientId = dte.client_id +// where dte.clean_fail is null and dte.text_entry_id IS NOT NULL // ), // existing_results AS ( // -- Valid results that exist diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go index 66a092a6..5fea2f78 100644 --- a/internal/database/repository/collector.sql.go +++ b/internal/database/repository/collector.sql.go @@ -71,21 +71,31 @@ func (q *Queries) GetCollectorByClientID(ctx context.Context, clientid string) ( } const listCollectorQueries = `-- name: ListCollectorQueries :many -SELECT clientid, queryid, querytype, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 +SELECT clientId, queryId, queryType, queryVersion, requiredIds + FROM collectorQueryDependencyTreeByClient($1) ` +type ListCollectorQueriesRow struct { + Clientid *string `db:"clientid"` + Queryid *uuid.UUID `db:"queryid"` + Querytype NullQuerytype `db:"querytype"` + Queryversion *int32 `db:"queryversion"` + Requiredids []uuid.UUID `db:"requiredids"` +} + // ListCollectorQueries // -// SELECT clientid, queryid, querytype, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 -func (q *Queries) ListCollectorQueries(ctx context.Context, clientid string) ([]*Collectorquerydependencytree, error) { +// SELECT clientId, queryId, queryType, queryVersion, requiredIds +// FROM collectorQueryDependencyTreeByClient($1) +func (q *Queries) ListCollectorQueries(ctx context.Context, clientid string) ([]*ListCollectorQueriesRow, error) { rows, err := q.db.Query(ctx, listCollectorQueries, clientid) if err != nil { return nil, err } defer rows.Close() - items := []*Collectorquerydependencytree{} + items := []*ListCollectorQueriesRow{} for rows.Next() { - var i Collectorquerydependencytree + var i ListCollectorQueriesRow if err := rows.Scan( &i.Clientid, &i.Queryid, diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index e5e4a2fc..d6652521 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -142,20 +142,28 @@ func TestCollector(t *testing.T) { qs, err := queries.ListCollectorQueries(ctx, clientId) require.NoError(t, err) assert.Len(t, qs, 2) - assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{ + jsonV := int32(1) + contextV := int32(0) + assert.ElementsMatch(t, []*repository.ListCollectorQueriesRow{ { - Clientid: clientId, + Clientid: &clientId, Queryid: &jsonId, - Queryversion: 1, - Querytype: repository.QuerytypeJsonExtractor, - Requiredids: []uuid.UUID{contextId}, + Queryversion: &jsonV, + Querytype: repository.NullQuerytype{ + Querytype: repository.QuerytypeJsonExtractor, + Valid: true, + }, + Requiredids: []uuid.UUID{contextId}, }, { - Clientid: clientId, + Clientid: &clientId, Queryid: &contextId, - Queryversion: 0, - Querytype: repository.QuerytypeContextFull, - Requiredids: []uuid.UUID{}, + Queryversion: &contextV, + Querytype: repository.NullQuerytype{ + Querytype: repository.QuerytypeContextFull, + Valid: true, + }, + Requiredids: []uuid.UUID{}, }, }, qs) @@ -212,3 +220,48 @@ func TestCollector(t *testing.T) { Fields: nil, }, coll) } + +func BenchmarkListCollectorQueries(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := "EXAMPLE" + err := queries.CreateClient(ctx, &repository.CreateClientParams{ + Name: "example_client", + Clientid: clientId, + }) + require.NoError(b, err) + + b.ResetTimer() + + for b.Loop() { + _, _ = queries.ListCollectorQueries(ctx, clientId) + } +} + +func BenchmarkGetCollectorByClientID(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := "EXAMPLE" + err := queries.CreateClient(ctx, &repository.CreateClientParams{ + Name: "example_client", + Clientid: clientId, + }) + require.NoError(b, err) + + b.ResetTimer() + for b.Loop() { + _, _ = queries.GetCollectorByClientID(ctx, clientId) + } +} diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index d525d6ba..345d96b1 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -346,30 +346,28 @@ func (q *Queries) ListQueryClientIDs(ctx context.Context, queryid *uuid.UUID) ([ const listQueryDirectDependentsByDocumentID = `-- name: ListQueryDirectDependentsByDocumentID :many WITH doc AS ( - SELECT id, clientId FROM documents where id = $2 + SELECT clientId FROM documents WHERE id = $2 ) SELECT dt.queryId - FROM doc as d - JOIN collectorQueryDependencyTree as dt - on d.clientId = dt.clientId - and $1 = any(dt.requiredIds) +FROM doc, + LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt +WHERE $1 = ANY(dt.requiredIds) ` type ListQueryDirectDependentsByDocumentIDParams struct { - Queryid uuid.UUID `db:"queryid"` - Documentid uuid.UUID `db:"documentid"` + Queryid *uuid.UUID `db:"queryid"` + Documentid *uuid.UUID `db:"documentid"` } // ListQueryDirectDependentsByDocumentID // // WITH doc AS ( -// SELECT id, clientId FROM documents where id = $2 +// SELECT clientId FROM documents WHERE id = $2 // ) // SELECT dt.queryId -// FROM doc as d -// JOIN collectorQueryDependencyTree as dt -// on d.clientId = dt.clientId -// and $1 = any(dt.requiredIds) +// FROM doc, +// LATERAL collectorQueryDependencyTreeByClient(doc.clientId) AS dt +// WHERE $1 = ANY(dt.requiredIds) func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]*uuid.UUID, error) { rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid) if err != nil { diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 39b85d47..19d08bda 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -266,8 +266,8 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []uuid.UUID{}, dependents) @@ -283,8 +283,8 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []uuid.UUID{}, dependents) @@ -297,14 +297,14 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []uuid.UUID{}, dependents) @@ -318,14 +318,14 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents) @@ -341,14 +341,14 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents) @@ -361,20 +361,20 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: secondJsonQueryID, + Documentid: &docID, + Queryid: &secondJsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents) @@ -388,20 +388,20 @@ func TestQueryDependencyTree(t *testing.T) { require.NoError(t, err) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: secondJsonQueryID, + Documentid: &docID, + Queryid: &secondJsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: jsonQueryID, + Documentid: &docID, + Queryid: &jsonQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{&secondJsonQueryID}, dependents) dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: docID, - Queryid: contextQueryID, + Documentid: &docID, + Queryid: &contextQueryID, }) require.NoError(t, err) assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, dependents) @@ -590,3 +590,37 @@ func TestListQueryClients(t *testing.T) { require.NoError(t, err) assert.ElementsMatch(t, []string{clientOneID}, clients) } + +func BenchmarkListQueryDirectDependentsByDocId(b *testing.B) { + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientID := "EXAMPLE" + err := queries.CreateClient(ctx, &repository.CreateClientParams{ + Name: "example_client", + Clientid: clientID, + }) + require.NoError(b, err) + + docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientID, + Hash: "sample", + }) + require.NoError(b, err) + + contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) + require.NoError(b, err) + + b.ResetTimer() + for b.Loop() { + _, _ = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + Documentid: &docID, + Queryid: &contextQueryID, + }) + } +} diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index 5f296a2b..e3949114 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -234,11 +234,19 @@ const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDo WITH docs as ( SELECT id, clientId from documents where id = $1 ), +dependency_tree as ( + select q.queryId, q.requiredIds, q.clientId, q.queryVersion + from docs, + LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q +), +text_entries as ( + select id, documentId from currentTextEntries where documentId = $1 +), unsyncedQueries AS ( SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d - JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId - JOIN currentTextEntries as cte on cte.documentId = d.id + JOIN dependency_tree as dt on d.clientId = dt.clientId + JOIN text_entries as cte on cte.documentId = d.id LEFT JOIN results as r on r.queryId = dt.queryId and r.queryVersion = dt.queryVersion @@ -256,11 +264,19 @@ SELECT DISTINCT queryId FROM unsyncedQueries as baseuq // WITH docs as ( // SELECT id, clientId from documents where id = $1 // ), +// dependency_tree as ( +// select q.queryId, q.requiredIds, q.clientId, q.queryVersion +// from docs, +// LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q +// ), +// text_entries as ( +// select id, documentId from currentTextEntries where documentId = $1 +// ), // unsyncedQueries AS ( // SELECT DISTINCT dt.queryId, dt.requiredIds // from docs as d -// JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId -// JOIN currentTextEntries as cte on cte.documentId = d.id +// JOIN dependency_tree as dt on d.clientId = dt.clientId +// JOIN text_entries as cte on cte.documentId = d.id // LEFT JOIN results as r // on r.queryId = dt.queryId // and r.queryVersion = dt.queryVersion diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go index bcfa6b66..8176f0aa 100644 --- a/internal/database/repository/sync_test.go +++ b/internal/database/repository/sync_test.go @@ -2,7 +2,6 @@ package repository_test import ( "context" - "sync" "testing" "time" @@ -128,30 +127,19 @@ type docSyncStateParams struct { } func getDocumentSyncState(t testing.TB, ctx context.Context, queries *repository.Queries, params docSyncStateParams) { - var wg sync.WaitGroup + t.Helper() + isSynced, err := queries.IsClientSynced(ctx, ¶ms.clientId) + require.NoError(t, err) + assert.Equal(t, params.isSynced, isSynced) - wg.Add(1) - go func() { - isSynced, err := queries.IsClientSynced(ctx, ¶ms.clientId) - require.NoError(t, err) - assert.Equal(t, params.isSynced, isSynced) - wg.Done() - }() - - wg.Add(1) - go func() { - doc, err := queries.GetDocumentExternal(ctx, params.documentId) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: params.documentId, - Clientid: params.clientId, - Hash: params.hash, - Fields: []byte(params.fields), - }, doc) - wg.Done() - }() - - wg.Wait() + doc, err := queries.GetDocumentExternal(ctx, params.documentId) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: params.documentId, + Clientid: params.clientId, + Hash: params.hash, + Fields: []byte(params.fields), + }, doc) } func createQuery(t testing.TB, queries *repository.Queries, queryType repository.Querytype) uuid.UUID { diff --git a/internal/document/service_test.go b/internal/document/service_test.go deleted file mode 100644 index db557ca5..00000000 --- a/internal/document/service_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package document_test - -import ( - "testing" - - "queryorchestration/internal/document" - - "github.com/stretchr/testify/assert" -) - -func TestNewDocumentService(t *testing.T) { - svc := document.New(nil) - assert.NotNil(t, svc) -} diff --git a/internal/export/service_test.go b/internal/export/service_test.go deleted file mode 100644 index 7a6bf83b..00000000 --- a/internal/export/service_test.go +++ /dev/null @@ -1,14 +0,0 @@ -package export_test - -import ( - "testing" - - "queryorchestration/internal/export" - - "github.com/stretchr/testify/assert" -) - -func TestNewExportService(t *testing.T) { - svc := export.New() - assert.NotNil(t, svc) -} diff --git a/internal/query/result/processor/parse.go b/internal/query/result/processor/parse.go index e66710d5..f35678a7 100644 --- a/internal/query/result/processor/parse.go +++ b/internal/query/result/processor/parse.go @@ -65,8 +65,8 @@ func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) { return repository.NullQuerytype{Querytype: dbType, Valid: true}, nil } -func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, error) { - qType, err := ParseDBType(q.Querytype) +func ParseDBCollectorQuery(q *repository.ListCollectorQueriesRow) (*Query, error) { + qType, err := ParseDBNullType(q.Querytype) if err != nil { return nil, err } @@ -78,7 +78,7 @@ func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, return &Query{ ID: *q.Queryid, - Version: q.Queryversion, + Version: *q.Queryversion, Type: qType, RequiredQueryIDs: reqIds, }, nil diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go index c2cedb6d..ba0abb19 100644 --- a/internal/query/result/processor/parse_test.go +++ b/internal/query/result/processor/parse_test.go @@ -12,12 +12,17 @@ import ( ) func TestParseDBCollectorQuery(t *testing.T) { - dbResult := repository.Collectorquerydependencytree{ - Clientid: "hello", - Queryid: &uuid.UUID{}, - Requiredids: []uuid.UUID{}, - Querytype: repository.QuerytypeJsonExtractor, - Queryversion: 1, + id := "hello" + version := int32(1) + dbResult := repository.ListCollectorQueriesRow{ + Clientid: &id, + Queryid: &uuid.UUID{}, + Requiredids: []uuid.UUID{}, + Querytype: repository.NullQuerytype{ + Querytype: repository.QuerytypeJsonExtractor, + Valid: true, + }, + Queryversion: &version, } value, err := resultprocessor.ParseDBCollectorQuery(&dbResult) require.NoError(t, err) @@ -26,7 +31,7 @@ func TestParseDBCollectorQuery(t *testing.T) { assert.Equal(t, int32(1), value.Version) assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value.Type) - dbResult.Querytype = repository.Querytype("") + dbResult.Querytype = repository.NullQuerytype{} _, err = resultprocessor.ParseDBCollectorQuery(&dbResult) assert.EqualError(t, err, "invalid database query type") } diff --git a/internal/query/result/service_test.go b/internal/query/result/service_test.go deleted file mode 100644 index 0e680a2e..00000000 --- a/internal/query/result/service_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package result_test - -import ( - "testing" - - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query/result" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := result.New(cfg, &result.Services{}) - assert.NotNil(t, svc) -} diff --git a/internal/query/result/set/set.go b/internal/query/result/set/set.go index 7b843677..19bcbffa 100644 --- a/internal/query/result/set/set.go +++ b/internal/query/result/set/set.go @@ -86,8 +86,8 @@ func (s *Service) storeResult(ctx context.Context, textVersion *repository.Curre func (s *Service) informQueryDependents(ctx context.Context, params *Set) error { ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ - Documentid: params.DocumentID, - Queryid: params.QueryID, + Documentid: ¶ms.DocumentID, + Queryid: ¶ms.QueryID, }) if err != nil { return err diff --git a/internal/query/result/set/set_test.go b/internal/query/result/set/set_test.go index beaf354d..feeeea29 100644 --- a/internal/query/result/set/set_test.go +++ b/internal/query/result/set/set_test.go @@ -104,7 +104,7 @@ func TestSet(t *testing.T) { pool.ExpectExec("name: AddResultDependency :exec").WithArgs(resultId, requiredResultId). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() - pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(params.QueryID, params.DocumentID). + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(¶ms.QueryID, ¶ms.DocumentID). WillReturnRows( pgxmock.NewRows([]string{"queryId"}). AddRow(&requiredQuery), @@ -159,7 +159,7 @@ func TestInformQueryDependents(t *testing.T) { QueryID: uuid.New(), } - pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(params.QueryID, params.DocumentID). + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(¶ms.QueryID, ¶ms.DocumentID). WillReturnRows( pgxmock.NewRows([]string{"queryId"}). AddRow(&reqOne). diff --git a/internal/query/service_test.go b/internal/query/service_test.go deleted file mode 100644 index 51a40be4..00000000 --- a/internal/query/service_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package query_test - -import ( - "testing" - - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := query.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/query/test/service_test.go b/internal/query/test/service_test.go deleted file mode 100644 index 9077259e..00000000 --- a/internal/query/test/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package querytest_test - -import ( - "testing" - - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query" - "queryorchestration/internal/serviceconfig" - - "github.com/stretchr/testify/require" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := query.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/query/types/contextFull/creator_test.go b/internal/query/types/contextFull/creator_test.go index 2dbcd44f..a552f87c 100644 --- a/internal/query/types/contextFull/creator_test.go +++ b/internal/query/types/contextFull/creator_test.go @@ -10,7 +10,6 @@ import ( "queryorchestration/internal/serviceconfig" "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,7 +23,6 @@ func TestCreatorValidate(t *testing.T) { cfg.DBQueries = repository.New(pool) svc := contextfull.NewCreator() - assert.NotNil(t, svc) entity := &resultprocessor.Create{ Type: resultprocessor.TypeContextFull, diff --git a/internal/query/types/contextFull/updator_test.go b/internal/query/types/contextFull/updator_test.go index e4721e9a..b614bcf5 100644 --- a/internal/query/types/contextFull/updator_test.go +++ b/internal/query/types/contextFull/updator_test.go @@ -13,7 +13,6 @@ import ( "github.com/stretchr/testify/require" "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" ) func TestUpdatorValidate(t *testing.T) { @@ -26,7 +25,6 @@ func TestUpdatorValidate(t *testing.T) { cfg.DBQueries = repository.New(pool) svc := contextfull.NewUpdator() - assert.NotNil(t, svc) current := &resultprocessor.Query{ ID: uuid.New(), diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go index 3805b50a..d31c1414 100644 --- a/internal/query/types/jsonExtractor/creator_test.go +++ b/internal/query/types/jsonExtractor/creator_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" ) func TestCreatorValidate(t *testing.T) { @@ -25,7 +24,6 @@ func TestCreatorValidate(t *testing.T) { cfg.DBQueries = repository.New(pool) svc := jsonextractor.NewCreator() - assert.NotNil(t, svc) ccfg := "{}" entity := &resultprocessor.Create{ diff --git a/internal/query/types/jsonExtractor/updator_test.go b/internal/query/types/jsonExtractor/updator_test.go index a40fba13..32999f34 100644 --- a/internal/query/types/jsonExtractor/updator_test.go +++ b/internal/query/types/jsonExtractor/updator_test.go @@ -11,7 +11,6 @@ import ( "github.com/google/uuid" "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -25,7 +24,6 @@ func TestUpdatorValidate(t *testing.T) { cfg.DBQueries = repository.New(pool) svc := jsonextractor.NewUpdator() - assert.NotNil(t, svc) current := &resultprocessor.Query{ ID: uuid.New(), diff --git a/internal/query/update/service_test.go b/internal/query/update/service_test.go deleted file mode 100644 index 080c69ec..00000000 --- a/internal/query/update/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package queryupdate_test - -import ( - "testing" - - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query" - "queryorchestration/internal/serviceconfig" - - "github.com/stretchr/testify/require" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := query.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/query/versionsync/service_test.go b/internal/query/versionsync/service_test.go deleted file mode 100644 index 160557e9..00000000 --- a/internal/query/versionsync/service_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package queryversionsync_test - -import ( - "testing" - - "queryorchestration/internal/query" - "queryorchestration/internal/serviceconfig" - - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - cfg := &serviceconfig.BaseConfig{} - svc := query.New(cfg) - assert.NotNil(t, svc) -} diff --git a/internal/server/runner/listener_test.go b/internal/server/runner/listener_test.go index c07ce81b..979001cf 100644 --- a/internal/server/runner/listener_test.go +++ b/internal/server/runner/listener_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "sync" "testing" "queryorchestration/internal/serviceconfig" @@ -29,8 +30,18 @@ func TestNewRunner(t *testing.T) { _ = serviceconfig.InitializeConfig(cfg) net := test.GetNetwork(t) - a := test.CreateAWSContainer(t, cfg, net) - test.SetQueueClient(t, t.Context(), cfg, a.ExternalEndpoint) + var wg sync.WaitGroup + wg.Add(1) + go func() { + a := test.CreateAWSContainer(t, cfg, net) + test.SetQueueClient(t, t.Context(), cfg, a.ExternalEndpoint) + cfg.QueueURL = test.CreateQueue(t, t.Context(), cfg, "queueName") + cfg.SetSQSEndpoint(a.ExternalEndpoint) + wg.Done() + }() + + wg.Wait() + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{ NoMigrations: true, }) @@ -38,8 +49,6 @@ func TestNewRunner(t *testing.T) { cfg.ControllerFunc = func() Controller[interface{}] { return runnermock.NewMockController[interface{}](t) } - cfg.QueueURL = test.CreateQueue(t, t.Context(), cfg, "queueName") - cfg.SetSQSEndpoint(a.ExternalEndpoint) srvPtr, err := New(t.Context(), cfg) require.NoError(t, err) diff --git a/internal/serviceconfig/threadpool/config_test.go b/internal/serviceconfig/threadpool/config_test.go index 9103a0ec..e2982e22 100644 --- a/internal/serviceconfig/threadpool/config_test.go +++ b/internal/serviceconfig/threadpool/config_test.go @@ -35,7 +35,7 @@ func TestStopAndWait(t *testing.T) { assert.LessOrEqual(t, 10, cfg.MaxWorkers) assert.NotNil(t, cfg.pool) - for i := range 1000 { + for i := range 2 { index := i cfg.pool.Submit(func() { sl.Info("new thread", "index", index) diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index 63eb05ad..519f6bb4 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -1,29 +1,18 @@ package test import ( - "bufio" - "bytes" "context" - "encoding/json" - "errors" "fmt" - "io" - "log/slog" - "net/http" "strings" "sync" "testing" - "time" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/objectstore" queryapi "queryorchestration/pkg/queryAPI" - "github.com/docker/go-connections/nat" - "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" ) type Network struct { @@ -34,12 +23,13 @@ type Network struct { } func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Network, func()) { - deps, clean := CreateFullDependencies(t, ctx, cfg) + deps := CreateFullDependencies(t, ctx, cfg) var wg sync.WaitGroup apiContainers := make(map[APIName]*Container, len(apis)) apiClean := make([]func(), len(apis)) + var apiMu sync.Mutex for i, s := range apis { wg.Add(1) go func() { @@ -47,7 +37,11 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo API: s, MockHTTP: string(deps.MockServer.Internal), }) + + apiMu.Lock() apiContainers[s.Name] = c + apiMu.Unlock() + apiClean[i] = ccleanup wg.Done() }() @@ -55,6 +49,7 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo runnerContainers := make(map[RunnerName]*Container, len(runners)) runnerClean := make([]func(), len(runners)) + var runnerMu sync.Mutex for i, r := range runners { wg.Add(1) go func() { @@ -63,7 +58,10 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo MockHTTP: string(deps.MockServer.Internal), }) + runnerMu.Lock() runnerContainers[r.Name] = c + runnerMu.Unlock() + runnerClean[i] = ccleanup wg.Done() }() @@ -87,9 +85,6 @@ func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCo for _, c := range runnerClean { c() } - - clean() - } } @@ -106,14 +101,13 @@ type Dependencies struct { Network string } -func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Dependencies, func()) { +func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) Dependencies { network := GetNetwork(t) deps := Dependencies{ Network: network, QueueURLs: map[RunnerName]string{}, } - cs := map[string]func(){} var wg sync.WaitGroup @@ -138,9 +132,8 @@ func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenc wg.Done() }() - mockServer, cleanMock := CreateMockServer(t, ctx, network) + mockServer := CreateMockServer(t, network) deps.MockServer = mockServer - cs["mock"] = cleanMock CreateDB(t, cfg, network, &CreateDatabaseConfig{ NoMigrations: true, @@ -148,11 +141,7 @@ func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenc wg.Wait() - return deps, func() { - for _, c := range cs { - c() - } - } + return deps } type APINetwork struct { @@ -161,7 +150,7 @@ type APINetwork struct { } func CreateAPINetwork(t testing.TB, ctx context.Context, cfg FullDependenciesConfig, api API) (*APINetwork, func()) { - deps, depsclean := CreateFullDependencies(t, ctx, cfg) + deps := CreateFullDependencies(t, ctx, cfg) c, ccleanup := CreateAPI(t, ctx, cfg, deps.Network, &APIConfig{ API: api, @@ -169,92 +158,9 @@ func CreateAPINetwork(t testing.TB, ctx context.Context, cfg FullDependenciesCon }) return &APINetwork{ - Dependencies: deps, - API: c, - }, func() { - depsclean() - ccleanup() - } -} - -type Address string - -type MockServer struct { - Internal Address - External Address - Container testcontainers.Container - Client *http.Client -} - -type MockBody any -type MockQueries map[string][]string -type MockHeaders map[string][]string - -type MockRequest struct { - Method string `json:"method"` - Path string `json:"path"` - Headers MockHeaders `json:"headers"` - Body MockBody `json:"body"` - Query MockQueries `json:"queryStringParameters"` -} - -type MockResponse struct { - Code int `json:"statusCode"` - Headers MockHeaders `json:"headers"` - Body MockBody `json:"body"` -} - -type MockExpectation struct { - Request MockRequest `json:"httpRequest"` - Response MockResponse `json:"httpResponse"` -} - -func CreateMockServer(t testing.TB, ctx context.Context, network string) (MockServer, func()) { - name := GetAlias(t, "mockserver") - port, err := nat.NewPort("tcp", "1080") - require.NoError(t, err) - - req := testcontainers.ContainerRequest{ - Image: "mockserver/mockserver:latest", - ExposedPorts: []string{port.Port()}, - Env: map[string]string{ - "MOCKSERVER_LOG_LEVEL": "INFO", - }, - WaitingFor: wait.ForAll( - wait.ForExposedPort(), - wait.ForListeningPort(port), - ), - Networks: []string{network}, - NetworkAliases: map[string][]string{ - network: {name}, - }, - } - - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - ContainerRequest: req, - Started: true, - }) - require.NoError(t, err) - - time.Sleep(2 * time.Second) - - host, err := container.Host(ctx) - require.NoError(t, err) - - externalPort, err := container.MappedPort(ctx, port) - require.NoError(t, err) - - server := MockServer{ - Client: &http.Client{}, - Internal: Address(fmt.Sprintf("http://%s:%d", name, port.Int())), - External: Address(fmt.Sprintf("http://%s:%d", host, externalPort.Int())), - Container: container, - } - - return server, func() { - err := container.Terminate(ctx) - require.NoError(t, err) - } + Dependencies: deps, + API: c, + }, ccleanup } func GetAlias(t testing.TB, baseName string) string { @@ -266,153 +172,3 @@ func GetAlias(t testing.TB, baseName string) string { name = strings.ReplaceAll(name, "_", "") return name } - -func CreateMockExpectation(t testing.TB, server MockServer, expectation MockExpectation) { - jsonData, err := json.Marshal(expectation) - require.NoError(t, err) - - url := fmt.Sprintf("%s/mockserver/expectation", server.External) - req, err := http.NewRequest("PUT", url, bytes.NewBuffer(jsonData)) - require.NoError(t, err) - req.Header.Set("Content-Type", "application/json") - - resp, err := server.Client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - t.Fatalf("failed to configure MockServer, status: %d, response: %s", resp.StatusCode, body) - } -} - -func CreateDetectDocumentTextExpectation(t testing.TB, mockServer MockServer, body string) MockExpectation { - childId := uuid.NewString() - expectation := MockExpectation{ - Request: MockRequest{ - Method: "POST", - Path: "/", - Headers: MockHeaders{ - "X-Amz-Target": []string{"Textract.AnalyzeDocument"}, - }, - Body: map[string]interface{}{ - "Document": map[string]interface{}{}, - "FeatureTypes": []string{"LAYOUT", "SIGNATURES"}, - }, - Query: MockQueries{}, - }, - Response: MockResponse{ - Code: 200, - Headers: MockHeaders{ - "Content-Type": {"application/json"}, - }, - Body: map[string]interface{}{ - "Blocks": []map[string]interface{}{ - { - "BlockType": "PAGE", - "Relationships": []map[string]interface{}{ - { - "Type": "CHILD", - "Ids": []string{ - childId, - }, - }, - }, - }, - { - "Id": childId, - "Text": body, - }, - }, - }, - }, - } - - CreateMockExpectation(t, mockServer, expectation) - - return expectation -} - -func WaitForMockEndpoint(t testing.TB, server MockServer, request MockRequest) MockRequest { - t.Helper() - - verificationRequest := map[string]any{ - "httpRequest": request, - "times": map[string]any{ - "atLeast": 1, - }, - } - - jsonData, err := json.Marshal(verificationRequest) - require.NoError(t, err) - - verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) - - clientTimeout := 500 * time.Millisecond - client := &http.Client{} - - timeout := time.After(60 * time.Second) - ticker := time.NewTicker(clientTimeout) - defer ticker.Stop() - - slog.Info("Attempting to process request", "body", jsonData) - - for { - select { - case <-timeout: - require.NoError(t, errors.New("Timeout waiting for mock http request to be fulfilled")) - case <-ticker.C: - req, err := http.NewRequest("PUT", verifyURL, bytes.NewBuffer(jsonData)) - require.NoError(t, err) - req.Header.Set("Content-Type", "application/json") - - resp, err := client.Do(req) - require.NoError(t, err) - - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - slog.Info(line) - } - - if resp.StatusCode == http.StatusAccepted { - slog.Info("request found") - resp.Body.Close() - return retrieveMatchingRequest(t, server, request) - } - - resp.Body.Close() - slog.Error("no request found") - } - } -} - -func retrieveMatchingRequest(t testing.TB, server MockServer, request MockRequest) MockRequest { - retrieveURL := fmt.Sprintf("%s/mockserver/retrieve?type=REQUESTS", server.External) - - req, err := http.NewRequest("PUT", retrieveURL, nil) - require.NoError(t, err) - - client := &http.Client{} - resp, err := client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - - var requests []MockRequest - err = json.Unmarshal(body, &requests) - require.NoError(t, err) - - for i := len(requests) - 1; i >= 0; i-- { - attemptRequest := requests[i] - - if attemptRequest.Method == request.Method && attemptRequest.Path == request.Path { - return attemptRequest - } - } - - require.Fail(t, "no request found") - return MockRequest{} -} diff --git a/internal/test/mockserver.go b/internal/test/mockserver.go new file mode 100644 index 00000000..12662742 --- /dev/null +++ b/internal/test/mockserver.go @@ -0,0 +1,261 @@ +package test + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +type Address string + +type MockServer struct { + Internal Address + External Address + Container testcontainers.Container + Client *http.Client +} + +type MockBody any +type MockQueries map[string][]string +type MockHeaders map[string][]string + +type MockRequest struct { + Method string `json:"method"` + Path string `json:"path"` + Headers MockHeaders `json:"headers"` + Body MockBody `json:"body"` + Query MockQueries `json:"queryStringParameters"` +} + +type MockResponse struct { + Code int `json:"statusCode"` + Headers MockHeaders `json:"headers"` + Body MockBody `json:"body"` +} + +type MockExpectation struct { + Request MockRequest `json:"httpRequest"` + Response MockResponse `json:"httpResponse"` +} + +const ( + mockServerAlias = "mockserver" + mockServerPort = 1080 +) + +func CreateMockServer(t testing.TB, network string) MockServer { + port, err := nat.NewPort("tcp", strconv.Itoa(mockServerPort)) + require.NoError(t, err) + + req := testcontainers.ContainerRequest{ + Image: "mockserver/mockserver:latest", + Name: "mockserver_test_queryorchestration", + ExposedPorts: []string{port.Port()}, + Env: map[string]string{ + "MOCKSERVER_LOG_LEVEL": "INFO", + "MOCKSERVER_LIVENESS_HTTP_GET_PATH": "/liveness/probe", + }, + WaitingFor: wait.ForAll( + wait.ForExposedPort(), + wait.ForListeningPort(port), + wait.ForHTTP("/liveness/probe"). + WithPort(port). + WithResponseMatcher(func(body io.Reader) bool { + return true + }). + WithStatusCodeMatcher(func(statusCode int) bool { + return statusCode == http.StatusOK + }), + ), + Networks: []string{network}, + NetworkAliases: map[string][]string{ + network: {mockServerAlias}, + }, + } + + container, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + Reuse: true, + }) + require.NoError(t, err) + + host, err := container.Host(t.Context()) + require.NoError(t, err) + + externalPort, err := container.MappedPort(t.Context(), port) + require.NoError(t, err) + + server := MockServer{ + Client: &http.Client{}, + Internal: Address(fmt.Sprintf("http://%s:%d", mockServerAlias, port.Int())), + External: Address(fmt.Sprintf("http://%s:%d", host, externalPort.Int())), + Container: container, + } + + return server +} + +func CreateMockExpectation(t testing.TB, server MockServer, expectation MockExpectation) { + jsonData, err := json.Marshal(expectation) + require.NoError(t, err) + + url := fmt.Sprintf("%s/mockserver/expectation", server.External) + req, err := http.NewRequest("PUT", url, bytes.NewBuffer(jsonData)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + resp, err := server.Client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("failed to configure MockServer, status: %d, response: %s", resp.StatusCode, body) + } +} + +func CreateDetectDocumentTextExpectation(t testing.TB, mockServer MockServer, body string) MockExpectation { + childId := uuid.NewString() + expectation := MockExpectation{ + Request: MockRequest{ + Method: "POST", + Path: "/", + Headers: MockHeaders{ + "X-Amz-Target": []string{"Textract.AnalyzeDocument"}, + }, + Body: map[string]interface{}{ + "Document": map[string]interface{}{}, + "FeatureTypes": []string{"LAYOUT", "SIGNATURES"}, + }, + Query: MockQueries{}, + }, + Response: MockResponse{ + Code: 200, + Headers: MockHeaders{ + "Content-Type": {"application/json"}, + }, + Body: map[string]interface{}{ + "Blocks": []map[string]interface{}{ + { + "BlockType": "PAGE", + "Relationships": []map[string]interface{}{ + { + "Type": "CHILD", + "Ids": []string{ + childId, + }, + }, + }, + }, + { + "Id": childId, + "Text": body, + }, + }, + }, + }, + } + + CreateMockExpectation(t, mockServer, expectation) + + return expectation +} + +func WaitForMockEndpoint(t testing.TB, server MockServer, request MockRequest) MockRequest { + t.Helper() + + verificationRequest := map[string]any{ + "httpRequest": request, + "times": map[string]any{ + "atLeast": 1, + }, + } + + jsonData, err := json.Marshal(verificationRequest) + require.NoError(t, err) + + verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) + + clientTimeout := 500 * time.Millisecond + client := &http.Client{} + + timeout := time.After(60 * time.Second) + ticker := time.NewTicker(clientTimeout) + defer ticker.Stop() + + slog.Info("Attempting to process request", "body", jsonData) + + for { + select { + case <-timeout: + require.NoError(t, errors.New("Timeout waiting for mock http request to be fulfilled")) + case <-ticker.C: + req, err := http.NewRequest("PUT", verifyURL, bytes.NewBuffer(jsonData)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + require.NoError(t, err) + + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + slog.Info(line) + } + + if resp.StatusCode == http.StatusAccepted { + slog.Info("request found") + resp.Body.Close() + return retrieveMatchingRequest(t, server, request) + } + + resp.Body.Close() + slog.Error("no request found") + } + } +} + +func retrieveMatchingRequest(t testing.TB, server MockServer, request MockRequest) MockRequest { + retrieveURL := fmt.Sprintf("%s/mockserver/retrieve?type=REQUESTS", server.External) + + req, err := http.NewRequest("PUT", retrieveURL, nil) + require.NoError(t, err) + + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var requests []MockRequest + err = json.Unmarshal(body, &requests) + require.NoError(t, err) + + for i := len(requests) - 1; i >= 0; i-- { + attemptRequest := requests[i] + + if attemptRequest.Method == request.Method && attemptRequest.Path == request.Path { + return attemptRequest + } + } + + require.Fail(t, "no request found") + return MockRequest{} +} diff --git a/internal/test/ecosystem_test.go b/internal/test/mockserver_test.go similarity index 96% rename from internal/test/ecosystem_test.go rename to internal/test/mockserver_test.go index e52f9529..44281633 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/mockserver_test.go @@ -57,8 +57,7 @@ func TestWaitForMockEndpoint(t *testing.T) { } net := GetNetwork(t) - server, cleanup := CreateMockServer(t, t.Context(), net) - defer cleanup() + server := CreateMockServer(t, net) body := strings.NewReader(`{"team":"hello"}`) req, err := http.NewRequest("GET", string(server.External), body) diff --git a/internal/test/network.go b/internal/test/network.go index 9e2d81cd..3a0e030b 100644 --- a/internal/test/network.go +++ b/internal/test/network.go @@ -33,7 +33,6 @@ func GetNetwork(t testing.TB) string { } slog.Info("created network", "name", networkName) - }) return networkName diff --git a/internal/test/queryAPI/service.go b/internal/test/queryAPI/service.go index 0b21956d..c8eb46ba 100644 --- a/internal/test/queryAPI/service.go +++ b/internal/test/queryAPI/service.go @@ -16,25 +16,38 @@ import ( "github.com/stretchr/testify/require" ) -func CreateClientWithSync(t testing.TB, ctx context.Context, client queryapi.ClientWithResponsesInterface) *queryapi.DocClient { +func CreateClientWithSync(t testing.TB, client queryapi.ClientWithResponsesInterface) queryapi.ClientID { t.Helper() - clientCreateRes, err := client.CreateClientWithResponse(ctx, queryapi.ClientCreate{ + clientCreateRes, err := client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{ Name: "example_name", Id: "ID", }) require.NoError(t, err) canSync := true - _, err = client.UpdateClientWithResponse(ctx, clientCreateRes.JSON201.Id, queryapi.ClientUpdate{ + _, err = client.UpdateClientWithResponse(t.Context(), clientCreateRes.JSON201.Id, queryapi.ClientUpdate{ CanSync: &canSync, }) require.NoError(t, err) - clientRes, err := client.GetClientWithResponse(ctx, clientCreateRes.JSON201.Id) - require.NoError(t, err) + return clientCreateRes.JSON201.Id +} - return clientRes.JSON200 +func SetQueryForClient(t testing.TB, client queryapi.ClientWithResponsesInterface, clientId queryapi.ClientID, queryId uuid.UUID) { + t.Helper() + + newActiveVersion := int32(1) + _, err := client.SetCollectorByClientIdWithResponse(t.Context(), clientId, queryapi.CollectorSet{ + ActiveVersion: &newActiveVersion, + Fields: &[]queryapi.CollectorField{ + { + Name: "JSON_QUERY", + QueryId: queryId, + }, + }, + }) + require.NoError(t, err) } func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) { @@ -66,7 +79,7 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli } } -func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses, clientId string) (uuid.UUID, uuid.UUID) { +func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses) (uuid.UUID, uuid.UUID) { contextQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{ Type: queryapi.CONTEXTFULL, }) @@ -80,18 +93,6 @@ func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses, }) require.NoError(t, err) require.Equal(t, http.StatusCreated, jsonQueryRes.StatusCode()) - newActiveVersion := int32(1) - collRes, err := client.SetCollectorByClientIdWithResponse(t.Context(), clientId, queryapi.CollectorSet{ - ActiveVersion: &newActiveVersion, - Fields: &[]queryapi.CollectorField{ - { - Name: "JSON_QUERY", - QueryId: jsonQueryRes.JSON201.Id, - }, - }, - }) - require.NoError(t, err) - require.Equal(t, http.StatusOK, collRes.StatusCode()) return contextQueryRes.JSON201.Id, jsonQueryRes.JSON201.Id } diff --git a/internal/test/queryAPI/service_test.go b/internal/test/queryAPI/service_test.go deleted file mode 100644 index f5c65588..00000000 --- a/internal/test/queryAPI/service_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package queryapitest_test - -import ( - "context" - "testing" - - queryapitest "queryorchestration/internal/test/queryAPI" - queryapimock "queryorchestration/mocks/queryapi" - queryapi "queryorchestration/pkg/queryAPI" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestCreateClientWithSync(t *testing.T) { - ctx := context.Background() - - svcClient := queryapimock.NewMockClientWithResponsesInterface(t) - - svcClient.EXPECT().CreateClientWithResponse( - mock.Anything, - mock.MatchedBy(func(create queryapi.ClientCreate) bool { - return create.Name == "example_name" && create.Id == "ID" - }), - mock.Anything, - ).Return(&queryapi.CreateClientResponse{ - JSON201: &queryapi.ClientIDBody{ - Id: "ID", - }, - }, nil) - - svcClient.EXPECT().UpdateClientWithResponse( - mock.Anything, - mock.MatchedBy(func(id string) bool { - return id == "ID" - }), - mock.MatchedBy(func(create queryapi.ClientUpdate) bool { - return *create.CanSync == true - }), - mock.Anything, - ).Return(&queryapi.UpdateClientResponse{}, nil) - - svcClient.EXPECT().GetClientWithResponse( - mock.Anything, - mock.MatchedBy(func(id string) bool { - return id == "ID" - }), - mock.Anything, - ).Return(&queryapi.GetClientResponse{ - JSON200: &queryapi.DocClient{ - Id: "ID", - }, - }, nil) - - client := queryapitest.CreateClientWithSync(t, ctx, svcClient) - - assert.NotNil(t, client) - assert.NotNil(t, client) - assert.EqualExportedValues(t, queryapi.DocClient{ - Id: "ID", - }, *client) -} - -func TestWaitForClientStatus(t *testing.T) { - ctx := context.Background() - client := queryapimock.NewMockClientWithResponsesInterface(t) - - client.EXPECT().GetStatusByClientIdWithResponse( - mock.Anything, - mock.MatchedBy(func(id string) bool { - return id == "id" - }), - ).Return(&queryapi.GetStatusByClientIdResponse{ - JSON200: &queryapi.ClientStatusBody{ - Status: queryapi.INSYNC, - }, - }, nil) - - queryapitest.WaitForClientStatus(t, ctx, client, "id", queryapi.INSYNC) -} diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 1b117b72..3ce7a202 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -66,7 +66,7 @@ tasks: - task: docker:lint - task: openapi:lint - task: compose:lint - go:lint: golangci-lint run --fix {{.CLI_ARGS}} + go:lint: golangci-lint run {{.CLI_ARGS}} yaml:lint: yamllint . -s {{.CLI_ARGS}} json:lint: jsonlint devbox.json -q -s -i {{.CLI_ARGS}} docs:generate: diff --git a/test/process_test.go b/test/process_test.go index ee2f1073..e9fe2921 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -1,8 +1,8 @@ package endtoend_test import ( - "context" "strings" + "sync" "testing" "time" @@ -25,58 +25,69 @@ type Config struct { } func TestProcess(t *testing.T) { - ctx := context.Background() - cfg := &Config{} - net, clean := test.CreateFullNetwork(t, ctx, cfg) + net, clean := test.CreateFullNetwork(t, t.Context(), cfg) defer clean() - client := queryapitest.CreateClientWithSync(t, ctx, net.Client) - _, jsonId := queryapitest.CreateDependentQueries(t, net.Client, client.Id) - - queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC) - + var wg sync.WaitGroup + var clientId queryapi.ClientID + var jsonId queryapi.QueryID + var textractExpectation test.MockExpectation textractBody := "Hello World" - textractExpectation := test.CreateDetectDocumentTextExpectation( - t, - net.Dependencies.MockServer, - textractBody, - ) + wg.Add(3) + go func(t testing.TB) { + _, jsonId = queryapitest.CreateDependentQueries(t, net.Client) + wg.Done() + }(t) + go func(t testing.TB) { + clientId = queryapitest.CreateClientWithSync(t, net.Client) + wg.Done() + }(t) + go func(t testing.TB) { + textractExpectation = test.CreateDetectDocumentTextExpectation( + t, + net.Dependencies.MockServer, + textractBody, + ) + wg.Done() + }(t) + + wg.Wait() + + queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId) part := uint16(0) importKey := objectstore.BucketKey{ - ClientID: client.Id, + ClientID: clientId, EntityID: uuid.New(), Location: objectstore.Import, CreatedAt: time.Now().UTC(), Part: &part, } inputFile := strings.NewReader(pdfHelloWorld) - test.PutObject(t, ctx, cfg, test.PutObjectParams{ + test.PutObject(t, t.Context(), cfg, test.PutObjectParams{ File: inputFile, Key: importKey, }) - queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED) - test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request) textKey := objectstore.BucketKey{ - ClientID: client.Id, + ClientID: clientId, EntityID: uuid.New(), Location: objectstore.Text, CreatedAt: time.Now().UTC(), Part: &part, } inputFile = strings.NewReader(textractBody) - test.PutObject(t, ctx, cfg, test.PutObjectParams{ + test.PutObject(t, t.Context(), cfg, test.PutObjectParams{ File: inputFile, Key: textKey, }) - queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC) + queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) - docs, err := net.Client.ListDocumentsByClientIdWithResponse(ctx, client.Id) + docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId) require.NoError(t, err) assert.Len(t, *docs.JSON200, 1) doc := (*docs.JSON200)[0] @@ -84,13 +95,13 @@ func TestProcess(t *testing.T) { expectedDoc := queryapi.Document{ Id: doc.Id, Hash: doc.Hash, - ClientId: client.Id, + ClientId: clientId, Fields: map[string]any{ "JSON_QUERY": "valueone", }, } - docRes, err := net.Client.GetDocumentWithResponse(ctx, doc.Id) + docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id) require.NoError(t, err) require.NotNil(t, docRes) require.NotNil(t, docRes.JSON200) @@ -99,7 +110,7 @@ func TestProcess(t *testing.T) { jcfg := `{"path":"keytwo"}` av := int32(2) - _, err = net.Client.UpdateQueryWithResponse(ctx, jsonId, queryapi.QueryUpdate{ + _, err = net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{ ActiveVersion: &av, Config: &jcfg, }) @@ -108,33 +119,28 @@ func TestProcess(t *testing.T) { "JSON_QUERY": "valuetwo", } - queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED) - queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC) + queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.NOTSYNCED) + queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC) - docRes, err = net.Client.GetDocumentWithResponse(ctx, doc.Id) + docRes, err = net.Client.GetDocumentWithResponse(t.Context(), doc.Id) require.NoError(t, err) require.NotNil(t, docRes) require.NotNil(t, docRes.JSON200) assert.EqualExportedValues(t, expectedDoc, *docRes.JSON200) - testRes, err := net.Client.TestQueryWithResponse(ctx, jsonId, queryapi.QueryTestRequest{ + testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{ QueryVersion: 1, DocumentId: doc.Id, }) require.NoError(t, err) assert.Equal(t, "valueone", testRes.JSON200.Value) - testRes, err = net.Client.TestQueryWithResponse(ctx, jsonId, queryapi.QueryTestRequest{ + + testRes, err = net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{ QueryVersion: 2, DocumentId: doc.Id, }) require.NoError(t, err) assert.Equal(t, "valuetwo", testRes.JSON200.Value) - - docRes, err = net.Client.GetDocumentWithResponse(ctx, doc.Id) - require.NoError(t, err) - require.NotNil(t, docRes) - require.NotNil(t, docRes.JSON200) - assert.EqualExportedValues(t, expectedDoc, *docRes.JSON200) } const pdfHelloWorld = `%PDF-1.4