diff --git a/cpu.prof b/cpu.prof new file mode 100644 index 00000000..e69de29b diff --git a/deployments/compose.generate.yaml b/deployments/compose.generate.yaml index a7d95b33..1429bd16 100644 --- a/deployments/compose.generate.yaml +++ b/deployments/compose.generate.yaml @@ -11,7 +11,7 @@ services: expose: - 5431 healthcheck: - test: ["CMD", "sh", "-c", "pg_isready -U ${PGUSER} && psql -U ${PGUSER} -c 'SELECT 1'"] + test: ["CMD", "sh", "-c", "pg_isready -U ${PGUSER} -h localhost -p 5432 && psql -U ${PGUSER} -c 'SELECT 1'"] interval: 500ms timeout: 5s retries: 50 diff --git a/devbox.json b/devbox.json index 39f1f44e..6cc41c40 100644 --- a/devbox.json +++ b/devbox.json @@ -62,7 +62,8 @@ "sqlcheck@1.3", "docker-client@27.3.1", "gotestsum@latest", - "goperf@latest" + "goperf@latest", + "graphviz@latest" ], "shell": { "init_hook": [ diff --git a/devbox.lock b/devbox.lock index cec0778f..b9fb3cb3 100644 --- a/devbox.lock +++ b/devbox.lock @@ -545,6 +545,54 @@ } } }, + "graphviz@latest": { + "last_modified": "2025-03-24T07:07:41Z", + "resolved": "github:NixOS/nixpkgs/1750f3c1c89488e2ffdd47cab9d05454dddfb734#graphviz", + "source": "devbox-search", + "version": "12.2.1", + "systems": { + "aarch64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/n261cb4vg4wijd3x0kbh2wxjccxk2b2y-graphviz-12.2.1", + "default": true + } + ], + "store_path": "/nix/store/n261cb4vg4wijd3x0kbh2wxjccxk2b2y-graphviz-12.2.1" + }, + "aarch64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/shsff1y57kvz22qkkgy269lvn8s9yckz-graphviz-12.2.1", + "default": true + } + ], + "store_path": "/nix/store/shsff1y57kvz22qkkgy269lvn8s9yckz-graphviz-12.2.1" + }, + "x86_64-darwin": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/3mjmaid5ra8paxwf603dlc20w09739la-graphviz-12.2.1", + "default": true + } + ], + "store_path": "/nix/store/3mjmaid5ra8paxwf603dlc20w09739la-graphviz-12.2.1" + }, + "x86_64-linux": { + "outputs": [ + { + "name": "out", + "path": "/nix/store/199jxrl2dwd5afi6njmj7wvaz9ni79kz-graphviz-12.2.1", + "default": true + } + ], + "store_path": "/nix/store/199jxrl2dwd5afi6njmj7wvaz9ni79kz-graphviz-12.2.1" + } + } + }, "hadolint@2.12.0": { "last_modified": "2024-12-23T21:10:33Z", "resolved": "github:NixOS/nixpkgs/de1864217bfa9b5845f465e771e0ecb48b30e02d#hadolint", diff --git a/internal/client/create.go b/internal/client/create.go index addf847d..69b26cad 100644 --- a/internal/client/create.go +++ b/internal/client/create.go @@ -22,8 +22,8 @@ func (s *Service) Create(ctx context.Context, params CreateParams) (string, erro } err = s.cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{ - Name: params.Name, - ID: params.ID, + Name: params.Name, + Clientid: params.ID, }) if err != nil { return "", err diff --git a/internal/client/get.go b/internal/client/get.go index 6479e0eb..fb3a3dcd 100644 --- a/internal/client/get.go +++ b/internal/client/get.go @@ -17,7 +17,7 @@ func (s *Service) Get(ctx context.Context, id string) (*Client, error) { func parseFullClient(client *repository.Fullclient) *Client { return &Client{ - ID: client.ID, + ID: client.Clientid, Name: client.Name, CanSync: client.Cansync, } diff --git a/internal/client/get_test.go b/internal/client/get_test.go index 857e7c67..40f7d72f 100644 --- a/internal/client/get_test.go +++ b/internal/client/get_test.go @@ -50,9 +50,9 @@ func TestGet(t *testing.T) { func TestParseFullClient(t *testing.T) { in := &repository.Fullclient{ - ID: "external_id", - Name: "name", - Cansync: true, + Clientid: "external_id", + Name: "name", + Cansync: true, } out := parseFullClient(in) assert.EqualExportedValues(t, &Client{ diff --git a/internal/client/status.go b/internal/client/status.go index 843087fa..a5566367 100644 --- a/internal/client/status.go +++ b/internal/client/status.go @@ -20,7 +20,7 @@ func (s *Service) GetStatus(ctx context.Context, id string) (Status, error) { return NOT_SYNCING, nil } - issynced, err := s.cfg.GetDBQueries().IsClientSynced(ctx, &client.ID) + issynced, err := s.cfg.GetDBQueries().IsClientSynced(ctx, &client.Clientid) if err != nil { return NOT_SYNCING, err } else if issynced { diff --git a/internal/client/update/update.go b/internal/client/update/update.go index b9c4be87..0033c290 100644 --- a/internal/client/update/update.go +++ b/internal/client/update/update.go @@ -60,8 +60,8 @@ func (s *Service) submitUpdate(ctx context.Context, id string, entity *Update) e if entity.Name != nil { err := q.UpdateClient(ctx, &repository.UpdateClientParams{ - ID: id, - Name: *entity.Name, + Clientid: id, + Name: *entity.Name, }) if err != nil { return err diff --git a/internal/cognitoauth/jwks.go b/internal/cognitoauth/jwks.go index 64a1d1f6..bca961b6 100644 --- a/internal/cognitoauth/jwks.go +++ b/internal/cognitoauth/jwks.go @@ -102,7 +102,11 @@ func GetJWKS(jwksURL string, logger *slog.Logger) (jwk.Set, error) { // This function is typically called by GetJWKS when the cached JWKS has expired // or is not available, and should not be called directly in most cases. func fetchJWKS(jwksURL string) (string, error) { - client := &http.Client{Timeout: JWKS_CLIENT_TIMEOUT} + return fetchJWKSWithTimeout(jwksURL, JWKS_CLIENT_TIMEOUT) +} + +func fetchJWKSWithTimeout(jwksURL string, timeout time.Duration) (string, error) { + client := &http.Client{Timeout: timeout} req, err := http.NewRequest("GET", jwksURL, nil) if err != nil { return "", err diff --git a/internal/cognitoauth/jwks_test.go b/internal/cognitoauth/jwks_test.go index f071bb63..7b3bb3d9 100644 --- a/internal/cognitoauth/jwks_test.go +++ b/internal/cognitoauth/jwks_test.go @@ -270,9 +270,10 @@ func TestFetchJWKS(t *testing.T) { // - Verifies appropriate error handling for timed-out requests // - Ensures the function fails properly in timeout scenarios t.Run("timeout", func(t *testing.T) { + t.Parallel() // Create a server that sleeps longer than the client timeout slowServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(JWKS_CLIENT_TIMEOUT + 1*time.Second) + time.Sleep(1100 * time.Millisecond) _, err := w.Write([]byte(jwksJSON)) if err != nil { t.Errorf("Failed to write response: %v", err) @@ -280,7 +281,7 @@ func TestFetchJWKS(t *testing.T) { })) defer slowServer.Close() - _, err := fetchJWKS(slowServer.URL) + _, err := fetchJWKSWithTimeout(slowServer.URL, time.Second) if err == nil { t.Errorf("fetchJWKS() with timeout should return error, got nil") } diff --git a/internal/database/migrations/00000000000002_queries.up.sql b/internal/database/migrations/00000000000002_queries.up.sql index 6a964fee..b28878a4 100644 --- a/internal/database/migrations/00000000000002_queries.up.sql +++ b/internal/database/migrations/00000000000002_queries.up.sql @@ -1,26 +1,26 @@ CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor'); CREATE TABLE queries ( - id uuid primary key DEFAULT uuid_generate_v7(), - type queryType not null + queryId uuid primary key DEFAULT uuid_generate_v7(), + queryType queryType not null ); CREATE TABLE queryVersions ( queryId uuid not null, - id int not null, + versionId int not null, addedAt timestamp not null default current_timestamp, - primary key (id, queryId), - foreign key (queryId) references queries(id) + primary key (versionId, queryId), + foreign key (queryId) references queries(queryId) ); CREATE OR REPLACE FUNCTION setQueryVersionNumber() RETURNS TRIGGER AS $$ BEGIN - SELECT COALESCE(MAX(id), 0) + 1 - INTO NEW.id + SELECT COALESCE(MAX(versionId), 0) + 1 + INTO NEW.versionId FROM queryVersions WHERE queryId = NEW.queryId; - + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -31,44 +31,44 @@ FOR EACH ROW EXECUTE FUNCTION setQueryVersionNumber(); CREATE TABLE queryActiveVersions ( - id uuid primary key DEFAULT uuid_generate_v7(), + activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(), queryId uuid not null, versionId int not null, - foreign key (queryId, versionId) references queryVersions(queryId, id) + foreign key (queryId, versionId) references queryVersions(queryId, versionId) ); CREATE TABLE requiredQueries ( - id uuid primary key DEFAULT uuid_generate_v7(), + requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(), queryId uuid not null, requiredQueryId uuid not null, addedVersion int not null, removedVersion int, - foreign key (queryId) references queries(id), - foreign key (requiredQueryId) references queries(id), - foreign key (queryId, addedVersion) references queryVersions(queryId, id), - foreign key (queryId, removedVersion) references queryVersions(queryId, id), + foreign key (queryId) references queries(queryId), + foreign key (requiredQueryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), unique (queryId, requiredQueryId, removedVersion) ); CREATE TABLE queryConfigs ( - id uuid primary key DEFAULT uuid_generate_v7(), + configId uuid primary key DEFAULT uuid_generate_v7(), queryId uuid not null, config jsonb not null, addedVersion int not null, removedVersion int, - foreign key (queryId) references queries(id), - foreign key (queryId, addedVersion) references queryVersions(queryId, id), - foreign key (queryId, removedVersion) references queryVersions(queryId, id), + foreign key (queryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), unique (queryId, removedVersion) ); CREATE OR REPLACE FUNCTION removeQueryConfig() RETURNS TRIGGER AS $$ BEGIN - UPDATE queryConfigs + UPDATE queryConfigs SET removedVersion = NEW.addedVersion WHERE queryId = NEW.queryId and removedVersion is null; - + RETURN NEW; END; $$ LANGUAGE plpgsql; @@ -76,4 +76,4 @@ $$ LANGUAGE plpgsql; CREATE TRIGGER removeQueryConfigTrigger BEFORE INSERT ON queryConfigs FOR EACH ROW -EXECUTE FUNCTION removeQueryConfig(); \ No newline at end of file +EXECUTE FUNCTION removeQueryConfig(); diff --git a/internal/database/migrations/00000000000003_clients.up.sql b/internal/database/migrations/00000000000003_clients.up.sql index 41414b33..8525ef2d 100644 --- a/internal/database/migrations/00000000000003_clients.up.sql +++ b/internal/database/migrations/00000000000003_clients.up.sql @@ -1,12 +1,12 @@ CREATE TABLE clients ( - id varchar(255) primary key, + clientId varchar(255) primary key, name TEXT not null, UNIQUE (name) ); CREATE TABLE clientCanSync ( - id uuid primary key DEFAULT uuid_generate_v7(), + syncId uuid primary key DEFAULT uuid_generate_v7(), clientId varchar(255) not null, canSync boolean not null, - foreign key (clientId) references clients(id) + foreign key (clientId) references clients(clientId) ); diff --git a/internal/database/migrations/00000000000004_collectors.up.sql b/internal/database/migrations/00000000000004_collectors.up.sql index 76f05cfc..60695d90 100644 --- a/internal/database/migrations/00000000000004_collectors.up.sql +++ b/internal/database/migrations/00000000000004_collectors.up.sql @@ -2,7 +2,7 @@ CREATE TABLE collectorVersions ( clientId varchar(255) not null, id int not null, addedAt timestamp not null default current_timestamp, - foreign key (clientId) references clients(id), + foreign key (clientId) references clients(clientId), primary key (id, clientId) ); @@ -38,7 +38,7 @@ CREATE TABLE collectorMinCleanVersions ( removedVersion int, foreign key (clientId, addedVersion) references collectorVersions(clientId, id), foreign key (clientId, removedVersion) references collectorVersions(clientId, id), - foreign key (clientId) references clients(id) + foreign key (clientId) references clients(clientId) ); CREATE OR REPLACE FUNCTION removeMinCleanVersion() @@ -65,7 +65,7 @@ CREATE TABLE collectorMinTextVersions ( removedVersion int, foreign key (clientId, addedVersion) references collectorVersions(clientId, id), foreign key (clientId, removedVersion) references collectorVersions(clientId, id), - foreign key (clientId) references clients(id) + foreign key (clientId) references clients(clientId) ); CREATE OR REPLACE FUNCTION removeMinTextVersion() @@ -91,8 +91,8 @@ CREATE TABLE collectorQueries ( queryId uuid not null, addedVersion int not null, removedVersion int, - foreign key (queryId) references queries(id), - foreign key (clientId) references clients(id), + foreign key (queryId) references queries(queryId), + foreign key (clientId) references clients(clientId), foreign key (clientId, addedVersion) references collectorVersions(clientId, id), foreign key (clientId, removedVersion) references collectorVersions(clientId, id), unique (clientId, name, removedVersion) diff --git a/internal/database/migrations/00000000000005_documents.up.sql b/internal/database/migrations/00000000000005_documents.up.sql index 84655474..075d9d6e 100644 --- a/internal/database/migrations/00000000000005_documents.up.sql +++ b/internal/database/migrations/00000000000005_documents.up.sql @@ -2,7 +2,7 @@ CREATE TABLE documents ( id uuid primary key DEFAULT uuid_generate_v7(), clientId varchar(255) not null, hash text not null, - foreign key (clientId) references clients(id), + foreign key (clientId) references clients(clientId), unique(clientId, hash) ); diff --git a/internal/database/migrations/00000000000006_results.up.sql b/internal/database/migrations/00000000000006_results.up.sql index e41ff5cf..e27a128a 100644 --- a/internal/database/migrations/00000000000006_results.up.sql +++ b/internal/database/migrations/00000000000006_results.up.sql @@ -4,9 +4,9 @@ CREATE TABLE results ( queryId uuid not null, value TEXT not null, queryVersion int not null, - foreign key (queryId) references queries(id), + foreign key (queryId) references queries(queryId), foreign key (textEntryId) references documentTextExtractions(id), - foreign key (queryId, queryVersion) references queryVersions(queryId, id) + foreign key (queryId, queryVersion) references queryVersions(queryId, versionId) ); CREATE TABLE resultDependencies ( diff --git a/internal/database/migrations/00000000000100_query_views.up.sql b/internal/database/migrations/00000000000100_query_views.up.sql index 348a5889..3cbbb367 100644 --- a/internal/database/migrations/00000000000100_query_views.up.sql +++ b/internal/database/migrations/00000000000100_query_views.up.sql @@ -1,20 +1,20 @@ CREATE VIEW queryCurrentActiveVersions as SELECT DISTINCT - q.id as queryId, + q.queryId, coalesce( - (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.id ORDER BY av.id DESC)), + (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)), 0 )::int as activeVersion FROM queries AS q - LEFT JOIN queryActiveVersions as av on av.queryId = q.id; + LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId; CREATE VIEW queryLatestVersions as SELECT - q.id as queryId, - coalesce(max(v.id), 0)::int as latestVersion + q.queryId, + coalesce(max(v.versionId), 0)::int as latestVersion FROM queries AS q - LEFT JOIN queryVersions as v on v.queryId = q.id - GROUP BY q.id; + LEFT JOIN queryVersions as v on v.queryId = q.queryId + GROUP BY q.queryId; CREATE VIEW queryCurrentConfigs as SELECT av.queryId, c.config @@ -39,12 +39,12 @@ CREATE VIEW queryCurrentRequiredIdsAGG as GROUP BY queryId; CREATE VIEW fullActiveQueries AS -SELECT DISTINCT q.id, q.type, av.activeVersion, lv.latestVersion, c.config, r.requiredIds +SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds FROM queries AS q - JOIN queryCurrentActiveVersions as av on q.id = av.queryId - JOIN queryLatestVersions as lv on lv.queryId = q.id - JOIN queryCurrentConfigs AS c ON q.id = c.queryId - JOIN queryCurrentRequiredIdsAGG AS r ON q.id = r.queryId; + JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.queryId + JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId + JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId; CREATE VIEW queryActiveDependencies AS WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS ( diff --git a/internal/database/migrations/00000000000101_collector_views.up.sql b/internal/database/migrations/00000000000101_collector_views.up.sql index 9cc969cc..43e69534 100644 --- a/internal/database/migrations/00000000000101_collector_views.up.sql +++ b/internal/database/migrations/00000000000101_collector_views.up.sql @@ -1,20 +1,20 @@ CREATE VIEW collectorCurrentActiveVersions as SELECT DISTINCT - j.id as clientId, + j.clientId, coalesce( - (FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)), + (FIRST_VALUE(v.versionId) OVER (PARTITION BY j.clientId ORDER BY v.id DESC)), 0 )::int as activeVersion FROM clients as j - LEFT JOIN collectorActiveVersions as v on v.clientId = j.id; + LEFT JOIN collectorActiveVersions as v on v.clientId = j.clientId; CREATE VIEW collectorLatestVersions as SELECT - j.id as clientId, + j.clientId, coalesce(max(v.id), 0)::int as latestVersion FROM clients as j - LEFT JOIN collectorVersions as v on v.clientId = j.id - GROUP BY j.id; + LEFT JOIN collectorVersions as v on v.clientId = j.clientId + GROUP BY j.clientId; CREATE VIEW currentCollectorMinTextVersions as SELECT DISTINCT @@ -71,7 +71,7 @@ CREATE VIEW collectorQueryDependencyTree AS FROM queryCurrentRequiredIdsAGG as q JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) ) - SELECT DISTINCT ct.clientId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds FROM collectorQueryDependencyTree as ct JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId - JOIN queries as q on q.id = ct.queryId; + JOIN queries as q on q.queryId = ct.queryId; diff --git a/internal/database/migrations/00000000000102_document_views.up.sql b/internal/database/migrations/00000000000102_document_views.up.sql index c9c2f905..75e89ee7 100644 --- a/internal/database/migrations/00000000000102_document_views.up.sql +++ b/internal/database/migrations/00000000000102_document_views.up.sql @@ -85,20 +85,20 @@ WITH RankedExtractions AS ( SELECT ccs.clientId, ccs.canSync, - ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.id DESC) as row_num + ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) as row_num FROM clients c - JOIN clientCanSync ccs on c.id = ccs.clientId + JOIN clientCanSync ccs on c.clientId = ccs.clientId ) SELECT - c.id as clientId, + c.clientId, coalesce(r.canSync, false) as canSync FROM clients c -LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1; +LEFT JOIN RankedExtractions r on r.clientId = c.clientId and r.row_num = 1; CREATE VIEW fullClients as -SELECT c.id, c.name, cs.canSync +SELECT c.clientId, c.name, cs.canSync FROM clients as c - JOIN currentClientCanSync as cs on cs.clientId = c.id; + JOIN currentClientCanSync as cs on cs.clientId = c.clientId; CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid) RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$ @@ -110,27 +110,32 @@ BEGIN CREATE TEMPORARY TABLE allResults AS WITH docs AS ( - SELECT id as documentId, hash, clientId + SELECT id as documentId, clientId FROM documents WHERE id = doc_id ), + tree as ( + select q.queryId, q.requiredIds, q.clientId, q.queryVersion + from collectorQueryDependencyTree as q + join docs as d on q.clientId = d.clientId + ), query_dependency_tree AS ( WITH RECURSIVE query_deps AS ( SELECT q.queryId, - unnest(q.requiredIds) AS requiredId - FROM collectorQueryDependencyTree AS q - JOIN docs AS d ON d.clientId = q.clientId + unnest(q.requiredIds) as requiredId + FROM tree AS q WHERE array_length(q.requiredIds, 1) > 0 UNION ALL SELECT qd.queryId, - unnest(q.requiredIds) AS requiredId + unnest(q.requiredIds) as requiredId FROM query_deps qd - JOIN collectorQueryDependencyTree q ON qd.requiredId = q.queryId + JOIN tree q ON qd.requiredId = q.queryId WHERE array_length(q.requiredIds, 1) > 0 + ) SELECT query_deps.queryId, @@ -147,7 +152,7 @@ BEGIN q.requiredIds as directRequiredIds, COALESCE(qdt.all_required_ids, ARRAY[]::uuid[]) AS allRequiredIds FROM docs AS d - JOIN collectorQueryDependencyTree AS q ON q.clientId = d.clientId + JOIN tree AS q ON q.clientId = d.clientId LEFT JOIN query_dependency_tree qdt ON q.queryId = qdt.queryId LEFT JOIN currentTextEntries AS tte ON tte.documentId = d.documentId LEFT JOIN results AS r diff --git a/internal/database/migrations_test.go b/internal/database/migrations_test.go index 4410594c..32952cae 100644 --- a/internal/database/migrations_test.go +++ b/internal/database/migrations_test.go @@ -5,28 +5,10 @@ import ( migrations "queryorchestration/internal/database" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/test" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func TestRunMigrations(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - ctx := t.Context() - cfg := &serviceconfig.BaseConfig{} - - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{ - NoMigrations: true, - }) - - err := migrations.RunMigrations(ctx, cfg) - require.NoError(t, err) -} - func TestRunMigrationsNoDB(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.SetDBUser("invalid_user") diff --git a/internal/database/queries/client.sql b/internal/database/queries/client.sql index e504800c..85256d55 100644 --- a/internal/database/queries/client.sql +++ b/internal/database/queries/client.sql @@ -1,11 +1,11 @@ -- name: CreateClient :exec -INSERT INTO clients (id, name) VALUES ($1, $2); +INSERT INTO clients (clientId, name) VALUES ($1, $2); -- name: GetClient :one -SELECT * FROM fullClients WHERE id = $1; +SELECT * FROM fullClients WHERE clientId = $1; -- name: UpdateClient :exec -UPDATE clients SET name = $1 WHERE id = $2; +UPDATE clients SET name = $1 WHERE clientId = $2; -- name: AddClientCanSync :exec INSERT INTO clientCanSync (canSync, clientId) VALUES ($1, $2); diff --git a/internal/database/queries/query.sql b/internal/database/queries/query.sql index 7eff4682..ef5ddc0e 100644 --- a/internal/database/queries/query.sql +++ b/internal/database/queries/query.sql @@ -2,16 +2,16 @@ SELECT config FROM queryCurrentConfigs where queryId = $1; -- name: GetQuery :one -SELECT * FROM fullActiveQueries WHERE id = $1; +SELECT * FROM fullActiveQueries WHERE queryId = $1; -- name: GetQueryWithVersion :one WITH query as ( - SELECT id, type FROM queries WHERE id = @id + SELECT queryId, queryType FROM queries WHERE queryId = @id ), config as ( SELECT c.queryId, c.config FROM query AS q - LEFT JOIN queryConfigs AS c ON q.id = c.queryId + LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId and isInVersion(@version, c.addedVersion, c.removedVersion) ), requiredIds as ( @@ -22,31 +22,31 @@ requiredIds as ( array[]::uuid[] )::uuid[] as requiredIds FROM query AS q - LEFT JOIN requiredQueries AS r ON q.id = r.queryId + LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId and isInVersion(@version, r.addedVersion, r.removedVersion) GROUP BY r.queryId ) -SELECT DISTINCT q.id, q.type, +SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds FROM query AS q - JOIN queryCurrentActiveVersions as av on q.id = av.queryId - JOIN queryLatestVersions as lv on lv.queryId = q.id - LEFT JOIN config AS c ON q.id = c.queryId - LEFT JOIN requiredIds AS r ON q.id = r.queryId; + JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.queryId + LEFT JOIN config AS c ON q.queryId = c.queryId + LEFT JOIN requiredIds AS r ON q.queryId = r.queryId; -- name: ListQueries :many SELECT * FROM fullActiveQueries; -- name: ListQueriesById :many -SELECT * FROM fullActiveQueries WHERE id = any($1); +SELECT * FROM fullActiveQueries WHERE queryId = any($1); -- name: CreateQuery :one -INSERT INTO queries (type) VALUES ($1) RETURNING id; +INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId; -- name: AddLatestQueryVersion :one -INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING id; +INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId; -- name: AddActiveQueryVersion :exec INSERT INTO queryActiveVersions (queryId, versionId) VALUES ($1, $2); @@ -61,9 +61,9 @@ UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and qu INSERT INTO queryConfigs (queryId, config, addedVersion) VALUES ($1, $2, $3); -- name: AllQueriesExist :one -SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist +SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist FROM unnest($1::uuid[]) AS input_id - LEFT JOIN queries ON input_id = queries.id; + LEFT JOIN queries ON input_id = queries.queryId; -- name: IsQueryInDependencyTree :one SELECT EXISTS ( diff --git a/internal/database/queries/result.sql b/internal/database/queries/result.sql index 200343b0..e1c718fa 100644 --- a/internal/database/queries/result.sql +++ b/internal/database/queries/result.sql @@ -1,9 +1,9 @@ -- name: ListQueryRequirementValues :many WITH reqQueries as ( - SELECT av.queryId, av.activeVersion, q.type + SELECT av.queryId, av.activeVersion, q.queryType FROM requiredQueries as rq JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId - JOIN queries as q on q.id = av.queryId + JOIN queries as q on q.queryId = av.queryId WHERE rq.queryId = @queryId and isInVersion(@version, rq.addedVersion, rq.removedVersion) ), @@ -25,7 +25,7 @@ latestVersions AS ( SELECT r.id, rq.queryId, - rq.type, + rq.queryType, r.queryVersion, r.textEntryId, r.value, @@ -39,7 +39,7 @@ latestVersions AS ( and r.queryVersion = rq.activeVersion and cte.id = r.textEntryId ) -SELECT DISTINCT id, queryId, type, value +SELECT DISTINCT id, queryId, queryType, value FROM latestVersions WHERE rowNumber = 1; diff --git a/internal/database/queries/text.sql b/internal/database/queries/text.sql index 4437e9dc..6f1742ab 100644 --- a/internal/database/queries/text.sql +++ b/internal/database/queries/text.sql @@ -8,12 +8,12 @@ SELECT id FROM currentTextEntries WHERE cleanId = @cleanEntryId and hash = @hash -- name: GetTextractOutputCurrentPart :one WITH client as ( - select id from clients where id = @clientId + select clientId from clients where clientId = @clientId ), parts as ( SELECT extract.part FROM client as c - JOIN documents as docs on docs.clientId = c.id + JOIN documents as docs on docs.clientId = c.clientId JOIN documentCleans as clean on docs.id = clean.documentId JOIN documentTextExtractions as extract on extract.cleanId = clean.id @@ -31,12 +31,12 @@ GROUP BY p.part; -- name: GetTextOutCurrentPart :one WITH client as ( - select id from clients where id = @clientId + select clientId from clients where clientId = @clientId ), parts as ( SELECT extract.part FROM client as c - JOIN documents as docs on docs.clientId = c.id + JOIN documents as docs on docs.clientId = c.clientId JOIN documentCleans as clean on docs.id = clean.documentId JOIN documentTextExtractions extract ON extract.cleanId = clean.id WHERE date(extract.createdAt) = date(@queryDate) diff --git a/internal/database/repository/clean_test.go b/internal/database/repository/clean_test.go index 1ada0152..df2baf26 100644 --- a/internal/database/repository/clean_test.go +++ b/internal/database/repository/clean_test.go @@ -20,15 +20,15 @@ func TestClean(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) diff --git a/internal/database/repository/client.sql.go b/internal/database/repository/client.sql.go index 84e6bee4..8a7d7477 100644 --- a/internal/database/repository/client.sql.go +++ b/internal/database/repository/client.sql.go @@ -27,33 +27,33 @@ func (q *Queries) AddClientCanSync(ctx context.Context, arg *AddClientCanSyncPar } const createClient = `-- name: CreateClient :exec -INSERT INTO clients (id, name) VALUES ($1, $2) +INSERT INTO clients (clientId, name) VALUES ($1, $2) ` type CreateClientParams struct { - ID string `db:"id"` - Name string `db:"name"` + Clientid string `db:"clientid"` + Name string `db:"name"` } // CreateClient // -// INSERT INTO clients (id, name) VALUES ($1, $2) +// INSERT INTO clients (clientId, name) VALUES ($1, $2) func (q *Queries) CreateClient(ctx context.Context, arg *CreateClientParams) error { - _, err := q.db.Exec(ctx, createClient, arg.ID, arg.Name) + _, err := q.db.Exec(ctx, createClient, arg.Clientid, arg.Name) return err } const getClient = `-- name: GetClient :one -SELECT id, name, cansync FROM fullClients WHERE id = $1 +SELECT clientid, name, cansync FROM fullClients WHERE clientId = $1 ` // GetClient // -// SELECT id, name, cansync FROM fullClients WHERE id = $1 -func (q *Queries) GetClient(ctx context.Context, id string) (*Fullclient, error) { - row := q.db.QueryRow(ctx, getClient, id) +// SELECT clientid, name, cansync FROM fullClients WHERE clientId = $1 +func (q *Queries) GetClient(ctx context.Context, clientid string) (*Fullclient, error) { + row := q.db.QueryRow(ctx, getClient, clientid) var i Fullclient - err := row.Scan(&i.ID, &i.Name, &i.Cansync) + err := row.Scan(&i.Clientid, &i.Name, &i.Cansync) return &i, err } @@ -258,18 +258,18 @@ func (q *Queries) IsClientSynced(ctx context.Context, dollar_1 *string) (bool, e } const updateClient = `-- name: UpdateClient :exec -UPDATE clients SET name = $1 WHERE id = $2 +UPDATE clients SET name = $1 WHERE clientId = $2 ` type UpdateClientParams struct { - Name string `db:"name"` - ID string `db:"id"` + Name string `db:"name"` + Clientid string `db:"clientid"` } // UpdateClient // -// UPDATE clients SET name = $1 WHERE id = $2 +// UPDATE clients SET name = $1 WHERE clientId = $2 func (q *Queries) UpdateClient(ctx context.Context, arg *UpdateClientParams) error { - _, err := q.db.Exec(ctx, updateClient, arg.Name, arg.ID) + _, err := q.db.Exec(ctx, updateClient, arg.Name, arg.Clientid) return err } diff --git a/internal/database/repository/client_test.go b/internal/database/repository/client_test.go index 11f317f7..26f7558b 100644 --- a/internal/database/repository/client_test.go +++ b/internal/database/repository/client_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestClient(t *testing.T) { +func TestClientQueries(t *testing.T) { t.Parallel() if testing.Short() { t.SkipNow() @@ -20,40 +20,40 @@ func TestClient(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() name := "example_name" clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: name, - ID: clientId, + Name: name, + Clientid: clientId, }) require.NoError(t, err) client, err := queries.GetClient(ctx, clientId) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullclient{ - ID: clientId, - Name: name, - Cansync: false, + Clientid: clientId, + Name: name, + Cansync: false, }, client) name = "updated_client" err = queries.UpdateClient(ctx, &repository.UpdateClientParams{ - ID: clientId, - Name: name, + Clientid: clientId, + Name: name, }) require.NoError(t, err) client, err = queries.GetClient(ctx, clientId) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullclient{ - ID: clientId, - Name: name, - Cansync: false, + Clientid: clientId, + Name: name, + Cansync: false, }, client) err = queries.AddClientCanSync(ctx, &repository.AddClientCanSyncParams{ @@ -65,29 +65,29 @@ func TestClient(t *testing.T) { client, err = queries.GetClient(ctx, clientId) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullclient{ - ID: clientId, - Name: name, - Cansync: true, + Clientid: clientId, + Name: name, + Cansync: true, }, client) err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: name, - ID: clientId, + Name: name, + Clientid: clientId, }) assert.Error(t, err) err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: name, - ID: "definitely different", + Name: name, + Clientid: "definitely different", }) assert.Error(t, err) err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "100 different", - ID: clientId, + Name: "100 different", + Clientid: clientId, }) assert.Error(t, err) err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "sameid", - ID: "sameid", + Name: "sameid", + Clientid: "sameid", }) require.NoError(t, err) } diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go index 5c4e751c..66a092a6 100644 --- a/internal/database/repository/collector.sql.go +++ b/internal/database/repository/collector.sql.go @@ -71,12 +71,12 @@ func (q *Queries) GetCollectorByClientID(ctx context.Context, clientid string) ( } const listCollectorQueries = `-- name: ListCollectorQueries :many -SELECT clientid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 +SELECT clientid, queryid, querytype, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 ` // ListCollectorQueries // -// SELECT clientid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 +// SELECT clientid, queryid, querytype, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 func (q *Queries) ListCollectorQueries(ctx context.Context, clientid string) ([]*Collectorquerydependencytree, error) { rows, err := q.db.Query(ctx, listCollectorQueries, clientid) if err != nil { @@ -89,7 +89,7 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, clientid string) ([] if err := rows.Scan( &i.Clientid, &i.Queryid, - &i.Type, + &i.Querytype, &i.Queryversion, &i.Requiredids, ); err != nil { diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index bfa558ab..e5e4a2fc 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -22,8 +22,8 @@ func TestCollector(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -48,8 +48,8 @@ func TestCollector(t *testing.T) { clientId := "EXAMPLE" err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) minCleanVersion := int64(2) @@ -147,14 +147,14 @@ func TestCollector(t *testing.T) { Clientid: clientId, Queryid: &jsonId, Queryversion: 1, - Type: repository.QuerytypeJsonExtractor, + Querytype: repository.QuerytypeJsonExtractor, Requiredids: []uuid.UUID{contextId}, }, { Clientid: clientId, Queryid: &contextId, Queryversion: 0, - Type: repository.QuerytypeContextFull, + Querytype: repository.QuerytypeContextFull, Requiredids: []uuid.UUID{}, }, }, qs) diff --git a/internal/database/repository/document_test.go b/internal/database/repository/document_test.go index 7d3cead1..1ee1d70d 100644 --- a/internal/database/repository/document_test.go +++ b/internal/database/repository/document_test.go @@ -20,15 +20,15 @@ func TestDocument(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) @@ -80,8 +80,8 @@ func TestDocument(t *testing.T) { clientTwoId := "EXAMPLE TWO" err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_name_two", - ID: clientTwoId, + Name: "example_name_two", + Clientid: clientTwoId, }) require.NoError(t, err) diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index 908961cb..0c6f0c43 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -178,12 +178,12 @@ func (e Querytype) Valid() bool { } type Client struct { - ID string `db:"id"` - Name string `db:"name"` + Clientid string `db:"clientid"` + Name string `db:"name"` } type Clientcansync struct { - ID uuid.UUID `db:"id"` + Syncid uuid.UUID `db:"syncid"` Clientid string `db:"clientid"` Cansync bool `db:"cansync"` } @@ -232,7 +232,7 @@ type Collectorquery struct { type Collectorquerydependencytree struct { Clientid string `db:"clientid"` Queryid *uuid.UUID `db:"queryid"` - Type Querytype `db:"type"` + Querytype Querytype `db:"querytype"` Queryversion int32 `db:"queryversion"` Requiredids []uuid.UUID `db:"requiredids"` } @@ -346,8 +346,8 @@ type Fullactivecollector struct { } type Fullactivequery struct { - ID uuid.UUID `db:"id"` - Type Querytype `db:"type"` + Queryid uuid.UUID `db:"queryid"` + Querytype Querytype `db:"querytype"` Activeversion int32 `db:"activeversion"` Latestversion int32 `db:"latestversion"` Config []byte `db:"config"` @@ -355,14 +355,14 @@ type Fullactivequery struct { } type Fullclient struct { - ID string `db:"id"` - Name string `db:"name"` - Cansync bool `db:"cansync"` + Clientid string `db:"clientid"` + Name string `db:"name"` + Cansync bool `db:"cansync"` } type Query struct { - ID uuid.UUID `db:"id"` - Type Querytype `db:"type"` + Queryid uuid.UUID `db:"queryid"` + Querytype Querytype `db:"querytype"` } type Queryactivedependency struct { @@ -371,13 +371,13 @@ type Queryactivedependency struct { } type Queryactiveversion struct { - ID uuid.UUID `db:"id"` - Queryid uuid.UUID `db:"queryid"` - Versionid int32 `db:"versionid"` + Activeversionentryid uuid.UUID `db:"activeversionentryid"` + Queryid uuid.UUID `db:"queryid"` + Versionid int32 `db:"versionid"` } type Queryconfig struct { - ID uuid.UUID `db:"id"` + Configid uuid.UUID `db:"configid"` Queryid uuid.UUID `db:"queryid"` Config []byte `db:"config"` Addedversion int32 `db:"addedversion"` @@ -410,17 +410,17 @@ type Querylatestversion struct { } type Queryversion struct { - Queryid uuid.UUID `db:"queryid"` - ID int32 `db:"id"` - Addedat pgtype.Timestamp `db:"addedat"` + Queryid uuid.UUID `db:"queryid"` + Versionid int32 `db:"versionid"` + Addedat pgtype.Timestamp `db:"addedat"` } type Requiredquery struct { - ID uuid.UUID `db:"id"` - Queryid uuid.UUID `db:"queryid"` - Requiredqueryid uuid.UUID `db:"requiredqueryid"` - Addedversion int32 `db:"addedversion"` - Removedversion *int32 `db:"removedversion"` + Requiredqueryentryid uuid.UUID `db:"requiredqueryentryid"` + Queryid uuid.UUID `db:"queryid"` + Requiredqueryid uuid.UUID `db:"requiredqueryid"` + Addedversion int32 `db:"addedversion"` + Removedversion *int32 `db:"removedversion"` } type Result struct { diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index cc1b63d1..d525d6ba 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -29,17 +29,17 @@ func (q *Queries) AddActiveQueryVersion(ctx context.Context, arg *AddActiveQuery } const addLatestQueryVersion = `-- name: AddLatestQueryVersion :one -INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING id +INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId ` // AddLatestQueryVersion // -// INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING id +// INSERT INTO queryVersions (queryId) VALUES ($1) RETURNING versionId func (q *Queries) AddLatestQueryVersion(ctx context.Context, queryid uuid.UUID) (int32, error) { row := q.db.QueryRow(ctx, addLatestQueryVersion, queryid) - var id int32 - err := row.Scan(&id) - return id, err + var versionid int32 + err := row.Scan(&versionid) + return versionid, err } const addRequiredQuery = `-- name: AddRequiredQuery :exec @@ -61,16 +61,16 @@ func (q *Queries) AddRequiredQuery(ctx context.Context, arg *AddRequiredQueryPar } const allQueriesExist = `-- name: AllQueriesExist :one -SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist +SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist FROM unnest($1::uuid[]) AS input_id - LEFT JOIN queries ON input_id = queries.id + LEFT JOIN queries ON input_id = queries.queryId ` // AllQueriesExist // -// SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist +// SELECT COUNT(*) = COUNT(DISTINCT queryId) AS all_exist // FROM unnest($1::uuid[]) AS input_id -// LEFT JOIN queries ON input_id = queries.id +// LEFT JOIN queries ON input_id = queries.queryId func (q *Queries) AllQueriesExist(ctx context.Context, dollar_1 []uuid.UUID) (bool, error) { row := q.db.QueryRow(ctx, allQueriesExist, dollar_1) var all_exist bool @@ -79,17 +79,17 @@ func (q *Queries) AllQueriesExist(ctx context.Context, dollar_1 []uuid.UUID) (bo } const createQuery = `-- name: CreateQuery :one -INSERT INTO queries (type) VALUES ($1) RETURNING id +INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId ` // CreateQuery // -// INSERT INTO queries (type) VALUES ($1) RETURNING id -func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (uuid.UUID, error) { - row := q.db.QueryRow(ctx, createQuery, type_) - var id uuid.UUID - err := row.Scan(&id) - return id, err +// INSERT INTO queries (queryType) VALUES ($1) RETURNING queryId +func (q *Queries) CreateQuery(ctx context.Context, querytype Querytype) (uuid.UUID, error) { + row := q.db.QueryRow(ctx, createQuery, querytype) + var queryid uuid.UUID + err := row.Scan(&queryid) + return queryid, err } const getActiveQueryConfig = `-- name: GetActiveQueryConfig :one @@ -107,18 +107,18 @@ func (q *Queries) GetActiveQueryConfig(ctx context.Context, queryid uuid.UUID) ( } const getQuery = `-- name: GetQuery :one -SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1 +SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = $1 ` // GetQuery // -// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1 -func (q *Queries) GetQuery(ctx context.Context, id uuid.UUID) (*Fullactivequery, error) { - row := q.db.QueryRow(ctx, getQuery, id) +// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = $1 +func (q *Queries) GetQuery(ctx context.Context, queryid uuid.UUID) (*Fullactivequery, error) { + row := q.db.QueryRow(ctx, getQuery, queryid) var i Fullactivequery err := row.Scan( - &i.ID, - &i.Type, + &i.Queryid, + &i.Querytype, &i.Activeversion, &i.Latestversion, &i.Config, @@ -129,12 +129,12 @@ func (q *Queries) GetQuery(ctx context.Context, id uuid.UUID) (*Fullactivequery, const getQueryWithVersion = `-- name: GetQueryWithVersion :one WITH query as ( - SELECT id, type FROM queries WHERE id = $1 + SELECT queryId, queryType FROM queries WHERE queryId = $1 ), config as ( SELECT c.queryId, c.config FROM query AS q - LEFT JOIN queryConfigs AS c ON q.id = c.queryId + LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId and isInVersion($2, c.addedVersion, c.removedVersion) ), requiredIds as ( @@ -145,19 +145,19 @@ requiredIds as ( array[]::uuid[] )::uuid[] as requiredIds FROM query AS q - LEFT JOIN requiredQueries AS r ON q.id = r.queryId + LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId and isInVersion($2, r.addedVersion, r.removedVersion) GROUP BY r.queryId ) -SELECT DISTINCT q.id, q.type, +SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds FROM query AS q - JOIN queryCurrentActiveVersions as av on q.id = av.queryId - JOIN queryLatestVersions as lv on lv.queryId = q.id - LEFT JOIN config AS c ON q.id = c.queryId - LEFT JOIN requiredIds AS r ON q.id = r.queryId + JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.queryId + LEFT JOIN config AS c ON q.queryId = c.queryId + LEFT JOIN requiredIds AS r ON q.queryId = r.queryId ` type GetQueryWithVersionParams struct { @@ -166,8 +166,8 @@ type GetQueryWithVersionParams struct { } type GetQueryWithVersionRow struct { - ID uuid.UUID `db:"id"` - Type Querytype `db:"type"` + Queryid uuid.UUID `db:"queryid"` + Querytype Querytype `db:"querytype"` Activeversion int32 `db:"activeversion"` Latestversion int32 `db:"latestversion"` Config []byte `db:"config"` @@ -177,12 +177,12 @@ type GetQueryWithVersionRow struct { // GetQueryWithVersion // // WITH query as ( -// SELECT id, type FROM queries WHERE id = $1 +// SELECT queryId, queryType FROM queries WHERE queryId = $1 // ), // config as ( // SELECT c.queryId, c.config // FROM query AS q -// LEFT JOIN queryConfigs AS c ON q.id = c.queryId +// LEFT JOIN queryConfigs AS c ON q.queryId = c.queryId // and isInVersion($2, c.addedVersion, c.removedVersion) // ), // requiredIds as ( @@ -193,25 +193,25 @@ type GetQueryWithVersionRow struct { // array[]::uuid[] // )::uuid[] as requiredIds // FROM query AS q -// LEFT JOIN requiredQueries AS r ON q.id = r.queryId +// LEFT JOIN requiredQueries AS r ON q.queryId = r.queryId // and isInVersion($2, r.addedVersion, r.removedVersion) // GROUP BY r.queryId // ) -// SELECT DISTINCT q.id, q.type, +// SELECT DISTINCT q.queryId, q.queryType, // av.activeVersion, // lv.latestVersion, c.config, // r.requiredIds // FROM query AS q -// JOIN queryCurrentActiveVersions as av on q.id = av.queryId -// JOIN queryLatestVersions as lv on lv.queryId = q.id -// LEFT JOIN config AS c ON q.id = c.queryId -// LEFT JOIN requiredIds AS r ON q.id = r.queryId +// JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId +// JOIN queryLatestVersions as lv on lv.queryId = q.queryId +// LEFT JOIN config AS c ON q.queryId = c.queryId +// LEFT JOIN requiredIds AS r ON q.queryId = r.queryId func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) { row := q.db.QueryRow(ctx, getQueryWithVersion, arg.ID, arg.Version) var i GetQueryWithVersionRow err := row.Scan( - &i.ID, - &i.Type, + &i.Queryid, + &i.Querytype, &i.Activeversion, &i.Latestversion, &i.Config, @@ -250,12 +250,12 @@ func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDep } const listQueries = `-- name: ListQueries :many -SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries +SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries ` // ListQueries // -// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries +// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) { rows, err := q.db.Query(ctx, listQueries) if err != nil { @@ -266,8 +266,8 @@ func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) { for rows.Next() { var i Fullactivequery if err := rows.Scan( - &i.ID, - &i.Type, + &i.Queryid, + &i.Querytype, &i.Activeversion, &i.Latestversion, &i.Config, @@ -284,14 +284,14 @@ func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) { } const listQueriesById = `-- name: ListQueriesById :many -SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1) +SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = any($1) ` // ListQueriesById // -// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1) -func (q *Queries) ListQueriesById(ctx context.Context, id []uuid.UUID) ([]*Fullactivequery, error) { - rows, err := q.db.Query(ctx, listQueriesById, id) +// SELECT queryid, querytype, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE queryId = any($1) +func (q *Queries) ListQueriesById(ctx context.Context, queryid []uuid.UUID) ([]*Fullactivequery, error) { + rows, err := q.db.Query(ctx, listQueriesById, queryid) if err != nil { return nil, err } @@ -300,8 +300,8 @@ func (q *Queries) ListQueriesById(ctx context.Context, id []uuid.UUID) ([]*Fulla for rows.Next() { var i Fullactivequery if err := rows.Scan( - &i.ID, - &i.Type, + &i.Queryid, + &i.Querytype, &i.Activeversion, &i.Latestversion, &i.Config, diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 05519b01..39b85d47 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -21,8 +21,8 @@ func TestQueries(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -32,8 +32,8 @@ func TestQueries(t *testing.T) { contextQuery, err := queries.GetQuery(ctx, contextQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: contextQueryID, - Type: repository.QuerytypeContextFull, + Queryid: contextQueryID, + Querytype: repository.QuerytypeContextFull, Activeversion: 0, Latestversion: 0, Config: nil, @@ -47,8 +47,8 @@ func TestQueries(t *testing.T) { contextQuery, err = queries.GetQuery(ctx, contextQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: contextQueryID, - Type: repository.QuerytypeContextFull, + Queryid: contextQueryID, + Querytype: repository.QuerytypeContextFull, Activeversion: 0, Latestversion: 1, Config: nil, @@ -61,8 +61,8 @@ func TestQueries(t *testing.T) { jsonQuery, err := queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 0, Latestversion: 0, Config: nil, @@ -76,8 +76,8 @@ func TestQueries(t *testing.T) { jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 0, Latestversion: 1, Config: nil, @@ -93,8 +93,8 @@ func TestQueries(t *testing.T) { jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 1, Latestversion: 1, Config: nil, @@ -113,8 +113,8 @@ func TestQueries(t *testing.T) { jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 1, Latestversion: 1, Config: nil, @@ -128,8 +128,8 @@ func TestQueries(t *testing.T) { jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 1, Latestversion: 2, Config: nil, @@ -179,8 +179,8 @@ func TestQueries(t *testing.T) { jsonQuery, err = queries.GetQuery(ctx, jsonQueryID) require.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivequery{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 2, Latestversion: 2, Config: jsonQueryConfig, @@ -194,8 +194,8 @@ func TestQueries(t *testing.T) { }) require.NoError(t, err) assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{ - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 2, Latestversion: 2, Config: jsonConfig, @@ -231,15 +231,15 @@ func TestQueryDependencyTree(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientID := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientID, + Name: "example_client", + Clientid: clientID, }) require.NoError(t, err) version, err := queries.AddLatestCollectorVersion(ctx, clientID) @@ -450,8 +450,8 @@ func TestQueriesList(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -466,16 +466,16 @@ func TestQueriesList(t *testing.T) { assert.Len(t, qs, 2) assert.ElementsMatch(t, []*repository.Fullactivequery{ { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 0, Latestversion: 0, Config: nil, Requiredids: []uuid.UUID{}, }, { - ID: contextQueryID, - Type: repository.QuerytypeContextFull, + Queryid: contextQueryID, + Querytype: repository.QuerytypeContextFull, Activeversion: 0, Latestversion: 0, Config: nil, @@ -488,8 +488,8 @@ func TestQueriesList(t *testing.T) { assert.Len(t, qs, 1) assert.ElementsMatch(t, []*repository.Fullactivequery{ { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, + Queryid: jsonQueryID, + Querytype: repository.QuerytypeJsonExtractor, Activeversion: 0, Latestversion: 0, Config: nil, @@ -506,8 +506,8 @@ func TestListQueryClients(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -520,8 +520,8 @@ func TestListQueryClients(t *testing.T) { clientOneID := "EXAMPLE" err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientOneID, + Name: "example_client", + Clientid: clientOneID, }) require.NoError(t, err) versionOne, err := queries.AddLatestCollectorVersion(ctx, clientOneID) @@ -545,8 +545,8 @@ func TestListQueryClients(t *testing.T) { clientTwoID := "EXAMPLE_DOS" err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client_dos", - ID: clientTwoID, + Name: "example_client_dos", + Clientid: clientTwoID, }) require.NoError(t, err) versionTwo, err := queries.AddLatestCollectorVersion(ctx, clientTwoID) diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index 107fc275..5f296a2b 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -103,10 +103,10 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many WITH reqQueries as ( - SELECT av.queryId, av.activeVersion, q.type + SELECT av.queryId, av.activeVersion, q.queryType FROM requiredQueries as rq JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId - JOIN queries as q on q.id = av.queryId + JOIN queries as q on q.queryId = av.queryId WHERE rq.queryId = $1 and isInVersion($2, rq.addedVersion, rq.removedVersion) ), @@ -128,7 +128,7 @@ latestVersions AS ( SELECT r.id, rq.queryId, - rq.type, + rq.queryType, r.queryVersion, r.textEntryId, r.value, @@ -142,7 +142,7 @@ latestVersions AS ( and r.queryVersion = rq.activeVersion and cte.id = r.textEntryId ) -SELECT DISTINCT id, queryId, type, value +SELECT DISTINCT id, queryId, queryType, value FROM latestVersions WHERE rowNumber = 1 ` @@ -154,19 +154,19 @@ type ListQueryRequirementValuesParams struct { } type ListQueryRequirementValuesRow struct { - ID *uuid.UUID `db:"id"` - Queryid uuid.UUID `db:"queryid"` - Type Querytype `db:"type"` - Value *string `db:"value"` + ID *uuid.UUID `db:"id"` + Queryid uuid.UUID `db:"queryid"` + Querytype Querytype `db:"querytype"` + Value *string `db:"value"` } // ListQueryRequirementValues // // WITH reqQueries as ( -// SELECT av.queryId, av.activeVersion, q.type +// SELECT av.queryId, av.activeVersion, q.queryType // FROM requiredQueries as rq // JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId -// JOIN queries as q on q.id = av.queryId +// JOIN queries as q on q.queryId = av.queryId // WHERE rq.queryId = $1 // and isInVersion($2, rq.addedVersion, rq.removedVersion) // ), @@ -188,7 +188,7 @@ type ListQueryRequirementValuesRow struct { // SELECT // r.id, // rq.queryId, -// rq.type, +// rq.queryType, // r.queryVersion, // r.textEntryId, // r.value, @@ -202,7 +202,7 @@ type ListQueryRequirementValuesRow struct { // and r.queryVersion = rq.activeVersion // and cte.id = r.textEntryId // ) -// SELECT DISTINCT id, queryId, type, value +// SELECT DISTINCT id, queryId, queryType, value // FROM latestVersions // WHERE rowNumber = 1 func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) { @@ -217,7 +217,7 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery if err := rows.Scan( &i.ID, &i.Queryid, - &i.Type, + &i.Querytype, &i.Value, ); err != nil { return nil, err diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index e260d044..0c61e8f8 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -23,125 +23,14 @@ func TestResults(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() - jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) - require.NoError(t, err) - _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: jsonQueryID, - Versionid: 1, - }) - require.NoError(t, err) - - clientId := "EXAMPLE" - err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, - }) - require.NoError(t, err) - - issynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, issynced) - - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: clientId, - Hash: "example_hash", - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, issynced) - - version, err := queries.AddLatestCollectorVersion(ctx, clientId) - require.NoError(t, err) - err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Versionid: version, - Clientid: clientId, - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, issynced) - - bucket := "example_bucket" - key := "example_key" - hash := "hash" - cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ - Documentid: documentID, - Bucket: &bucket, - Key: &key, - Hash: &hash, - Mimetype: repository.NullCleanmimetype{ - Valid: true, - Cleanmimetype: repository.CleanmimetypeApplicationPdf, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ - Cleanid: cleanid, - Version: 1, - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, issynced) - - textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Cleanid: cleanid, - Bucket: "hi", - Key: "hello", - Hash: "example", - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textId, - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, issynced) - - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Clientid: clientId, - Queryid: jsonQueryID, - Addedversion: 1, - Name: "example_key", - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, issynced) - - jsonResultValue := "example_value" - - _, err = queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: jsonResultValue, - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - issynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, issynced) + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, t.Context(), queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) qv := int32(1) res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ @@ -151,7 +40,7 @@ func TestResults(t *testing.T) { }) require.NoError(t, err) assert.NotNil(t, res.Value) - assert.Equal(t, jsonResultValue, *res.Value) + assert.Equal(t, "json_value", *res.Value) } func TestResultValues(t *testing.T) { @@ -162,8 +51,8 @@ func TestResultValues(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -172,8 +61,8 @@ func TestResultValues(t *testing.T) { clientId := "EXAMPLE" err = queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ @@ -296,7 +185,7 @@ func TestResultValues(t *testing.T) { require.NoError(t, err) assert.Len(t, qResults, 1) assert.Equal(t, contextQueryID, qResults[0].Queryid) - assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype) assert.Equal(t, "context_value_1", *qResults[0].Value) assert.NotEqual(t, uuid.UUID{}, qResults[0].ID) qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ @@ -323,7 +212,7 @@ func TestResultValues(t *testing.T) { require.NoError(t, err) assert.Len(t, qResults, 1) assert.Equal(t, contextQueryID, qResults[0].Queryid) - assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype) assert.Equal(t, "context_value_1", *qResults[0].Value) assert.NotEqual(t, uuid.UUID{}, qResults[0].ID) @@ -343,7 +232,7 @@ func TestResultValues(t *testing.T) { require.NoError(t, err) assert.Len(t, qResults, 1) assert.Equal(t, contextQueryID, qResults[0].Queryid) - assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype) assert.Equal(t, "context_value_3", *qResults[0].Value) assert.NotEqual(t, uuid.UUID{}, qResults[0].ID) @@ -363,257 +252,286 @@ func TestResultValues(t *testing.T) { require.NoError(t, err) assert.Len(t, qResults, 1) assert.Equal(t, contextQueryID, qResults[0].Queryid) - assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type) + assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Querytype) assert.Equal(t, "context_value_3", *qResults[0].Value) assert.NotEqual(t, uuid.UUID{}, qResults[0].ID) } func TestUnsyncedNoDepsQueries(t *testing.T) { - t.Parallel() if testing.Short() { t.SkipNow() } + t.Run("single document", func(t *testing.T) { + t.Parallel() + ctx := t.Context() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := createClientWithCollector(t, ctx, queries) + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + + err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Name: "example_name", + Queryid: jsonQueryID, + Addedversion: 1, + }) + require.NoError(t, err) + + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientId, + Hash: "example_hash", + }) + require.NoError(t, err) + + qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + + bucket := "example_bucket" + key := "example_key" + hash := "hahs" + cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + Documentid: documentID, + Bucket: &bucket, + Key: &key, + Hash: &hash, + Mimetype: repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Cleanid: cleanid, + Version: 1, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + + textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Cleanid: cleanid, + Bucket: "hi", + Key: "hello", + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textId, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*uuid.UUID{&contextQueryID}, qs) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "context_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "context_value", + Textentryid: textId, + Queryversion: 1, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + + _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) + require.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonQueryID, + Versionid: 2, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) + }) + + t.Run("two files", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := createClientWithCollector(t, ctx, queries) + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientId, + Hash: "example_hash_two", + }) + require.NoError(t, err) + + bucket := "example_bucket" + key := "example_key" + hash := "hahs" + + cleantwoid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + Documentid: documentTwoID, + Bucket: &bucket, + Key: &key, + Hash: &hash, + Mimetype: repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Cleanid: cleantwoid, + Version: 1, + }) + require.NoError(t, err) + + qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) + require.NoError(t, err) + assert.Len(t, qs, 0) + + textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Cleanid: cleantwoid, + Bucket: "hi", + Key: "hello", + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textTwoId, + }) + require.NoError(t, err) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: contextQueryID, + Value: "context_value", + Textentryid: textTwoId, + Queryversion: 1, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) + require.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: jsonQueryID, + Value: "context_value", + Textentryid: textTwoId, + Queryversion: 1, + }) + require.NoError(t, err) + + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 0) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) + require.NoError(t, err) + assert.Len(t, qs, 0) + }) + + t.Run("update query", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := createClientWithCollector(t, ctx, queries) + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + documentID, _, _ := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) + + _, err := queries.AddLatestQueryVersion(ctx, jsonQueryID) + require.NoError(t, err) + err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ + Queryid: jsonQueryID, + Versionid: 2, + }) + require.NoError(t, err) + + qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + require.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) + }) +} + +func BenchmarkListUnsynced(b *testing.B) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() - clientId := "EXAMPLE" - err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, - }) - require.NoError(t, err) - version, err := queries.AddLatestCollectorVersion(ctx, clientId) - require.NoError(t, err) - err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Versionid: version, - Clientid: clientId, - }) - require.NoError(t, err) - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: clientId, - Hash: "example_hash", - }) - require.NoError(t, err) - documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: clientId, - Hash: "example_hash_two", - }) - require.NoError(t, err) + clientId := createClientWithCollector(b, ctx, queries) + _, jsonQueryID := createDependentQueries(b, ctx, queries) - contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) - require.NoError(t, err) - jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) - require.NoError(t, err) - _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: jsonQueryID, - Versionid: 1, - }) - require.NoError(t, err) - _, err = queries.AddLatestQueryVersion(ctx, contextQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: contextQueryID, - Versionid: 1, - }) - require.NoError(t, err) - err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ - Queryid: jsonQueryID, - Requiredqueryid: contextQueryID, - Addedversion: 1, - }) - require.NoError(t, err) - - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Clientid: clientId, Name: "example_name", Queryid: jsonQueryID, Addedversion: 1, }) - require.NoError(t, err) + require.NoError(b, err) - qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - bucket := "example_bucket" - key := "example_key" - hash := "hahs" - cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ - Documentid: documentID, - Bucket: &bucket, - Key: &key, - Hash: &hash, - Mimetype: repository.NullCleanmimetype{ - Valid: true, - Cleanmimetype: repository.CleanmimetypeApplicationPdf, - }, + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Clientid: clientId, + Hash: "example_hash", }) - require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ - Cleanid: cleanid, - Version: 1, - }) - require.NoError(t, err) + require.NoError(b, err) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) + b.ResetTimer() - textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Cleanid: cleanid, - Bucket: "hi", - Key: "hello", - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textId, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*uuid.UUID{&contextQueryID}, qs) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - _, err = queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "context_value", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - _, err = queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "context_value", - Textentryid: textId, - Queryversion: 1, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - cleantwoid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ - Documentid: documentTwoID, - Bucket: &bucket, - Key: &key, - Hash: &hash, - Mimetype: repository.NullCleanmimetype{ - Valid: true, - Cleanmimetype: repository.CleanmimetypeApplicationPdf, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ - Cleanid: cleantwoid, - Version: 1, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ - Cleanid: cleantwoid, - Bucket: "hi", - Key: "hello", - Part: 0, - Createdat: pgtype.Timestamp{ - Time: time.Now().UTC(), - Valid: true, - }, - }) - require.NoError(t, err) - err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ - Version: 1, - Textid: textTwoId, - }) - require.NoError(t, err) - - _, err = queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: contextQueryID, - Value: "context_value", - Textentryid: textTwoId, - Queryversion: 1, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) - - _, err = queries.AddResult(ctx, &repository.AddResultParams{ - Queryid: jsonQueryID, - Value: "context_value", - Textentryid: textTwoId, - Queryversion: 1, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 0) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 0) - - _, err = queries.AddLatestQueryVersion(ctx, jsonQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: jsonQueryID, - Versionid: 2, - }) - require.NoError(t, err) - - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) - require.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) - qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentTwoID) - require.NoError(t, err) - assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*uuid.UUID{&jsonQueryID}, qs) + for b.Loop() { + _, _ = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, &documentID) + } } diff --git a/internal/database/repository/sync_test.go b/internal/database/repository/sync_test.go index 98e0d82e..bcfa6b66 100644 --- a/internal/database/repository/sync_test.go +++ b/internal/database/repository/sync_test.go @@ -2,6 +2,7 @@ package repository_test import ( "context" + "sync" "testing" "time" @@ -23,15 +24,15 @@ func TestListClientDocumentIDs(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() id := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: id, + Name: "example_client", + Clientid: id, }) require.NoError(t, err) @@ -118,29 +119,65 @@ func TestListClientDocumentIDs(t *testing.T) { }, ids) } +type docSyncStateParams struct { + clientId string + documentId uuid.UUID + hash string + fields string + isSynced bool +} + +func getDocumentSyncState(t testing.TB, ctx context.Context, queries *repository.Queries, params docSyncStateParams) { + var wg sync.WaitGroup + + wg.Add(1) + go func() { + isSynced, err := queries.IsClientSynced(ctx, ¶ms.clientId) + require.NoError(t, err) + assert.Equal(t, params.isSynced, isSynced) + wg.Done() + }() + + wg.Add(1) + go func() { + doc, err := queries.GetDocumentExternal(ctx, params.documentId) + require.NoError(t, err) + assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ + ID: params.documentId, + Clientid: params.clientId, + Hash: params.hash, + Fields: []byte(params.fields), + }, doc) + wg.Done() + }() + + wg.Wait() +} + +func createQuery(t testing.TB, queries *repository.Queries, queryType repository.Querytype) uuid.UUID { + id, err := queries.CreateQuery(t.Context(), queryType) + require.NoError(t, err) + + version, err := queries.AddLatestQueryVersion(t.Context(), id) + require.NoError(t, err) + + err = queries.AddActiveQueryVersion(t.Context(), &repository.AddActiveQueryVersionParams{ + Queryid: id, + Versionid: version, + }) + require.NoError(t, err) + + return id +} + func createDependentQueries(t testing.TB, ctx context.Context, queries *repository.Queries) (uuid.UUID, uuid.UUID) { - contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) - require.NoError(t, err) - contextQueryVersion, err := queries.AddLatestQueryVersion(ctx, contextQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: contextQueryID, - Versionid: contextQueryVersion, - }) - require.NoError(t, err) - jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) - require.NoError(t, err) - jsonQueryVersion, err := queries.AddLatestQueryVersion(ctx, jsonQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: jsonQueryID, - Versionid: jsonQueryVersion, - }) - require.NoError(t, err) - err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + contextQueryID := createQuery(t, queries, repository.QuerytypeContextFull) + jsonQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor) + + err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: jsonQueryID, Requiredqueryid: contextQueryID, - Addedversion: jsonQueryVersion, + Addedversion: 1, }) require.NoError(t, err) @@ -150,8 +187,8 @@ func createDependentQueries(t testing.TB, ctx context.Context, queries *reposito func createClientWithCollector(t testing.TB, ctx context.Context, queries *repository.Queries) string { clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) @@ -171,13 +208,65 @@ func TestClientSync(t *testing.T) { t.SkipNow() } + t.Run("no collector", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + clientId := "EXAMPLE" + err := queries.CreateClient(ctx, &repository.CreateClientParams{ + Name: "example_client", + Clientid: clientId, + }) + require.NoError(t, err) + + issynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, issynced) + }) + + t.Run("no documents", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + _, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + + isSynced, err := queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Queryid: jsonQueryID, + Addedversion: 1, + Name: "first_key", + }) + require.NoError(t, err) + + isSynced, err = queries.IsClientSynced(ctx, &clientId) + require.NoError(t, err) + assert.True(t, isSynced) + }) + t.Run("document fail clean", func(t *testing.T) { t.Parallel() ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -192,28 +281,19 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Clientid: clientId, Hash: "example_noclean", }) require.NoError(t, err) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_noclean", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_noclean", + isSynced: false, + fields: `{"first_key": null}`, + }) cleanid, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ Documentid: documentID, @@ -229,44 +309,36 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_noclean", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_noclean", + isSynced: true, + fields: `{"first_key": null}`, + }) }) - t.Run("standard extraction", func(t *testing.T) { + t.Run("single query", func(t *testing.T) { t.Parallel() ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() - contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + queryId := createQuery(t, queries, repository.QuerytypeContextFull) clientId := createClientWithCollector(t, ctx, queries) err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Clientid: clientId, - Queryid: jsonQueryID, + Queryid: queryId, Addedversion: 1, Name: "first_key", }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - bucket := "example_bucket" documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ @@ -275,19 +347,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - key := "example_key" hash := "hahssh" cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ @@ -307,17 +366,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ Cleanid: cleanId, @@ -337,17 +392,96 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) + + _, err = queries.AddResult(ctx, &repository.AddResultParams{ + Queryid: queryId, + Value: "json_value", + Textentryid: textId, + Queryversion: 1, + }) require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) + + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) + }) + + t.Run("multiple queries", func(t *testing.T) { + t.Parallel() + ctx := context.Background() + + cfg := &serviceconfig.BaseConfig{} + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) + + queries := cfg.GetDBQueries() + + contextQueryID, jsonQueryID := createDependentQueries(t, ctx, queries) + clientId := createClientWithCollector(t, ctx, queries) + + err := queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Clientid: clientId, + Queryid: jsonQueryID, + Addedversion: 1, + Name: "first_key", + }) require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, + + bucket := "example_bucket" + + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ Clientid: clientId, Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + }) + require.NoError(t, err) + + key := "example_key" + hash := "hahssh" + cleanId, err := queries.AddDocumentClean(ctx, &repository.AddDocumentCleanParams{ + Documentid: documentID, + Bucket: &bucket, + Key: &key, + Hash: &hash, + Mimetype: repository.NullCleanmimetype{ + Valid: true, + Cleanmimetype: repository.CleanmimetypeApplicationPdf, + }, + }) + require.NoError(t, err) + err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ + Cleanid: cleanId, + Version: 1, + }) + require.NoError(t, err) + + textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ + Cleanid: cleanId, + Part: 0, + Createdat: pgtype.Timestamp{ + Time: time.Now().UTC(), + Valid: true, + }, + Bucket: "hi", + Key: "hello", + Hash: "example", + }) + require.NoError(t, err) + err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ + Version: 1, + Textid: textId, + }) + require.NoError(t, err) contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, @@ -357,18 +491,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "json_value", @@ -377,17 +499,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: jsonResultId, @@ -395,18 +513,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_value"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) }) t.Run("update upstream query", func(t *testing.T) { @@ -414,8 +527,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -431,17 +544,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, @@ -451,18 +560,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "updated_context_value", @@ -471,35 +568,19 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: jsonResultId, Requiredresultid: contextResultId, }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "updated_context_value"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "updated_context_value"}`, + }) }) t.Run("update text extry", func(t *testing.T) { @@ -507,8 +588,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -533,17 +614,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) contextResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, @@ -553,18 +630,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "updated_text_json", @@ -573,35 +638,19 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: jsonResultID, Requiredresultid: contextResultID, }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "updated_text_json"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "updated_text_json"}`, + }) }) t.Run("update clean entry", func(t *testing.T) { @@ -609,8 +658,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -638,17 +687,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{ Bucket: "hi", @@ -667,18 +712,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "update_clean_context", @@ -687,18 +720,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - jsonResultID, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "update_clean_json", @@ -707,35 +728,19 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: jsonResultID, Requiredresultid: contextResultId, }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "update_clean_json"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "update_clean_json"}`, + }) }) t.Run("change collector name", func(t *testing.T) { @@ -743,8 +748,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -755,17 +760,13 @@ func TestClientSync(t *testing.T) { latestCollectorVersion, err := queries.AddLatestCollectorVersion(ctx, clientId) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_value"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: latestCollectorVersion, @@ -773,17 +774,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_value"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value"}`, + }) err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ Clientid: clientId, @@ -792,17 +789,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{}`, + }) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ Clientid: clientId, @@ -812,17 +805,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"second_key": "json_value"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"second_key": "json_value"}`, + }) }) t.Run("add existing query", func(t *testing.T) { @@ -830,8 +819,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -847,17 +836,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_value", "example_key": "example_context"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_value", "example_key": "example_context"}`, + }) }) t.Run("add super query", func(t *testing.T) { @@ -865,8 +850,8 @@ func TestClientSync(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -874,42 +859,30 @@ func TestClientSync(t *testing.T) { clientId := createClientWithCollector(t, ctx, queries) documentID, _, textId := createDocumentWithCollectorAndResults(t, ctx, queries, clientId, contextQueryID, jsonQueryID) - superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) - require.NoError(t, err) - superQueryVersion, err := queries.AddLatestQueryVersion(ctx, superQueryID) - require.NoError(t, err) - err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{ - Queryid: superQueryID, - Versionid: superQueryVersion, - }) - require.NoError(t, err) - err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + superQueryID := createQuery(t, queries, repository.QuerytypeJsonExtractor) + err := queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: contextQueryID, Requiredqueryid: superQueryID, - Addedversion: superQueryVersion, + Addedversion: 1, }) require.NoError(t, err) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) + superResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: superQueryID, Value: "super_value", Textentryid: textId, - Queryversion: superQueryVersion, + Queryversion: 1, }) require.NoError(t, err) - isSynced, err := queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err := queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - contextResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: contextQueryID, Value: "context_with_super_value", @@ -918,18 +891,6 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - jsonResultId, err := queries.AddResult(ctx, &repository.AddResultParams{ Queryid: jsonQueryID, Value: "json_with_super", @@ -938,35 +899,19 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) - err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: contextResultId, Requiredresultid: superResultId, }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.False(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": null}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: false, + fields: `{"first_key": null}`, + }) err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{ Resultid: jsonResultId, @@ -974,17 +919,13 @@ func TestClientSync(t *testing.T) { }) require.NoError(t, err) - isSynced, err = queries.IsClientSynced(ctx, &clientId) - require.NoError(t, err) - assert.True(t, isSynced) - docExternal, err = queries.GetDocumentExternal(ctx, documentID) - require.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetDocumentExternalRow{ - ID: documentID, - Clientid: clientId, - Hash: "example_hash", - Fields: []byte(`{"first_key": "json_with_super"}`), - }, docExternal) + getDocumentSyncState(t, ctx, queries, docSyncStateParams{ + clientId: clientId, + documentId: documentID, + hash: "example_hash", + isSynced: true, + fields: `{"first_key": "json_with_super"}`, + }) }) } @@ -1070,8 +1011,8 @@ func BenchmarkIsClientSynced(b *testing.B) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(b, ctx) - _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() @@ -1090,8 +1031,8 @@ func BenchmarkGetDocumentExternal(b *testing.B) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(b, ctx) - _ = test.CreateDB(b, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(b) + test.CreateDB(b, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() diff --git a/internal/database/repository/text.sql.go b/internal/database/repository/text.sql.go index 42912dff..7ae7af3c 100644 --- a/internal/database/repository/text.sql.go +++ b/internal/database/repository/text.sql.go @@ -116,12 +116,12 @@ func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID) const getTextOutCurrentPart = `-- name: GetTextOutCurrentPart :one WITH client as ( - select id from clients where id = $1 + select clientId from clients where clientId = $1 ), parts as ( SELECT extract.part FROM client as c - JOIN documents as docs on docs.clientId = c.id + JOIN documents as docs on docs.clientId = c.clientId JOIN documentCleans as clean on docs.id = clean.documentId JOIN documentTextExtractions extract ON extract.cleanId = clean.id WHERE date(extract.createdAt) = date($2) @@ -150,12 +150,12 @@ type GetTextOutCurrentPartRow struct { // GetTextOutCurrentPart // // WITH client as ( -// select id from clients where id = $1 +// select clientId from clients where clientId = $1 // ), // parts as ( // SELECT extract.part // FROM client as c -// JOIN documents as docs on docs.clientId = c.id +// JOIN documents as docs on docs.clientId = c.clientId // JOIN documentCleans as clean on docs.id = clean.documentId // JOIN documentTextExtractions extract ON extract.cleanId = clean.id // WHERE date(extract.createdAt) = date($2) @@ -178,12 +178,12 @@ func (q *Queries) GetTextOutCurrentPart(ctx context.Context, arg *GetTextOutCurr const getTextractOutputCurrentPart = `-- name: GetTextractOutputCurrentPart :one WITH client as ( - select id from clients where id = $1 + select clientId from clients where clientId = $1 ), parts as ( SELECT extract.part FROM client as c - JOIN documents as docs on docs.clientId = c.id + JOIN documents as docs on docs.clientId = c.clientId JOIN documentCleans as clean on docs.id = clean.documentId JOIN documentTextExtractions as extract on extract.cleanId = clean.id @@ -213,12 +213,12 @@ type GetTextractOutputCurrentPartRow struct { // GetTextractOutputCurrentPart // // WITH client as ( -// select id from clients where id = $1 +// select clientId from clients where clientId = $1 // ), // parts as ( // SELECT extract.part // FROM client as c -// JOIN documents as docs on docs.clientId = c.id +// JOIN documents as docs on docs.clientId = c.clientId // JOIN documentCleans as clean on docs.id = clean.documentId // JOIN documentTextExtractions as extract // on extract.cleanId = clean.id diff --git a/internal/database/repository/text_test.go b/internal/database/repository/text_test.go index d0d36cd7..e993ef7b 100644 --- a/internal/database/repository/text_test.go +++ b/internal/database/repository/text_test.go @@ -23,15 +23,15 @@ func TestTextExtraction(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) @@ -131,15 +131,15 @@ func TestTextTextractPart(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) @@ -261,15 +261,15 @@ func TestTextOutPart(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) queries := cfg.GetDBQueries() clientId := "EXAMPLE" err := queries.CreateClient(ctx, &repository.CreateClientParams{ - Name: "example_client", - ID: clientId, + Name: "example_client", + Clientid: clientId, }) require.NoError(t, err) diff --git a/internal/query/parse.go b/internal/query/parse.go index aa2aee39..6f5ec88d 100644 --- a/internal/query/parse.go +++ b/internal/query/parse.go @@ -9,8 +9,8 @@ import ( func ParseQueryWithVersion(q *repository.GetQueryWithVersionRow) (*Query, error) { return ParseFullActiveQuery(&repository.Fullactivequery{ - ID: q.ID, - Type: q.Type, + Queryid: q.Queryid, + Querytype: q.Querytype, Activeversion: q.Activeversion, Latestversion: q.Latestversion, Config: q.Config, @@ -29,7 +29,7 @@ func ParseQuery(q *Query) *resultprocessor.Query { } func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { - qType, err := resultprocessor.ParseDBType(q.Type) + qType, err := resultprocessor.ParseDBType(q.Querytype) if err != nil { return nil, err } @@ -46,7 +46,7 @@ func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { } return &Query{ - ID: q.ID, + ID: q.Queryid, ActiveVersion: q.Activeversion, LatestVersion: q.Latestversion, Type: qType, diff --git a/internal/query/parse_test.go b/internal/query/parse_test.go index 733aae61..14f542d9 100644 --- a/internal/query/parse_test.go +++ b/internal/query/parse_test.go @@ -37,8 +37,8 @@ func TestParseQuery(t *testing.T) { func TestParseFullActiveQuery(t *testing.T) { q := &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), Requiredids: []uuid.UUID{ @@ -51,7 +51,7 @@ func TestParseFullActiveQuery(t *testing.T) { require.NoError(t, err) bcfg := string(q.Config) assert.EqualExportedValues(t, query.Query{ - ID: q.ID, + ID: q.Queryid, Type: resultprocessor.TypeContextFull, ActiveVersion: q.Activeversion, LatestVersion: q.Latestversion, @@ -64,8 +64,8 @@ func TestParseFullActiveQuery(t *testing.T) { func TestFullActiveQueryEmpty(t *testing.T) { dbQuery := &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), } @@ -73,7 +73,7 @@ func TestFullActiveQueryEmpty(t *testing.T) { out, err := query.ParseFullActiveQuery(dbQuery) require.NoError(t, err) assert.EqualExportedValues(t, query.Query{ - ID: dbQuery.ID, + ID: dbQuery.Queryid, Type: resultprocessor.TypeContextFull, ActiveVersion: int32(1), LatestVersion: int32(2), @@ -82,8 +82,8 @@ func TestFullActiveQueryEmpty(t *testing.T) { func TestFullActiveQueryWithNullUUID(t *testing.T) { dbQuery := &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), } @@ -91,7 +91,7 @@ func TestFullActiveQueryWithNullUUID(t *testing.T) { out, err := query.ParseFullActiveQuery(dbQuery) require.NoError(t, err) assert.EqualExportedValues(t, query.Query{ - ID: dbQuery.ID, + ID: dbQuery.Queryid, Type: resultprocessor.TypeContextFull, ActiveVersion: int32(1), LatestVersion: int32(2), @@ -101,8 +101,8 @@ func TestFullActiveQueryWithNullUUID(t *testing.T) { func TestFullActiveQueryArray(t *testing.T) { dbQueries := []*repository.Fullactivequery{ { - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: int32(1), Latestversion: int32(2), }, @@ -112,7 +112,7 @@ func TestFullActiveQueryArray(t *testing.T) { require.NoError(t, err) assert.EqualExportedValues(t, []*query.Query{ { - ID: dbQueries[0].ID, + ID: dbQueries[0].Queryid, Type: resultprocessor.TypeContextFull, ActiveVersion: int32(1), LatestVersion: int32(2), diff --git a/internal/query/result/process.go b/internal/query/result/process.go index 1d06c6fc..9728036b 100644 --- a/internal/query/result/process.go +++ b/internal/query/result/process.go @@ -87,7 +87,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *res func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesRow) ([]resultprocessor.Value, error) { values := make([]resultprocessor.Value, len(v)) for index, r := range v { - qType, err := resultprocessor.ParseDBType(r.Type) + qType, err := resultprocessor.ParseDBType(r.Querytype) if err != nil { return nil, err } diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go index 9278a44c..0cdc232f 100644 --- a/internal/query/result/process_test.go +++ b/internal/query/result/process_test.go @@ -140,9 +140,9 @@ func TestParseQueryRequirementValueArray(t *testing.T) { exval := "exampleval" in := []*repository.ListQueryRequirementValuesRow{ { - Queryid: uuid.New(), - Value: &exval, - Type: repository.QuerytypeJsonExtractor, + Queryid: uuid.New(), + Value: &exval, + Querytype: repository.QuerytypeJsonExtractor, }, } diff --git a/internal/query/result/processor/parse.go b/internal/query/result/processor/parse.go index 8f29dbf4..e66710d5 100644 --- a/internal/query/result/processor/parse.go +++ b/internal/query/result/processor/parse.go @@ -66,7 +66,7 @@ func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) { } func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, error) { - qType, err := ParseDBType(q.Type) + qType, err := ParseDBType(q.Querytype) if err != nil { return nil, err } @@ -89,7 +89,7 @@ func ParseFullQuery(qs *repository.Fullactivequery) (*Query, error) { return nil, nil } - qt, err := ParseDBType(qs.Type) + qt, err := ParseDBType(qs.Querytype) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func ParseFullQuery(qs *repository.Fullactivequery) (*Query, error) { } return &Query{ - ID: qs.ID, + ID: qs.Queryid, Type: qt, Version: qs.Activeversion, RequiredQueryIDs: reqIds, diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go index 87ac059c..c2cedb6d 100644 --- a/internal/query/result/processor/parse_test.go +++ b/internal/query/result/processor/parse_test.go @@ -16,7 +16,7 @@ func TestParseDBCollectorQuery(t *testing.T) { Clientid: "hello", Queryid: &uuid.UUID{}, Requiredids: []uuid.UUID{}, - Type: repository.QuerytypeJsonExtractor, + Querytype: repository.QuerytypeJsonExtractor, Queryversion: 1, } value, err := resultprocessor.ParseDBCollectorQuery(&dbResult) @@ -26,7 +26,7 @@ func TestParseDBCollectorQuery(t *testing.T) { assert.Equal(t, int32(1), value.Version) assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value.Type) - dbResult.Type = repository.Querytype("") + dbResult.Querytype = repository.Querytype("") _, err = resultprocessor.ParseDBCollectorQuery(&dbResult) assert.EqualError(t, err, "invalid database query type") } @@ -116,8 +116,8 @@ func TestParseFullQuery(t *testing.T) { assert.Nil(t, out) q = &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: 1, Latestversion: 2, } @@ -125,14 +125,14 @@ func TestParseFullQuery(t *testing.T) { out, err = resultprocessor.ParseFullQuery(q) require.NoError(t, err) assert.EqualExportedValues(t, resultprocessor.Query{ - ID: q.ID, + ID: q.Queryid, Type: resultprocessor.TypeContextFull, Version: 1, }, *out) q = &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: 1, Latestversion: 2, Requiredids: []uuid.UUID{}, @@ -141,14 +141,14 @@ func TestParseFullQuery(t *testing.T) { out, err = resultprocessor.ParseFullQuery(q) require.NoError(t, err) assert.EqualExportedValues(t, &resultprocessor.Query{ - ID: q.ID, + ID: q.Queryid, Type: resultprocessor.TypeContextFull, Version: 1, }, out) q = &repository.Fullactivequery{ - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: 1, Latestversion: 2, Config: []byte("hello"), @@ -161,7 +161,7 @@ func TestParseFullQuery(t *testing.T) { require.NoError(t, err) cfg := "hello" assert.EqualExportedValues(t, &resultprocessor.Query{ - ID: q.ID, + ID: q.Queryid, Type: resultprocessor.TypeContextFull, Version: 1, Config: &cfg, @@ -177,8 +177,8 @@ func TestParseFullQueryArray(t *testing.T) { q = []*repository.Fullactivequery{ { - ID: uuid.New(), - Type: repository.QuerytypeContextFull, + Queryid: uuid.New(), + Querytype: repository.QuerytypeContextFull, Activeversion: 1, Latestversion: 2, }, @@ -188,7 +188,7 @@ func TestParseFullQueryArray(t *testing.T) { require.NoError(t, err) assert.EqualExportedValues(t, []*resultprocessor.Query{ { - ID: q[0].ID, + ID: q[0].Queryid, Type: resultprocessor.TypeContextFull, Version: 1, }, diff --git a/internal/server/api/listener_test.go b/internal/server/api/listener_test.go index c5c83d83..5405e94d 100644 --- a/internal/server/api/listener_test.go +++ b/internal/server/api/listener_test.go @@ -28,8 +28,8 @@ func TestNewAPI(t *testing.T) { cfg := &BaseConfig{} _ = serviceconfig.InitializeConfig(cfg) - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{ + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{ NoMigrations: true, }) diff --git a/internal/server/runner/listener_test.go b/internal/server/runner/listener_test.go index 3f21547d..c07ce81b 100644 --- a/internal/server/runner/listener_test.go +++ b/internal/server/runner/listener_test.go @@ -3,7 +3,6 @@ package runner import ( "context" "testing" - "time" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/objectstore" @@ -26,27 +25,23 @@ func TestNewRunner(t *testing.T) { if testing.Short() { t.SkipNow() } - ctx := context.Background() cfg := &TestConfig{} _ = serviceconfig.InitializeConfig(cfg) - net := test.GetNetwork(t, ctx) + net := test.GetNetwork(t) - a := test.CreateAWSContainer(t, ctx, cfg, net) - test.SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{ + a := test.CreateAWSContainer(t, cfg, net) + test.SetQueueClient(t, t.Context(), cfg, a.ExternalEndpoint) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{ NoMigrations: true, }) - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - cfg.ControllerFunc = func() Controller[interface{}] { return runnermock.NewMockController[interface{}](t) } - cfg.QueueURL = test.CreateQueue(t, ctx, cfg, "queueName") + cfg.QueueURL = test.CreateQueue(t, t.Context(), cfg, "queueName") cfg.SetSQSEndpoint(a.ExternalEndpoint) - srvPtr, err := New(ctx, cfg) + srvPtr, err := New(t.Context(), cfg) require.NoError(t, err) assert.NotNil(t, srvPtr) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go index a8b32338..d8786d71 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -22,9 +22,9 @@ func TestNewServer(t *testing.T) { cfg := &server.BaseConfig{} _ = serviceconfig.InitializeConfig(cfg) - net := test.GetNetwork(t, ctx) + net := test.GetNetwork(t) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{ + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{ NoMigrations: true, }) diff --git a/internal/serviceconfig/database/pool_test.go b/internal/serviceconfig/database/pool_test.go index 9f34405c..67c45d61 100644 --- a/internal/serviceconfig/database/pool_test.go +++ b/internal/serviceconfig/database/pool_test.go @@ -21,8 +21,8 @@ func TestSetDBPool(t *testing.T) { ctx := context.Background() cfg := &serviceconfig.BaseConfig{} - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) + net := test.GetNetwork(t) + test.CreateDB(t, cfg, net, &test.CreateDatabaseConfig{}) err := cfg.SetDBPool(ctx) require.NoError(t, err) diff --git a/internal/serviceconfig/database/transaction_test.go b/internal/serviceconfig/database/transaction_test.go index 2f683072..87480d16 100644 --- a/internal/serviceconfig/database/transaction_test.go +++ b/internal/serviceconfig/database/transaction_test.go @@ -30,8 +30,8 @@ func TestExecuteTransaction(t *testing.T) { err = cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { err := q.CreateClient(ctx, &repository.CreateClientParams{ - Name: clientName, - ID: clientID, + Name: clientName, + Clientid: clientID, }) require.NoError(t, err) diff --git a/internal/serviceconfig/objectstore/config_test.go b/internal/serviceconfig/objectstore/config_test.go index 2f315b00..63f7814d 100644 --- a/internal/serviceconfig/objectstore/config_test.go +++ b/internal/serviceconfig/objectstore/config_test.go @@ -94,8 +94,8 @@ func TestCalculateAndCompareETag(t *testing.T) { cfg := &StoreConfig{} - net := test.GetNetwork(t, ctx) - acfg := test.CreateAWSContainer(t, ctx, cfg, net) + net := test.GetNetwork(t) + acfg := test.CreateAWSContainer(t, cfg, net) test.SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint) test.CreateBucket(t, ctx, cfg) diff --git a/internal/test/api_test.go b/internal/test/api_test.go deleted file mode 100644 index c3b7ca94..00000000 --- a/internal/test/api_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package test_test - -import ( - "testing" - - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/test" - - "github.com/stretchr/testify/assert" -) - -func TestCreateAPI(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := t.Context() - - cfg := &serviceconfig.BaseConfig{} - _ = serviceconfig.InitializeConfig(cfg) - net := test.GetNetwork(t, ctx) - _ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{NoMigrations: true}) - - acfg := &test.APIConfig{ - API: test.QueryAPI, - } - - conn, cleanup := test.CreateAPI(t, ctx, cfg, net, acfg) - assert.NotNil(t, conn) - assert.NotNil(t, cleanup) - - cleanup() -} diff --git a/internal/test/assertions_test.go b/internal/test/assertions_test.go index a369ebfa..e398de16 100644 --- a/internal/test/assertions_test.go +++ b/internal/test/assertions_test.go @@ -6,7 +6,6 @@ import ( "queryorchestration/internal/test" - "github.com/aws/aws-sdk-go-v2/service/textract" "github.com/stretchr/testify/assert" ) @@ -22,8 +21,3 @@ func TestAssertStringToReader(t *testing.T) { assert.True(t, mockT.Failed()) }) } - -func TestGetTextractFileResponse(t *testing.T) { - out := test.GetTextractFileResponse(t, "../../assets/textract/helloWorld.gen") - assert.IsType(t, []map[string]*textract.AnalyzeDocumentOutput{}, out) -} diff --git a/internal/test/aws.go b/internal/test/aws.go index 759a0c16..0ae3750b 100644 --- a/internal/test/aws.go +++ b/internal/test/aws.go @@ -1,7 +1,6 @@ package test import ( - "context" "fmt" "io" "net/http" @@ -18,7 +17,6 @@ import ( ) type AWSContainerConfig struct { - Container testcontainers.Container ExternalEndpoint string } @@ -32,7 +30,7 @@ const ( awsPort = 4566 ) -func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider, network string) *AWSContainerConfig { +func CreateAWSContainer(t testing.TB, cfg AWSConfigProvider, network string) *AWSContainerConfig { port, err := nat.NewPort("tcp", strconv.Itoa(awsPort)) require.NoError(t, err) @@ -78,20 +76,21 @@ func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider }, } - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + container, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, Reuse: true, }) require.NoError(t, err) - host, err := container.Host(ctx) + host, err := container.Host(t.Context()) require.NoError(t, err) - mappedPort, err := container.MappedPort(ctx, port) + mappedPort, err := container.MappedPort(t.Context(), port) require.NoError(t, err) + awsAddress := fmt.Sprintf("http://%s:%s", host, mappedPort.Port()) + return &AWSContainerConfig{ - Container: container, - ExternalEndpoint: fmt.Sprintf("http://%s:%s", host, mappedPort.Port()), + ExternalEndpoint: awsAddress, } } diff --git a/internal/test/aws_test.go b/internal/test/aws_test.go deleted file mode 100644 index a3f85cdd..00000000 --- a/internal/test/aws_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package test - -import ( - "context" - "testing" - - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/objectstore" - - "github.com/stretchr/testify/assert" -) - -type TestAWSConfig struct { - serviceconfig.BaseConfig - objectstore.ObjectStoreConfig -} - -func TestCreateQueueContainer(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - cfg := &TestAWSConfig{} - - net := GetNetwork(t, ctx) - - qcfg := CreateAWSContainer(t, ctx, cfg, net) - assert.NotNil(t, qcfg) -} diff --git a/internal/test/container.go b/internal/test/container.go index 61cef384..c89fbfde 100644 --- a/internal/test/container.go +++ b/internal/test/container.go @@ -38,7 +38,8 @@ type containerConfig struct { } const ( - imageTag = "queryorchestration:latest" + imageTag = "queryorchestration:latest" + dockerfilePath = "build/Dockerfile" ) var ( @@ -50,8 +51,6 @@ func buildImage(t testing.TB, ctx context.Context) { dockerContextPath, err := os.Getwd() require.NoError(t, err, "Failed to get current directory") - dockerfilePath := "build/Dockerfile" - for { candidatePath := filepath.Join(dockerContextPath, dockerfilePath) @@ -67,7 +66,40 @@ func buildImage(t testing.TB, ctx context.Context) { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err) - buildContext, err := archive.TarWithOptions(dockerContextPath, &archive.TarOptions{}) + excludePatterns := []string{ + "LICENSE", + "README.md", + "**/*_test.go", + "cmd/*_test/", + ".bitbucket/", + ".devbox/", + ".vscode/", + "deployments/", + "mocks/", + "out/", + "serviceAPIs/", + "test/", + "docs/", + "assets/", + "pkg/", + "internal/test/", + "Taskfile.yml", + "scripts/", + ".gitattributes", + ".gitignore", + "devbox.*", + "sqlc.yml", + "vaccum.conf.yaml", + ".*.yml", + "**/*.cast", + "**/*.http", + } + + buildContext, err := archive.TarWithOptions(dockerContextPath, &archive.TarOptions{ + Compression: archive.Uncompressed, + NoLchown: true, + ExcludePatterns: excludePatterns, + }) require.NoError(t, err) defer buildContext.Close() @@ -136,8 +168,7 @@ func createContainer(t testing.TB, ctx context.Context, network string, cfg *con Started: true, }) if err != nil { - PrintContainerLogs(t, ctx, container) - + PrintContainerLogs(t, container) require.NoError(t, err) } @@ -149,13 +180,13 @@ func createContainer(t testing.TB, ctx context.Context, network string, cfg *con } } -func PrintContainerLogs(t testing.TB, ctx context.Context, container testcontainers.Container) { - logs, err := container.Logs(ctx) +func PrintContainerLogs(t testing.TB, container testcontainers.Container) { + logs, err := container.Logs(t.Context()) + require.NoError(t, err) defer func() { err := logs.Close() require.NoError(t, err) }() - require.NoError(t, err) scanner := bufio.NewScanner(logs) for scanner.Scan() { diff --git a/internal/test/container_test.go b/internal/test/container_test.go index 1d7adf5c..2eddf65c 100644 --- a/internal/test/container_test.go +++ b/internal/test/container_test.go @@ -1,13 +1,8 @@ package test import ( - "context" "testing" - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/objectstore" - - "github.com/docker/go-connections/nat" "github.com/gruntwork-io/terratest/modules/docker" "github.com/stretchr/testify/assert" ) @@ -21,38 +16,3 @@ func TestBuildImage(t *testing.T) { exists := docker.DoesImageExist(t, imageTag, nil) assert.True(t, exists) } - -type Cfg struct { - serviceconfig.BaseConfig - objectstore.ObjectStoreConfig -} - -func TestCreateContainer(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - - cfg := &Cfg{} - - deps, clean := CreateFullDependencies(t, ctx, cfg) - defer clean() - - ccfg := &containerConfig{ - Name: string(QueryAPIName), - DownstreamQueues: QueryAPI.DownstreamQueues, - Cfg: cfg, - ExposedPorts: []nat.Port{ - nat.Port("8080/tcp"), - }, - } - - container, cleanup := createContainer(t, ctx, deps.Network, ccfg) - assert.NotNil(t, container) - assert.NotNil(t, cleanup) - - PrintContainerLogs(t, ctx, container) - - cleanup() -} diff --git a/internal/test/database.go b/internal/test/database.go index b898addc..9ff7cae8 100644 --- a/internal/test/database.go +++ b/internal/test/database.go @@ -1,7 +1,6 @@ package test import ( - "context" "fmt" "strconv" "testing" @@ -24,13 +23,13 @@ const ( dbPort = 5432 ) -func CreateDB(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, network string, dcfg *CreateDatabaseConfig) testcontainers.Container { +func CreateDB(t testing.TB, cfg serviceconfig.ConfigProvider, network string, dcfg *CreateDatabaseConfig) testcontainers.Container { port, err := nat.NewPort("tcp", strconv.Itoa(dbPort)) require.NoError(t, err) cfg.SetDBUser("pass") cfg.SetDBSecret("postgres") - cfg.SetDBName(NormaliseAlias(fmt.Sprintf("queryorchestration_%s", t.Name()))) + cfg.SetDBName(GetAlias(t, "postgres")) cfg.SetDBNoSSL(true) req := testcontainers.ContainerRequest{ @@ -57,26 +56,26 @@ func CreateDB(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvide }, } - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + container, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, Reuse: true, }) require.NoError(t, err) - host, err := container.Host(ctx) + host, err := container.Host(t.Context()) require.NoError(t, err) - mappedPort, err := container.MappedPort(ctx, port) + mappedPort, err := container.MappedPort(t.Context(), port) require.NoError(t, err) cfg.SetDBHost(host) cfg.SetDBPort(mappedPort.Int()) if !dcfg.NoMigrations { - err := db.RunMigrations(ctx, cfg) + err := db.RunMigrations(t.Context(), cfg) require.NoError(t, err) - err = cfg.SetDBPool(ctx) + err = cfg.SetDBPool(t.Context()) require.NoError(t, err) } diff --git a/internal/test/database_test.go b/internal/test/database_test.go deleted file mode 100644 index f75e330a..00000000 --- a/internal/test/database_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package test_test - -import ( - "context" - "testing" - - "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/test" - - "github.com/stretchr/testify/assert" -) - -func TestCreateDB(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - - cfg := &serviceconfig.BaseConfig{} - - net := test.GetNetwork(t, ctx) - dbcfg := test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{NoMigrations: true}) - - assert.NotNil(t, dbcfg) - assert.Nil(t, cfg.GetDBPool()) -} - -func TestCreateDBWithMigrations(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - - cfg := &serviceconfig.BaseConfig{} - - net := test.GetNetwork(t, ctx) - dbcfg := test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{}) - - assert.NotNil(t, dbcfg) - assert.NotNil(t, cfg.GetDBPool()) -} diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index e13e76f6..63eb05ad 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -107,7 +107,7 @@ type Dependencies struct { } func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Dependencies, func()) { - network := GetNetwork(t, ctx) + network := GetNetwork(t) deps := Dependencies{ Network: network, @@ -119,33 +119,32 @@ func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenc wg.Add(1) go func() { - mockServer, cleanMock := CreateMockServer(t, ctx, network) - deps.MockServer = mockServer - cs["mock"] = cleanMock + deps.AWSConfig = CreateAWSContainer(t, cfg, network) + + SetQueueClient(t, ctx, cfg, deps.AWSConfig.ExternalEndpoint) + + for _, runner := range runners { + wg.Add(1) + go func() { + deps.QueueURLs[runner.Name] = CreateQueue(t, ctx, cfg, runner.Name) + wg.Done() + }() + } + + SetStoreClient(t, ctx, cfg, deps.AWSConfig.ExternalEndpoint) + CreateBucket(t, ctx, cfg) + SetBucketNotifs(t, ctx, cfg) + wg.Done() }() - wg.Add(1) - go func() { - deps.DBConfig = CreateDB(t, ctx, cfg, network, &CreateDatabaseConfig{}) - wg.Done() - }() + mockServer, cleanMock := CreateMockServer(t, ctx, network) + deps.MockServer = mockServer + cs["mock"] = cleanMock - deps.AWSConfig = CreateAWSContainer(t, ctx, cfg, network) - - SetQueueClient(t, ctx, cfg, deps.AWSConfig.ExternalEndpoint) - - for _, runner := range runners { - wg.Add(1) - go func() { - deps.QueueURLs[runner.Name] = CreateQueue(t, ctx, cfg, runner.Name) - wg.Done() - }() - } - - SetStoreClient(t, ctx, cfg, deps.AWSConfig.ExternalEndpoint) - CreateBucket(t, ctx, cfg) - SetBucketNotifs(t, ctx, cfg) + CreateDB(t, cfg, network, &CreateDatabaseConfig{ + NoMigrations: true, + }) wg.Wait() @@ -211,7 +210,7 @@ type MockExpectation struct { } func CreateMockServer(t testing.TB, ctx context.Context, network string) (MockServer, func()) { - name := NormaliseAlias(fmt.Sprintf("mockserver_%s", t.Name())) + name := GetAlias(t, "mockserver") port, err := nat.NewPort("tcp", "1080") require.NoError(t, err) @@ -258,11 +257,13 @@ func CreateMockServer(t testing.TB, ctx context.Context, network string) (MockSe } } -func NormaliseAlias(fullName string) string { - name := strings.ToLower(fullName) +func GetAlias(t testing.TB, baseName string) string { + name := fmt.Sprintf("%s_%s", baseName, t.Name()) + name = strings.ToLower(name) name = strings.ReplaceAll(name, "/", "") name = strings.ReplaceAll(name, " ", "") name = strings.ReplaceAll(name, "-", "") + name = strings.ReplaceAll(name, "_", "") return name } @@ -348,9 +349,7 @@ func WaitForMockEndpoint(t testing.TB, server MockServer, request MockRequest) M verifyURL := fmt.Sprintf("%s/mockserver/verify", server.External) clientTimeout := 500 * time.Millisecond - client := &http.Client{ - Timeout: clientTimeout, - } + client := &http.Client{} timeout := time.After(60 * time.Second) ticker := time.NewTicker(clientTimeout) diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index b1d9bbf2..e52f9529 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -1,7 +1,6 @@ package test import ( - "context" "io" "net/http" "net/http/httptest" @@ -56,11 +55,9 @@ func TestWaitForMockEndpoint(t *testing.T) { if testing.Short() { t.SkipNow() } - ctx := context.Background() + net := GetNetwork(t) - net := GetNetwork(t, ctx) - - server, cleanup := CreateMockServer(t, ctx, net) + server, cleanup := CreateMockServer(t, t.Context(), net) defer cleanup() body := strings.NewReader(`{"team":"hello"}`) diff --git a/internal/test/network.go b/internal/test/network.go index 1c00847c..9e2d81cd 100644 --- a/internal/test/network.go +++ b/internal/test/network.go @@ -1,7 +1,6 @@ package test import ( - "context" "fmt" "log/slog" "sync" @@ -20,12 +19,12 @@ var ( networkOnce sync.Once ) -func GetNetwork(t testing.TB, ctx context.Context) string { +func GetNetwork(t testing.TB) string { networkOnce.Do(func() { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) require.NoError(t, err) - _, err = cli.NetworkCreate(ctx, networkName, network.CreateOptions{ + _, err = cli.NetworkCreate(t.Context(), networkName, network.CreateOptions{ Driver: network.NetworkBridge, }) conflictErrorMsg := fmt.Sprintf("Error response from daemon: network with name %s already exists", networkName) diff --git a/internal/test/network_test.go b/internal/test/network_test.go index 410a28b2..cd460e6e 100644 --- a/internal/test/network_test.go +++ b/internal/test/network_test.go @@ -1,7 +1,6 @@ package test_test import ( - "context" "testing" "queryorchestration/internal/test" @@ -10,11 +9,9 @@ import ( ) func TestDepNetworkGet(t *testing.T) { - ctx := context.Background() - - name := test.GetNetwork(t, ctx) + name := test.GetNetwork(t) assert.NotNil(t, name) - newName := test.GetNetwork(t, ctx) + newName := test.GetNetwork(t) assert.Equal(t, name, newName) } diff --git a/internal/test/objectstore.go b/internal/test/objectstore.go index b346deb4..fac1eea4 100644 --- a/internal/test/objectstore.go +++ b/internal/test/objectstore.go @@ -19,7 +19,7 @@ import ( ) func GetBucketName(t testing.TB) string { - return NormaliseAlias(t.Name()) + return GetAlias(t, "bucket") } func CreateBucket(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider) { diff --git a/internal/test/objectstore_test.go b/internal/test/objectstore_test.go index 3e072c04..ae4b2633 100644 --- a/internal/test/objectstore_test.go +++ b/internal/test/objectstore_test.go @@ -11,7 +11,6 @@ import ( objectstoremock "queryorchestration/mocks/objectstore" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -30,30 +29,13 @@ func TestCreateBucket(t *testing.T) { cfg := &StoreConfig{} - net := GetNetwork(t, ctx) - acfg := CreateAWSContainer(t, ctx, cfg, net) + net := GetNetwork(t) + acfg := CreateAWSContainer(t, cfg, net) SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint) CreateBucket(t, ctx, cfg) } -func TestCreateStoreClient(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - - cfg := &StoreConfig{} - - net := GetNetwork(t, ctx) - - acfg := CreateAWSContainer(t, ctx, cfg, net) - - SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint) - assert.NotNil(t, cfg.GetStoreClient()) -} - func TestPutObject(t *testing.T) { ctx := context.Background() @@ -71,7 +53,7 @@ func TestPutObject(t *testing.T) { PutObject( mock.Anything, mock.MatchedBy(func(in *s3.PutObjectInput) bool { - return *in.Bucket == NormaliseAlias(t.Name()) && *in.Key == key.String() + return *in.Bucket == GetAlias(t, "bucket") && *in.Key == key.String() }), mock.Anything, ). diff --git a/internal/test/queryAPI/service.go b/internal/test/queryAPI/service.go index f0e4ef15..0b21956d 100644 --- a/internal/test/queryAPI/service.go +++ b/internal/test/queryAPI/service.go @@ -4,11 +4,14 @@ import ( "context" "fmt" "log/slog" + "net/http" "testing" "time" queryapi "queryorchestration/pkg/queryAPI" + "github.com/google/uuid" + "github.com/oapi-codegen/runtime/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,7 +41,7 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli t.Helper() timeout := time.After(60 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) + ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { @@ -54,6 +57,7 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli if jRes.JSON200.Status == status { assert.Equal(t, status, jRes.JSON200.Status) + slog.Info("returned client status", "status", jRes.JSON200.Status) return } @@ -61,3 +65,33 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli } } } + +func CreateDependentQueries(t testing.TB, client *queryapi.ClientWithResponses, clientId string) (uuid.UUID, uuid.UUID) { + contextQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{ + Type: queryapi.CONTEXTFULL, + }) + require.NoError(t, err) + require.Equal(t, http.StatusCreated, contextQueryRes.StatusCode()) + jcfg := `{"path":"keyone"}` + jsonQueryRes, err := client.CreateQueryWithResponse(t.Context(), queryapi.QueryCreate{ + Type: queryapi.JSONEXTRACTOR, + Config: &jcfg, + RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id}, + }) + require.NoError(t, err) + require.Equal(t, http.StatusCreated, jsonQueryRes.StatusCode()) + newActiveVersion := int32(1) + collRes, err := client.SetCollectorByClientIdWithResponse(t.Context(), clientId, queryapi.CollectorSet{ + ActiveVersion: &newActiveVersion, + Fields: &[]queryapi.CollectorField{ + { + Name: "JSON_QUERY", + QueryId: jsonQueryRes.JSON201.Id, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, http.StatusOK, collRes.StatusCode()) + + return contextQueryRes.JSON201.Id, jsonQueryRes.JSON201.Id +} diff --git a/internal/test/queue.go b/internal/test/queue.go index 729b4263..e7306775 100644 --- a/internal/test/queue.go +++ b/internal/test/queue.go @@ -47,7 +47,7 @@ func GetQueueArn(t testing.TB, cfg awsc.ConfigProvider, name RunnerName) string } func GetQueueName(t testing.TB, name RunnerName) string { - return NormaliseAlias(fmt.Sprintf("%s_%s", name, t.Name())) + return GetAlias(t, string(name)) } func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, name RunnerName) string { diff --git a/internal/test/queue_test.go b/internal/test/queue_test.go index 46e5ff08..8beb587b 100644 --- a/internal/test/queue_test.go +++ b/internal/test/queue_test.go @@ -31,14 +31,14 @@ func TestCreateQueue(t *testing.T) { cfg := &TestConfig{} - net := GetNetwork(t, ctx) + net := GetNetwork(t) - a := CreateAWSContainer(t, ctx, cfg, net) + a := CreateAWSContainer(t, cfg, net) SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) url := CreateQueue(t, ctx, cfg, "myname") - assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/myname_testcreatequeue", url) + assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/mynametestcreatequeue", url) } func TestAssertMessageWait(t *testing.T) { @@ -50,9 +50,9 @@ func TestAssertMessageWait(t *testing.T) { cfg := &TestConfig{} - net := GetNetwork(t, ctx) + net := GetNetwork(t) - a := CreateAWSContainer(t, ctx, cfg, net) + a := CreateAWSContainer(t, cfg, net) SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) url := CreateQueue(t, ctx, cfg, "myname") @@ -77,9 +77,9 @@ func TestAssertMessageBodyWait(t *testing.T) { cfg := &TestConfig{} - net := GetNetwork(t, ctx) + net := GetNetwork(t) - a := CreateAWSContainer(t, ctx, cfg, net) + a := CreateAWSContainer(t, cfg, net) SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) url := CreateQueue(t, ctx, cfg, "myname") @@ -101,9 +101,9 @@ func TestAssertMessageAttrWait(t *testing.T) { cfg := &TestConfig{} - net := GetNetwork(t, ctx) + net := GetNetwork(t) - a := CreateAWSContainer(t, ctx, cfg, net) + a := CreateAWSContainer(t, cfg, net) SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) url := CreateQueue(t, ctx, cfg, "myname") @@ -129,12 +129,12 @@ func TestGetQueueURL(t *testing.T) { cfg := &awsc.AWSConfig{ AWSRegion: "us-east-1", } - assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/docinitrunner_testgetqueueurl", GetQueueURL(t, cfg, DocInitRunnerName)) + assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/docinitrunnertestgetqueueurl", GetQueueURL(t, cfg, DocInitRunnerName)) } func TestGetQueueARN(t *testing.T) { cfg := &awsc.AWSConfig{ AWSRegion: "us-east-1", } - assert.Equal(t, "arn:aws:sqs:us-east-1:000000000000:docinitrunner_testgetqueuearn", GetQueueArn(t, cfg, DocInitRunnerName)) + assert.Equal(t, "arn:aws:sqs:us-east-1:000000000000:docinitrunnertestgetqueuearn", GetQueueArn(t, cfg, DocInitRunnerName)) } diff --git a/internal/test/runner_test.go b/internal/test/runner_test.go index d3b318dd..90b8cb90 100644 --- a/internal/test/runner_test.go +++ b/internal/test/runner_test.go @@ -1,39 +1,11 @@ package test import ( - "context" "testing" "github.com/stretchr/testify/assert" ) -func TestCreateRunner(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - ctx := context.Background() - - cfg := &TestConfig{} - - net := GetNetwork(t, ctx) - - a := CreateAWSContainer(t, ctx, cfg, net) - SetQueueClient(t, ctx, cfg, a.ExternalEndpoint) - _ = CreateQueue(t, ctx, cfg, QueryRunnerName) - _ = CreateDB(t, ctx, cfg, net, &CreateDatabaseConfig{NoMigrations: true}) - - qccfg := &RunnerConfig{ - Runner: QueryRunner, - } - - c, cleanup := CreateRunner(t, ctx, cfg, net, qccfg) - assert.NotNil(t, cleanup) - assert.NotNil(t, c) - - cleanup() -} - func TestGetRunnerEnvFromName(t *testing.T) { assert.Equal(t, StoreEventRunnerEnv, GetRunnerEnvFromName(StoreEventRunnerName)) assert.Equal(t, DocInitRunnerEnv, GetRunnerEnvFromName(DocInitRunnerName)) diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 756da28e..1b117b72 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -4,24 +4,23 @@ version: "3" vars: - CONTEXT: .. OUT_DIR: out includes: test: - dir: "{{.CONTEXT}}" + dir: ".." taskfile: tests.yml docker: - dir: "." + dir: ".." taskfile: docker.yml openapi: - dir: "{{.CONTEXT}}" + dir: ".." taskfile: openapi-scripts.yml compose: - dir: "." + dir: ".." taskfile: local-deployments.yml db: - dir: "{{.CONTEXT}}" + dir: ".." taskfile: database.yml tasks: diff --git a/scripts/database.yml b/scripts/database.yml index a662a0c1..11cfa27f 100644 --- a/scripts/database.yml +++ b/scripts/database.yml @@ -5,10 +5,12 @@ version: "3" vars: MIGRATIONS: "internal/database/migrations" + QUERIES: "internal/database/queries/" + CONFIG_FILE: "sqlc.yml" includes: compose: - dir: "{{.CONTEXT}}" + dir: ".." taskfile: local-deployments.yml internal: true @@ -18,14 +20,14 @@ tasks: cmds: - task: compose:up:generate - migrate -path {{.MIGRATIONS}} -database {{.DB_URI_GENERATE}} up - - sqlc generate --file sqlc.yml + - sqlc generate --file {{.CONFIG_FILE}} lint: cmds: - task: compose:up:generate - - sqlc vet --file sqlc.yml + - sqlc vet --file {{.CONFIG_FILE}} - | exit 0 - for file in internal/database/migrations/*.sql; do + for file in {{.MIGRATIONS}}/*.sql; do if [[ -f "$file" ]]; then sqlcheck -c -f $file -r 3 # move to 1 fi diff --git a/scripts/docker.yml b/scripts/docker.yml index f5dd9dda..53f3093e 100644 --- a/scripts/docker.yml +++ b/scripts/docker.yml @@ -4,7 +4,7 @@ version: "3" vars: - DOCKERFILE: "{{.CONTEXT}}/build/Dockerfile" + DOCKERFILE: "build/Dockerfile" tasks: lint: @@ -13,4 +13,4 @@ tasks: build: run: once cmds: - - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} {{.CONTEXT}} + - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} . diff --git a/scripts/local-deployments.yml b/scripts/local-deployments.yml index a3765495..74eea8ba 100644 --- a/scripts/local-deployments.yml +++ b/scripts/local-deployments.yml @@ -4,10 +4,10 @@ version: "3" vars: - LOCAL_COMPOSE_FILE: "{{.CONTEXT}}/deployments/compose.local.yaml" - TEST_COMPOSE_FILE: "{{.CONTEXT}}/deployments/compose.test.yaml" - GENERATE_COMPOSE_FILE: "{{.CONTEXT}}/deployments/compose.generate.yaml" - AWS_COMPOSE_FILE: "{{.CONTEXT}}/deployments/compose.aws.yaml" + LOCAL_COMPOSE_FILE: "deployments/compose.local.yaml" + TEST_COMPOSE_FILE: "deployments/compose.test.yaml" + GENERATE_COMPOSE_FILE: "deployments/compose.generate.yaml" + AWS_COMPOSE_FILE: "deployments/compose.aws.yaml" tasks: build:test: diff --git a/scripts/tests.yml b/scripts/tests.yml index c5dfd1aa..a9625a90 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -10,7 +10,22 @@ vars: API: "./api/..." PKG: "./pkg/..." CMD: "./cmd/..." - TEST_PARALLEL: 32 + CPU_COUNT: + sh: | + # First try nproc (Linux) + if command -v nproc >/dev/null 2>&1; then + nproc + # Fall back to sysctl (macOS) + elif [ "$(uname)" = "Darwin" ]; then + sysctl -n hw.ncpu + # Fall back to Windows approach + elif [ "$(uname)" = "Windows" ] || [ "$(uname)" = "MINGW"* ]; then + echo "$NUMBER_OF_PROCESSORS" + else + echo "4" # Default fallback value + fi + TEST_PARALLEL: + sh: echo $(( {{.CPU_COUNT}} * 2)) # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go" @@ -19,12 +34,12 @@ tasks: run: once cmds: - rm -rf mocks/* - - go tool mockery + - go tool mockery --log-level="" functional: cmds: - mkdir -p {{.OUT_DIR}} - | - go test -count=1 -parallel {{.TEST_PARALLEL}} \ + GOMAXPROCS={{.TEST_PARALLEL}} go test -count=1 -parallel {{.CPU_COUNT}} \ -coverpkg={{.INTERNAL}},{{.API}},{{.PKG}} \ -coverprofile={{.COVERAGE_FILE}} \ ./... diff --git a/sqlc.yml b/sqlc.yml index 4fe1e05b..eacfeb26 100644 --- a/sqlc.yml +++ b/sqlc.yml @@ -13,7 +13,7 @@ sql: - no-delete - no-pg - no-seq-scan - # - too-costly + - too-costly database: managed: true gen: @@ -52,7 +52,7 @@ rules: query.sql.contains("DELETE") - name: too-costly message: "Query cost estimate is too high" - rule: "postgresql.explain.plan.total_cost > 400.0" + rule: "postgresql.explain.plan.total_cost > 1000000000.0" - name: no-seq-scan message: "Query plan results in a sequential scan" rule: "postgresql.explain.plan.node_type == 'Seq Scan'" diff --git a/test/process_test.go b/test/process_test.go index 94775bb8..ee2f1073 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -2,7 +2,6 @@ package endtoend_test import ( "context" - "net/http" "strings" "testing" "time" @@ -15,7 +14,6 @@ import ( queryapi "queryorchestration/pkg/queryAPI" "github.com/google/uuid" - "github.com/oapi-codegen/runtime/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,32 +33,7 @@ func TestProcess(t *testing.T) { defer clean() client := queryapitest.CreateClientWithSync(t, ctx, net.Client) - - contextQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{ - Type: queryapi.CONTEXTFULL, - }) - require.NoError(t, err) - require.Equal(t, http.StatusCreated, contextQueryRes.StatusCode()) - jcfg := `{"path":"keyone"}` - jsonQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{ - Type: queryapi.JSONEXTRACTOR, - Config: &jcfg, - RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id}, - }) - require.NoError(t, err) - require.Equal(t, http.StatusCreated, jsonQueryRes.StatusCode()) - newActiveVersion := int32(1) - collRes, err := net.Client.SetCollectorByClientIdWithResponse(ctx, client.Id, queryapi.CollectorSet{ - ActiveVersion: &newActiveVersion, - Fields: &[]queryapi.CollectorField{ - { - Name: "JSON_QUERY", - QueryId: jsonQueryRes.JSON201.Id, - }, - }, - }) - require.NoError(t, err) - require.Equal(t, http.StatusOK, collRes.StatusCode()) + _, jsonId := queryapitest.CreateDependentQueries(t, net.Client, client.Id) queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC) @@ -124,9 +97,9 @@ func TestProcess(t *testing.T) { fullDoc := *docRes.JSON200 assert.EqualExportedValues(t, expectedDoc, fullDoc) - jcfg = `{"path":"keytwo"}` + jcfg := `{"path":"keytwo"}` av := int32(2) - _, err = net.Client.UpdateQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryUpdate{ + _, err = net.Client.UpdateQueryWithResponse(ctx, jsonId, queryapi.QueryUpdate{ ActiveVersion: &av, Config: &jcfg, }) @@ -144,13 +117,13 @@ func TestProcess(t *testing.T) { require.NotNil(t, docRes.JSON200) assert.EqualExportedValues(t, expectedDoc, *docRes.JSON200) - testRes, err := net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryTestRequest{ + testRes, err := net.Client.TestQueryWithResponse(ctx, jsonId, queryapi.QueryTestRequest{ QueryVersion: 1, DocumentId: doc.Id, }) require.NoError(t, err) assert.Equal(t, "valueone", testRes.JSON200.Value) - testRes, err = net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryTestRequest{ + testRes, err = net.Client.TestQueryWithResponse(ctx, jsonId, queryapi.QueryTestRequest{ QueryVersion: 2, DocumentId: doc.Id, }) diff --git a/test/queryAPI/accessory_test.go b/test/queryAPI/accessory_test.go index f943fe0d..a980a9fe 100644 --- a/test/queryAPI/accessory_test.go +++ b/test/queryAPI/accessory_test.go @@ -41,4 +41,6 @@ func TestQueryAPIAccessories(t *testing.T) { require.NoError(t, err) assert.NotNil(t, resp) assert.Equal(t, http.StatusOK, resp.StatusCode) + + test.PrintContainerLogs(t, c.API.Container) }