From 62886dbba81a759e46a1218b6cb8d86446a2fd3e Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Mon, 10 Mar 2025 11:03:00 +0000 Subject: [PATCH] Merged in feature/removejob (pull request #95) Remove Job * removejob * rmjob * sync * cleanup * precommit * startslow * startslow * startslow * openapi * clean * test * scripts * littlecleanercmds * mermaid --- .golangci.yml | 1 + .../runner.go | 10 +- .../runner_test.go | 20 +- api/docCleanRunner/runner_test.go | 10 +- api/docInitRunner/runner.go | 6 +- api/docInitRunner/runner_test.go | 38 +- api/docSyncRunner/runner_test.go | 39 +- api/queryService/api.gen.go | 397 ++++----- api/queryService/client.go | 2 +- api/queryService/client_test.go | 37 +- .../{jobcollector.go => collector.go} | 22 +- ...jobcollector_test.go => collector_test.go} | 56 +- api/queryService/controllers.go | 6 +- api/queryService/documents.go | 4 +- api/queryService/documents_test.go | 14 +- api/queryService/job.go | 69 -- api/queryService/job_test.go | 225 ----- api/queryService/parse.go | 8 +- api/queryService/parse_test.go | 13 +- api/queryService/query_test.go | 2 +- api/queryService/status.go | 21 + api/queryService/status_test.go | 70 ++ api/queryVersionSyncRunner/runner_test.go | 20 +- .../main.go | 18 +- cmd/docCleanRunner/main.go | 2 +- cmd/docInitRunner/main.go | 2 +- cmd/docSyncRunner/main.go | 21 +- cmd/docTextRunner/main.go | 2 +- cmd/queryService/main.go | 24 +- cmd/queryVersionSyncRunner/main.go | 6 +- database/diagram.mmd | 37 +- ...sql => 00000000000004_collectors.down.sql} | 0 ...p.sql => 00000000000004_collectors.up.sql} | 42 +- .../migrations/00000000000004_jobs.down.sql | 1 - .../migrations/00000000000004_jobs.up.sql | 12 - ....sql => 00000000000005_documents.down.sql} | 0 ...up.sql => 00000000000005_documents.up.sql} | 6 +- ...wn.sql => 00000000000006_results.down.sql} | 0 ...s.up.sql => 00000000000006_results.up.sql} | 0 .../00000000000101_collector_views.up.sql | 44 +- .../00000000000102_job_views.down.sql | 2 +- .../00000000000102_job_views.up.sql | 26 +- database/queries/client.sql | 89 +- database/queries/collector.sql | 21 +- database/queries/document.sql | 13 +- database/queries/job.sql | 101 --- database/queries/query.sql | 8 +- database/queries/result.sql | 12 +- deployments/compose.local.yaml | 10 +- deployments/prometheus.allmetrics.yaml | 48 +- deployments/prometheus.localservice.yaml | 6 +- devbox.json | 9 +- docs/README.md | 36 +- internal/client/create.go | 8 + internal/client/create_test.go | 19 +- internal/client/get_test.go | 2 +- internal/client/service.go | 13 +- internal/client/service_test.go | 2 +- internal/client/status.go | 35 + internal/client/status_test.go | 78 ++ internal/{job => client}/sync/service.go | 2 +- internal/{job => client}/sync/service_test.go | 2 +- internal/{job => client}/sync/sync.go | 6 +- internal/{job => client}/sync/sync_test.go | 30 +- internal/client/update_test.go | 4 +- internal/{job => }/collector/create.go | 16 +- internal/{job => }/collector/create_test.go | 14 +- .../{job => }/collector/createprivate_test.go | 16 +- internal/{job => }/collector/get.go | 13 +- internal/collector/get_test.go | 82 ++ internal/{job => }/collector/parse.go | 2 +- internal/{job => }/collector/parse_test.go | 8 +- internal/{job => }/collector/service.go | 2 +- internal/{job => }/collector/service_test.go | 2 +- .../{job => }/collector/update/service.go | 6 +- .../collector/update/service_test.go | 2 +- internal/{job => }/collector/update/update.go | 34 +- .../{job => }/collector/update/update_test.go | 30 +- .../collector/update/updateprivate_test.go | 52 +- internal/database/repository/clean_test.go | 12 +- internal/database/repository/client.sql.go | 182 +++++ internal/database/repository/collector.sql.go | 93 +-- .../database/repository/collector_test.go | 66 +- internal/database/repository/document.sql.go | 82 +- internal/database/repository/document_test.go | 40 +- internal/database/repository/job.sql.go | 289 ------- internal/database/repository/job_test.go | 182 ++--- internal/database/repository/models.go | 58 +- internal/database/repository/query.sql.go | 62 +- internal/database/repository/query_test.go | 73 +- internal/database/repository/result.sql.go | 24 +- internal/database/repository/result_test.go | 56 +- internal/database/repository/text_test.go | 6 +- internal/document/clean/clean_test.go | 10 +- internal/document/clean/create_test.go | 10 +- internal/document/get.go | 6 +- internal/document/get_test.go | 10 +- internal/document/init/create.go | 33 +- internal/document/init/create_test.go | 94 +-- internal/document/list.go | 4 +- internal/document/list_test.go | 8 +- internal/document/service.go | 6 +- internal/document/sync/service.go | 4 +- internal/document/sync/sync.go | 2 +- internal/document/sync/sync_test.go | 30 +- internal/job/collector/get_test.go | 115 --- internal/job/create.go | 28 - internal/job/create_test.go | 54 -- internal/job/get.go | 32 - internal/job/get_test.go | 76 -- internal/job/service.go | 32 - internal/job/service_test.go | 24 - internal/job/status.go | 31 - internal/job/status_test.go | 84 -- internal/job/update.go | 96 --- internal/job/update_test.go | 59 -- internal/job/updateprivate_test.go | 164 ---- internal/query/result/processor/parse_test.go | 2 +- internal/query/test/service.go | 2 +- internal/query/test/test_test.go | 2 +- internal/query/versionsync/service.go | 4 +- internal/query/versionsync/sync.go | 10 +- internal/query/versionsync/sync_test.go | 20 +- .../observability/prometheus/prometheus.md | 18 +- .../serviceconfig/queue/clientsync/config.go | 13 + .../queue/clientsync/config_test.go | 21 + .../serviceconfig/queue/jobsync/config.go | 13 - .../queue/jobsync/config_test.go | 21 - internal/test/ecosystem_test.go | 4 +- internal/test/runner.go | 4 +- internal/test/service_test.go | 2 +- pkg/queryService/api.gen.go | 766 ++++-------------- scripts/Taskfile.yml | 73 +- scripts/database.yml | 3 +- scripts/dependencies.yml | 14 - scripts/docker.yml | 7 +- scripts/local-deployments.yml | 9 +- scripts/openapi-scripts.yml | 32 + scripts/openapi.yml | 53 -- scripts/tests.yml | 52 +- serviceAPIs/queryService.yaml | 227 ++---- test/process_test.go | 42 +- test/queryService/accessory_test.go | 2 +- test/queryService/client_test.go | 2 +- ...rvice_test.go => collectorservice_test.go} | 26 +- test/queryService/exportservice_test.go | 2 +- test/queryService/job_test.go | 63 -- test/queryService/queryservice_test.go | 2 +- 148 files changed, 2088 insertions(+), 3828 deletions(-) rename api/{jobSyncRunner => clientSyncRunner}/runner.go (83%) rename api/{jobSyncRunner => clientSyncRunner}/runner_test.go (80%) rename api/queryService/{jobcollector.go => collector.go} (64%) rename api/queryService/{jobcollector_test.go => collector_test.go} (66%) delete mode 100644 api/queryService/job.go delete mode 100644 api/queryService/job_test.go create mode 100644 api/queryService/status.go create mode 100644 api/queryService/status_test.go rename cmd/{jobSyncRunner => clientSyncRunner}/main.go (60%) rename database/migrations/{00000000000005_collectors.down.sql => 00000000000004_collectors.down.sql} (100%) rename database/migrations/{00000000000005_collectors.up.sql => 00000000000004_collectors.up.sql} (61%) delete mode 100644 database/migrations/00000000000004_jobs.down.sql delete mode 100644 database/migrations/00000000000004_jobs.up.sql rename database/migrations/{00000000000006_documents.down.sql => 00000000000005_documents.down.sql} (100%) rename database/migrations/{00000000000006_documents.up.sql => 00000000000005_documents.up.sql} (93%) rename database/migrations/{00000000000007_results.down.sql => 00000000000006_results.down.sql} (100%) rename database/migrations/{00000000000007_results.up.sql => 00000000000006_results.up.sql} (100%) delete mode 100644 database/queries/job.sql create mode 100644 internal/client/status.go create mode 100644 internal/client/status_test.go rename internal/{job => client}/sync/service.go (95%) rename internal/{job => client}/sync/service_test.go (90%) rename internal/{job => client}/sync/sync.go (88%) rename internal/{job => client}/sync/sync_test.go (77%) rename internal/{job => }/collector/create.go (90%) rename internal/{job => }/collector/create_test.go (79%) rename internal/{job => }/collector/createprivate_test.go (90%) rename internal/{job => }/collector/get.go (64%) create mode 100644 internal/collector/get_test.go rename internal/{job => }/collector/parse.go (93%) rename internal/{job => }/collector/parse_test.go (81%) rename internal/{job => }/collector/service.go (96%) rename internal/{job => }/collector/service_test.go (91%) rename internal/{job => }/collector/update/service.go (80%) rename internal/{job => }/collector/update/service_test.go (92%) rename internal/{job => }/collector/update/update.go (90%) rename internal/{job => }/collector/update/update_test.go (63%) rename internal/{job => }/collector/update/updateprivate_test.go (92%) delete mode 100644 internal/database/repository/job.sql.go delete mode 100644 internal/job/collector/get_test.go delete mode 100644 internal/job/create.go delete mode 100644 internal/job/create_test.go delete mode 100644 internal/job/get.go delete mode 100644 internal/job/get_test.go delete mode 100644 internal/job/service.go delete mode 100644 internal/job/service_test.go delete mode 100644 internal/job/status.go delete mode 100644 internal/job/status_test.go delete mode 100644 internal/job/update.go delete mode 100644 internal/job/update_test.go delete mode 100644 internal/job/updateprivate_test.go create mode 100644 internal/serviceconfig/queue/clientsync/config.go create mode 100644 internal/serviceconfig/queue/clientsync/config_test.go delete mode 100644 internal/serviceconfig/queue/jobsync/config.go delete mode 100644 internal/serviceconfig/queue/jobsync/config_test.go delete mode 100644 scripts/dependencies.yml create mode 100644 scripts/openapi-scripts.yml delete mode 100644 scripts/openapi.yml rename test/queryService/{jobcollectorservice_test.go => collectorservice_test.go} (77%) delete mode 100644 test/queryService/job_test.go diff --git a/.golangci.yml b/.golangci.yml index 1b77a54d..7e261cdd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,7 @@ linters: - spancheck - godox - cyclop + - unused linters-settings: errcheck: diff --git a/api/jobSyncRunner/runner.go b/api/clientSyncRunner/runner.go similarity index 83% rename from api/jobSyncRunner/runner.go rename to api/clientSyncRunner/runner.go index 222c55b5..f77a36c2 100644 --- a/api/jobSyncRunner/runner.go +++ b/api/clientSyncRunner/runner.go @@ -1,11 +1,11 @@ -package jobsyncrunner +package clientsyncrunner import ( "context" "encoding/json" "log/slog" - jobsync "queryorchestration/internal/job/sync" + clientsync "queryorchestration/internal/client/sync" "github.com/go-playground/validator/v10" "github.com/google/uuid" @@ -13,10 +13,10 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs/types" ) -const Name = "jobSyncRunner" +const Name = "clientSyncRunner" type Services struct { - JobSync *jobsync.Service + ClientSync *clientsync.Service } type Runner struct { @@ -49,7 +49,7 @@ func (s *Runner) Process(ctx context.Context, req *types.Message) bool { return true } - err = s.svc.JobSync.Sync(ctx, body.ID) + err = s.svc.ClientSync.Sync(ctx, body.ID) if err != nil { slog.Error("unable to process", "error", err) return false diff --git a/api/jobSyncRunner/runner_test.go b/api/clientSyncRunner/runner_test.go similarity index 80% rename from api/jobSyncRunner/runner_test.go rename to api/clientSyncRunner/runner_test.go index cfcf4a0e..d69ff6c3 100644 --- a/api/jobSyncRunner/runner_test.go +++ b/api/clientSyncRunner/runner_test.go @@ -1,4 +1,4 @@ -package jobsyncrunner_test +package clientsyncrunner_test import ( "context" @@ -6,10 +6,10 @@ import ( "fmt" "testing" - jobsyncrunner "queryorchestration/api/jobSyncRunner" + clientsyncrunner "queryorchestration/api/clientSyncRunner" + clientsync "queryorchestration/internal/client/sync" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - jobsync "queryorchestration/internal/job/sync" "queryorchestration/internal/server/runner" "queryorchestration/internal/serviceconfig/queue/documentsync" queuemock "queryorchestration/mocks/queue" @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/require" ) -type JobSyncConfig struct { +type ClientSyncConfig struct { runner.BaseConfig documentsync.DocSyncConfig } @@ -35,22 +35,22 @@ func TestQueryRunner(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) - cfg := &JobSyncConfig{} + cfg := &ClientSyncConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS cfg.DocumentSyncURL = "/i/am/here" - svc := jobsync.New(cfg) + svc := clientsync.New(cfg) - runner := jobsyncrunner.New(validator.New(), &jobsyncrunner.Services{ - JobSync: svc, + runner := clientsyncrunner.New(validator.New(), &clientsyncrunner.Services{ + ClientSync: svc, }) assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { - bod := jobsyncrunner.Body{ + bod := clientsyncrunner.Body{ ID: uuid.New(), } bodyBytes, err := json.Marshal(bod) @@ -63,7 +63,7 @@ func TestQueryRunner(t *testing.T) { docId := uuid.New() total := int64(1) - pool.ExpectQuery("name: ListJobDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(bod.ID), int32(100), int32(0)). + pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(bod.ID), int32(100), int32(0)). WillReturnRows( pgxmock.NewRows([]string{"id", "totalCount"}). AddRow(database.MustToDBUUID(docId), &total), diff --git a/api/docCleanRunner/runner_test.go b/api/docCleanRunner/runner_test.go index 0125c6d7..69f6c2f7 100644 --- a/api/docCleanRunner/runner_test.go +++ b/api/docCleanRunner/runner_test.go @@ -71,9 +71,9 @@ func TestDocCleanRunner(t *testing.T) { } doc := document.Document{ - ID: bod.ID, - JobID: uuid.New(), - Hash: "example_hash", + ID: bod.ID, + ClientID: uuid.New(), + Hash: "example_hash", } inloc := document.Location{ Bucket: "bucket_name", @@ -87,8 +87,8 @@ func TestDocCleanRunner(t *testing.T) { ) pool.ExpectQuery("name: GetDocument :one").WithArgs(dbid). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(dbid, database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(dbid, database.MustToDBUUID(doc.ClientID), doc.Hash), ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). diff --git a/api/docInitRunner/runner.go b/api/docInitRunner/runner.go index dbd85af7..a4593727 100644 --- a/api/docInitRunner/runner.go +++ b/api/docInitRunner/runner.go @@ -97,14 +97,14 @@ func (s Runner) processRecord(ctx context.Context, record S3EventRecord) error { return nil } - jobID, err := s.svc.Document.GetJobIDFromKey(record.S3.Object.Key) + clientID, err := s.svc.Document.GetClientIDFromKey(record.S3.Object.Key) if err != nil { - slog.Error("unable to find job id", "key", record.S3.Object.Key, "error", err) + slog.Error("unable to find client id", "key", record.S3.Object.Key, "error", err) return nil } _, err = s.svc.Document.Create(ctx, &documentinit.Create{ - JobID: jobID, + ClientID: clientID, Location: document.Location{ Bucket: record.S3.Bucket.Name, Key: record.S3.Object.Key, diff --git a/api/docInitRunner/runner_test.go b/api/docInitRunner/runner_test.go index 751824d0..60ef43ed 100644 --- a/api/docInitRunner/runner_test.go +++ b/api/docInitRunner/runner_test.go @@ -6,11 +6,11 @@ import ( "fmt" "testing" + "queryorchestration/internal/client" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" documentinit "queryorchestration/internal/document/init" - "queryorchestration/internal/job" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/queue/documentsync" queuemock "queryorchestration/mocks/queue" @@ -48,16 +48,15 @@ func TestDocInitRunner(t *testing.T) { assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), + j := client.Client{ + ID: uuid.New(), } bucketName := "bucketName" - location := fmt.Sprintf("%s/%s/aaa", j.ClientID.String(), j.ID.String()) + location := fmt.Sprintf("%s/aaa", j.ID.String()) docinfo := document.Document{ - ID: uuid.New(), - JobID: j.ID, - Hash: "example_hash", + ID: uuid.New(), + ClientID: j.ID, + Hash: "example_hash", } doc := S3EventNotification{ Records: []S3EventRecord{ @@ -96,8 +95,8 @@ func TestDocInitRunner(t *testing.T) { pool.ExpectCommit() pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(docinfo.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.JobID), "example"), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.ClientID), "example"), ) mockSQS.EXPECT(). @@ -142,16 +141,15 @@ func TestProcessRecord(t *testing.T) { assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), + j := client.Client{ + ID: uuid.New(), } bucketName := "bucketName" - location := fmt.Sprintf("%s/%s/aaa", j.ClientID.String(), j.ID.String()) + location := fmt.Sprintf("%s/aaa", j.ID.String()) docinfo := document.Document{ - ID: uuid.New(), - JobID: j.ID, - Hash: "example_hash", + ID: uuid.New(), + ClientID: j.ID, + Hash: "example_hash", } record := S3EventRecord{ EventName: S3EventObjectCreatedPut, @@ -180,8 +178,8 @@ func TestProcessRecord(t *testing.T) { pool.ExpectCommit() pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(docinfo.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.JobID), "example"), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.ClientID), "example"), ) mockSQS.EXPECT(). @@ -206,7 +204,7 @@ func TestProcessRecord(t *testing.T) { assert.NoError(t, err) }) t.Run("invalid id", func(t *testing.T) { - location := fmt.Sprintf("%s/cc/aaa", uuid.New()) + location := "cc/aaa" record := S3EventRecord{ EventName: S3EventObjectCreatedPut, S3: S3EventRecordDetails{ diff --git a/api/docSyncRunner/runner_test.go b/api/docSyncRunner/runner_test.go index 4324b8a3..48e3db3d 100644 --- a/api/docSyncRunner/runner_test.go +++ b/api/docSyncRunner/runner_test.go @@ -3,6 +3,7 @@ package docsyncrunner_test import ( "context" "encoding/json" + "fmt" "testing" docsyncrunner "queryorchestration/api/docSyncRunner" @@ -11,15 +12,17 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/document" documentsync "queryorchestration/internal/document/sync" - "queryorchestration/internal/job" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/queue/documentclean" + queuemock "queryorchestration/mocks/queue" + "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/go-playground/validator/v10" "github.com/google/uuid" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -37,24 +40,24 @@ func TestDocInitRunner(t *testing.T) { cfg := &DocSyncConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS runner := docsyncrunner.New(validator.New(), &docsyncrunner.Services{ Document: documentsync.New(cfg, &documentsync.Services{ Document: document.New(cfg), - Job: job.New(cfg, &job.Services{ - Client: client.New(cfg), - }), + Client: client.New(cfg, &client.Services{}), }), }) assert.NotNil(t, runner) t.Run("valid", func(t *testing.T) { - j := job.Job{ + j := client.Client{ ID: uuid.New(), } docinfo := document.Document{ - ID: uuid.New(), - JobID: j.ID, + ID: uuid.New(), + ClientID: j.ID, } doc := docsyncrunner.Body{ ID: docinfo.ID, @@ -68,19 +71,25 @@ func TestDocInitRunner(t *testing.T) { pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(docinfo.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.JobID), "example"), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(docinfo.ID), database.MustToDBUUID(docinfo.ClientID), "example"), ) - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ID)). WillReturnRows( pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), + AddRow(database.MustToDBUUID(j.ID), "client_name", true), ) + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.DocumentCleanURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + assert.True(t, runner.Process(ctx, msg)) }) t.Run("invalid body", func(t *testing.T) { diff --git a/api/queryService/api.gen.go b/api/queryService/api.gen.go index 89f33eda..ba072596 100644 --- a/api/queryService/api.gen.go +++ b/api/queryService/api.gen.go @@ -19,6 +19,13 @@ import ( openapi_types "github.com/oapi-codegen/runtime/types" ) +// Defines values for ClientStatus. +const ( + INSYNC ClientStatus = "IN_SYNC" + NOTSYNCED ClientStatus = "NOT_SYNCED" + NOTSYNCING ClientStatus = "NOT_SYNCING" +) + // Defines values for ExportStatus. const ( Completed ExportStatus = "completed" @@ -38,12 +45,6 @@ const ( RightClosedInterval FieldFilterCondition = "right_closed_interval" ) -// Defines values for JobStatus. -const ( - INSYNC JobStatus = "IN_SYNC" - NOTSYNCED JobStatus = "NOT_SYNCED" -) - // Defines values for QueryType. const ( CONTEXTFULL QueryType = "CONTEXT_FULL" @@ -56,6 +57,18 @@ type ClientCreate struct { Name string `json:"name"` } +// ClientStatus Specifies the status of a client. +type ClientStatus string + +// ClientStatusBody defines model for ClientStatusBody. +type ClientStatusBody struct { + // ClientId The client id + ClientId openapi_types.UUID `json:"client_id"` + + // Status Specifies the status of a client. + Status ClientStatus `json:"status"` +} + // ClientUpdate defines model for ClientUpdate. type ClientUpdate struct { // CanSync If the client is allowing active syncs @@ -65,6 +78,63 @@ type ClientUpdate struct { Name *string `json:"name,omitempty"` } +// Collector Collector model. +type Collector struct { + // ActiveVersion The active version of the collector. + ActiveVersion int32 `json:"active_version"` + + // ClientId The ID of the associated client. + ClientId openapi_types.UUID `json:"client_id"` + + // Fields The fields in the collector. + Fields []CollectorField `json:"fields"` + + // LatestVersion The latest version of the collector. + LatestVersion int32 `json:"latest_version"` + + // MinimumCleanerVersion The minimum version for the document cleaner. + MinimumCleanerVersion int32 `json:"minimum_cleaner_version"` + + // MinimumTextVersion The minimum version for the text parser. + MinimumTextVersion int32 `json:"minimum_text_version"` +} + +// CollectorField The field properties for the collector. +type CollectorField struct { + // Name The output field name. + Name string `json:"name"` + + // QueryId The query id that will populate the result. + QueryId openapi_types.UUID `json:"query_id"` +} + +// CollectorUpdate Payload for updating a Collector. +type CollectorUpdate struct { + // ActiveVersion The active version of the collector. + ActiveVersion *int32 `json:"active_version,omitempty"` + + // Fields The fields in the collector. + Fields *[]CollectorField `json:"fields,omitempty"` + + // MinimumCleanerVersion The minimum version for the document cleaner. + MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"` + + // MinimumTextVersion The minimum version for the text parser. + MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"` +} + +// DocClient defines model for DocClient. +type DocClient struct { + // CanSync If the client is allowing active syncs + CanSync bool `json:"can_sync"` + + // Id The client id + Id openapi_types.UUID `json:"id"` + + // Name The client name + Name string `json:"name"` +} + // Document defines model for Document. type Document struct { // Bucket The bucket containing the document @@ -79,17 +149,17 @@ type Document struct { // ExportDetails Payload for export trigger response. type ExportDetails struct { - // JobId The job id relative to the export. - JobId openapi_types.UUID `json:"job_id"` + // ClientId The client id relative to the export. + ClientId openapi_types.UUID `json:"client_id"` // OutputLocation The location in which the export zip file will be found. OutputLocation *string `json:"output_location,omitempty"` - // Status The possible export job states. + // Status The possible export states. Status ExportStatus `json:"status"` } -// ExportStatus The possible export job states. +// ExportStatus The possible export states. type ExportStatus string // ExportTrigger Payload for triggering an export. @@ -128,100 +198,7 @@ type IdMessage struct { Id openapi_types.UUID `json:"id"` } -// Job defines model for Job. -type Job struct { - // CanSync Specifies whether the job is actively syncing - CanSync bool `json:"can_sync"` - - // ClientId The client id the job belongs to - ClientId openapi_types.UUID `json:"client_id"` - - // Id The job id - Id openapi_types.UUID `json:"id"` -} - -// JobClient defines model for JobClient. -type JobClient struct { - // CanSync If the client is allowing active syncs - CanSync bool `json:"can_sync"` - - // Id The client id - Id openapi_types.UUID `json:"id"` - - // Name The client name - Name string `json:"name"` -} - -// JobCollector JobCollector model. -type JobCollector struct { - // ActiveVersion The active version of the collector. - ActiveVersion int32 `json:"active_version"` - - // Fields The fields in the job collector. - Fields []JobCollectorField `json:"fields"` - - // JobId The ID of the associated job. - JobId openapi_types.UUID `json:"job_id"` - - // LatestVersion The latest version of the collector. - LatestVersion int32 `json:"latest_version"` - - // MinimumCleanerVersion The minimum version for the document cleaner. - MinimumCleanerVersion int32 `json:"minimum_cleaner_version"` - - // MinimumTextVersion The minimum version for the text parser. - MinimumTextVersion int32 `json:"minimum_text_version"` -} - -// JobCollectorField The field properties for the job collector. -type JobCollectorField struct { - // Name The output field name. - Name string `json:"name"` - - // QueryId The query id that will populate the result. - QueryId openapi_types.UUID `json:"query_id"` -} - -// JobCollectorUpdate Payload for updating a JobCollector. -type JobCollectorUpdate struct { - // ActiveVersion The active version of the collector. - ActiveVersion *int32 `json:"active_version,omitempty"` - - // Fields The fields in the job collector. - Fields *[]JobCollectorField `json:"fields,omitempty"` - - // MinimumCleanerVersion The minimum version for the document cleaner. - MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"` - - // MinimumTextVersion The minimum version for the text parser. - MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"` -} - -// JobCreate defines model for JobCreate. -type JobCreate struct { - // ClientId The client id the job belongs to - ClientId openapi_types.UUID `json:"client_id"` -} - -// JobStatus Specifies the status of a job. -type JobStatus string - -// JobStatusBody defines model for JobStatusBody. -type JobStatusBody struct { - // JobId The job id - JobId openapi_types.UUID `json:"job_id"` - - // Status Specifies the status of a job. - Status JobStatus `json:"status"` -} - -// JobUpdate defines model for JobUpdate. -type JobUpdate struct { - // CanSync Specifies whether the job is actively syncing - CanSync *bool `json:"can_sync,omitempty"` -} - -// ListDocuments The documents in the job. +// ListDocuments The documents in the client. type ListDocuments = []Document // ListQueries defines model for ListQueries. @@ -299,14 +276,8 @@ type CreateClientJSONRequestBody = ClientCreate // UpdateClientJSONRequestBody defines body for UpdateClient for application/json ContentType. type UpdateClientJSONRequestBody = ClientUpdate -// CreateJobJSONRequestBody defines body for CreateJob for application/json ContentType. -type CreateJobJSONRequestBody = JobCreate - -// UpdateJobJSONRequestBody defines body for UpdateJob for application/json ContentType. -type UpdateJobJSONRequestBody = JobUpdate - -// UpdateJobCollectorByJobIdJSONRequestBody defines body for UpdateJobCollectorByJobId for application/json ContentType. -type UpdateJobCollectorByJobIdJSONRequestBody = JobCollectorUpdate +// UpdateCollectorByClientIdJSONRequestBody defines body for UpdateCollectorByClientId for application/json ContentType. +type UpdateCollectorByClientIdJSONRequestBody = CollectorUpdate // TriggerExportJSONRequestBody defines body for TriggerExport for application/json ContentType. type TriggerExportJSONRequestBody = ExportTrigger @@ -331,33 +302,24 @@ type ServerInterface interface { // Update a client // (PATCH /client/{id}) UpdateClient(ctx echo.Context, id openapi_types.UUID) error - // Create a new job - // (POST /job) - CreateJob(ctx echo.Context) error - // Get a job by ID - // (GET /job/{id}) - GetJob(ctx echo.Context, id openapi_types.UUID) error - // Update a job - // (PATCH /job/{id}) - UpdateJob(ctx echo.Context, id openapi_types.UUID) error - // Get a job collector by ID - // (GET /job/{id}/collector) - GetJobCollectorByJobId(ctx echo.Context, id openapi_types.UUID) error - // Update a job collector - // (PATCH /job/{id}/collector) - UpdateJobCollectorByJobId(ctx echo.Context, id openapi_types.UUID) error - // List the documents for a job - // (GET /job/{id}/documents) - ListDocumentsByJobId(ctx echo.Context, id openapi_types.UUID) error + // Get a collector by client ID + // (GET /client/{id}/collector) + GetCollectorByClientId(ctx echo.Context, id openapi_types.UUID) error + // Update a collector + // (PATCH /client/{id}/collector) + UpdateCollectorByClientId(ctx echo.Context, id openapi_types.UUID) error + // List the documents for a client + // (GET /client/{id}/documents) + ListDocumentsByClientId(ctx echo.Context, id openapi_types.UUID) error // Check export state. - // (GET /job/{id}/export) + // (GET /client/{id}/export) ExportState(ctx echo.Context, id openapi_types.UUID) error // Trigger an export - // (POST /job/{id}/export) + // (POST /client/{id}/export) TriggerExport(ctx echo.Context, id openapi_types.UUID) error - // Get job status by job ID - // (GET /job/{id}/status) - GetJobStatusByJobId(ctx echo.Context, id openapi_types.UUID) error + // Get client sync status + // (GET /client/{id}/status) + GetStatusByClientId(ctx echo.Context, id openapi_types.UUID) error // List queries // (GET /query) ListQueries(ctx echo.Context) error @@ -421,17 +383,8 @@ func (w *ServerInterfaceWrapper) UpdateClient(ctx echo.Context) error { return err } -// CreateJob converts echo context to params. -func (w *ServerInterfaceWrapper) CreateJob(ctx echo.Context) error { - var err error - - // Invoke the callback with all the unmarshaled arguments - err = w.Handler.CreateJob(ctx) - return err -} - -// GetJob converts echo context to params. -func (w *ServerInterfaceWrapper) GetJob(ctx echo.Context) error { +// GetCollectorByClientId converts echo context to params. +func (w *ServerInterfaceWrapper) GetCollectorByClientId(ctx echo.Context) error { var err error // ------------- Path parameter "id" ------------- var id openapi_types.UUID @@ -442,12 +395,12 @@ func (w *ServerInterfaceWrapper) GetJob(ctx echo.Context) error { } // Invoke the callback with all the unmarshaled arguments - err = w.Handler.GetJob(ctx, id) + err = w.Handler.GetCollectorByClientId(ctx, id) return err } -// UpdateJob converts echo context to params. -func (w *ServerInterfaceWrapper) UpdateJob(ctx echo.Context) error { +// UpdateCollectorByClientId converts echo context to params. +func (w *ServerInterfaceWrapper) UpdateCollectorByClientId(ctx echo.Context) error { var err error // ------------- Path parameter "id" ------------- var id openapi_types.UUID @@ -458,12 +411,12 @@ func (w *ServerInterfaceWrapper) UpdateJob(ctx echo.Context) error { } // Invoke the callback with all the unmarshaled arguments - err = w.Handler.UpdateJob(ctx, id) + err = w.Handler.UpdateCollectorByClientId(ctx, id) return err } -// GetJobCollectorByJobId converts echo context to params. -func (w *ServerInterfaceWrapper) GetJobCollectorByJobId(ctx echo.Context) error { +// ListDocumentsByClientId converts echo context to params. +func (w *ServerInterfaceWrapper) ListDocumentsByClientId(ctx echo.Context) error { var err error // ------------- Path parameter "id" ------------- var id openapi_types.UUID @@ -474,39 +427,7 @@ func (w *ServerInterfaceWrapper) GetJobCollectorByJobId(ctx echo.Context) error } // Invoke the callback with all the unmarshaled arguments - err = w.Handler.GetJobCollectorByJobId(ctx, id) - return err -} - -// UpdateJobCollectorByJobId converts echo context to params. -func (w *ServerInterfaceWrapper) UpdateJobCollectorByJobId(ctx echo.Context) error { - var err error - // ------------- Path parameter "id" ------------- - var id openapi_types.UUID - - err = runtime.BindStyledParameterWithOptions("simple", "id", ctx.Param("id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - // Invoke the callback with all the unmarshaled arguments - err = w.Handler.UpdateJobCollectorByJobId(ctx, id) - return err -} - -// ListDocumentsByJobId converts echo context to params. -func (w *ServerInterfaceWrapper) ListDocumentsByJobId(ctx echo.Context) error { - var err error - // ------------- Path parameter "id" ------------- - var id openapi_types.UUID - - err = runtime.BindStyledParameterWithOptions("simple", "id", ctx.Param("id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Invalid format for parameter id: %s", err)) - } - - // Invoke the callback with all the unmarshaled arguments - err = w.Handler.ListDocumentsByJobId(ctx, id) + err = w.Handler.ListDocumentsByClientId(ctx, id) return err } @@ -542,8 +463,8 @@ func (w *ServerInterfaceWrapper) TriggerExport(ctx echo.Context) error { return err } -// GetJobStatusByJobId converts echo context to params. -func (w *ServerInterfaceWrapper) GetJobStatusByJobId(ctx echo.Context) error { +// GetStatusByClientId converts echo context to params. +func (w *ServerInterfaceWrapper) GetStatusByClientId(ctx echo.Context) error { var err error // ------------- Path parameter "id" ------------- var id openapi_types.UUID @@ -554,7 +475,7 @@ func (w *ServerInterfaceWrapper) GetJobStatusByJobId(ctx echo.Context) error { } // Invoke the callback with all the unmarshaled arguments - err = w.Handler.GetJobStatusByJobId(ctx, id) + err = w.Handler.GetStatusByClientId(ctx, id) return err } @@ -655,15 +576,12 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL router.POST(baseURL+"/client", wrapper.CreateClient) router.GET(baseURL+"/client/:id", wrapper.GetClient) router.PATCH(baseURL+"/client/:id", wrapper.UpdateClient) - router.POST(baseURL+"/job", wrapper.CreateJob) - router.GET(baseURL+"/job/:id", wrapper.GetJob) - router.PATCH(baseURL+"/job/:id", wrapper.UpdateJob) - router.GET(baseURL+"/job/:id/collector", wrapper.GetJobCollectorByJobId) - router.PATCH(baseURL+"/job/:id/collector", wrapper.UpdateJobCollectorByJobId) - router.GET(baseURL+"/job/:id/documents", wrapper.ListDocumentsByJobId) - router.GET(baseURL+"/job/:id/export", wrapper.ExportState) - router.POST(baseURL+"/job/:id/export", wrapper.TriggerExport) - router.GET(baseURL+"/job/:id/status", wrapper.GetJobStatusByJobId) + router.GET(baseURL+"/client/:id/collector", wrapper.GetCollectorByClientId) + router.PATCH(baseURL+"/client/:id/collector", wrapper.UpdateCollectorByClientId) + router.GET(baseURL+"/client/:id/documents", wrapper.ListDocumentsByClientId) + router.GET(baseURL+"/client/:id/export", wrapper.ExportState) + router.POST(baseURL+"/client/:id/export", wrapper.TriggerExport) + router.GET(baseURL+"/client/:id/status", wrapper.GetStatusByClientId) router.GET(baseURL+"/query", wrapper.ListQueries) router.POST(baseURL+"/query", wrapper.CreateQuery) router.GET(baseURL+"/query/:id", wrapper.GetQuery) @@ -675,50 +593,47 @@ func RegisterHandlersWithBaseURL(router EchoRouter, si ServerInterface, baseURL // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+Rb3W/bOBL/VwjdPRp2ut0nv3WTdOGg1/Sa9LCHRWFQ1NhmKpMKSTn1Bv7fF/zQBy3K", - "kh2rm8U+tYnI+Z4fhzPMc0T4OuMMmJLR9DmSZAVrbP57mVJg6lIAVqB/zgTPQCgK5ivDa/PbBCQRNFOU", - "s2ga3a8AEbMPmQWjSG0ziKaRVIKyZbTbjSIBjzkVkETT3y2Vr+UqHj8AUdFu5Jh/yZIgc4LZXG4ZaQow", - "WyBVyUAlwmnKnyhbIkwU3QDS22QlV8x5Cphplqdr1JD+ipN8DUw1JY9z8g1UmI39hghnClOmZdaqJAWt", - "BudRRJMwpWIPokk0ihZcrLGKplGem58bZL7BNkwnw2qFFO+QY8+nhofT09IOOfj6e8aFugKFaSqb3D/h", - "bcpxghZcIDBLkRJ0uQSBBMiMMwnjaLRn3Acez9tM8sBjRBMkIMUmDpxWlva4j5V4rrJczVNOsKUbYlN8", - "RZShpxUlqxoX9AfN0IKmgJ5omqIY0ILnLNHM4TteZ6nh93Y6mTxb8+0mzzb0aLKbPD/w2Pyr6Bqkwuts", - "N3+2hGmyG/9Bs5DQUmGVG+P8W8Aimkb/mlQJP3HZPrHOuLNr9/3prFrSavfmXcksEEpcShqnpS20PzRB", - "kEZ/lq81Ly1bCgo0O8rmmeBLAVLn6wLTFJIa80pHy/zehsfhUHIxZPCA1Zzvx9GCQprMFzRVIALqvDcf", - "jGMl4RmgGEtIEGfIbEQ2UNAGp7nVjipYd/rgvd5rSUcVpmAh8NakOluC1AKcIle5GWVY4DXo/U21gSXz", - "Am8DkY2lQvoz4ouK4Lgl6IRqJUUlWlDRk1gTXUN4W7ddi1mMyxHhab5mDc0JZwktkrqnky7LPbuRi5j2", - "E0R/0apq39ggoQw95gdsaKMnTM1+Q7mERZ4WSGbDwgu3BlE/qPbyvKbDqGaRUpSvhw1/Wbfhgfy36i9K", - "p5ScPCBIQcq5WmHNf2lKEFH8SFIuIZlTpkBscBqNIp4Bq/+cwkLNm8sEXa5Cv6eMpHkCBobt/0IoM0v+", - "A1LiZaAcCZ05Xxh9zAHRRMP3goKwRxlTVG17HDfNMzVk/hseH1Mc3WVAtCwSPa1ArRxWmJNRuvoo3ZoK", - "ScsQqpHscdR6yBZlV1ISjiHlbCmR4n2O2MOH9ylWq4s8qkzTYkxbdf6gerPTiH0sdp4y3NAuMr/LRjxN", - "gSgeQNr6V7TmCaTNQ8aaZb4BIVuxwpnOrSlwkxSkvfShTL39qVJQp/XSnqAGaFoQ1H7TIFwEqke914Fd", - "19bgYOjYPlSSzq4K1bCUnFCsINGy9KpGU106qcN2tGteasc1ZXSdr+dEBy6Iwyzd4pKnKbrqNxJH5Uje", - "Cr6r0xjrnbrqkT15thW/LprazdEibMNTo/0U6MozG1vtYYyqBCu1bkR038u7q10tYb0sWJw85iC2rXFt", - "vtpDACt718l4lms7GOEEyDxVxx+CDqBK5l2Gq7oH7ReCXK+xtWF96z8et/5xSR+MpJbe16BV0F7QV7xa", - "wr3t0l0Ve+ZCaFbpkMTFEVOU2rOP87v/f7yMRtHH23vz3+urYAVcMvuFJ9umVbq7L33OtX4Ni0rvU7oV", - "Nzw+vrP40uI5FGMfqFRFr1AebuXVk753qpd9yECGa97/zUE4xX0zPFYffJH0Lh1FbkFvSTSnbecVtGAb", - "cpql0JD0dFw250hPSCKcLeiySf3S/D4XtudXwE9JudcFJ3xTDEt4/mrwGCsUrpp3hkex0tUCsys/Ujr1", - "2Q9W+3OPELvXC4PXG0OiUXw1TNcaeq2HwVGhweDpQHj8XQ1sdrda7h6k+gym29U0X4FvwaOjuiGVpYTi", - "yIQxXmLKZL/Ova0YW5Pjf4GM0HxyCdZvIHWJeMr1oa7dvhwd9rJDjqbBTDeuqcRnU1Tv6QDWQIcrDEuw", - "XRoXGYeqC72zAShFfXFzd/txfv3b/ed3l/e3n6NRdHn78f76t/v5+y8fPgQLDcO37YTuAny7L9kD/Rfi", - "fEGUHIn33QldUE4HSexA+9zMEha8Kcm7T7MyzXz9jD/QrSArkMrpLkFsKLEVgKLKzK5C6959mkWjqPRW", - "9GZ8Mb4w47QMGM5oNI3eji/Gb/WFC6uVUXJCqj4cl4FpqcViibDBU1dvP1Flp22Z4BuaQIISO10c2xax", - "FWiWlPtdt896CaQqylrCmXLscZal1E70Jg/ShpuFwy6w9KbnOz/nlMjB/MLmuNH5p4s3Z+NdtaoN4z3b", - "WWsRI1mCZE4ISLnI03Q71n75+eIiAMRsg1MzPjWWQjFP9Gpdr+frNdaFmTOq5xMdG3gpNQpYtnc2aqKv", - "eqtz8+SZJjvNchkajH8GJShsjLOlBR1SeDzeIqokml01PfwrqJp7PTtfnM3OVce43c5lDPY1bX1C5xv4", - "V1AI13SfXR2wr06nglA0/f1w/9HRVBwJZ24DOnqhTsqiOTy1lZQfyKOasbqutl9NkpNVGwxKO5Wl5rz1", - "8lrHVGs6280/IJ3dmdQrnQOudiGRO8g/X+pZucrg6Ei7Bzc16gGt+np7NK7e8HggL1Q9mdeEqDc8HhpO", - "H4xJC6fe8Ljh0RNQ1HSmDkJo4cnh8LPNnmeHTaesh5meHXsApqZhLfVXQ2OZlz1wcdB0PAYRfw5ODIeE", - "w55pMyH1meZxCVRu7U6lsuv+y/aGx7Nk6NKkVKoNs0rRe2dbyNpVfnm22M+0UpwTUq68ifjTjleQhJXK", - "R6RjMBCGOS33ZmEvydNK1WEztmLUGUBeEif1RnpHEutYKm7cVYc93pY9/VAWe936H5DD/nSg7aAspdf6", - "vMwLprmovKGDTrwAjpZivSSZSy4DJbMXHfatZ2toXK6AfLNxQXIhdJluXqeaiVn9pagfEtW7VxgyEvzH", - "0oFIsAvQCksUAzBUvqM9DdKNNYqXusYO45r/ndKnXTUrSw4D38FLzYxRRQ1+115jZ4Jr9Go61b0ktmoO", - "hM3+m+XXdJu59l7bnxXlnbpVQh0IKi95q6FwD1wvXpbnsijJ2gDdlmVupv1DarLa/LwFz53kZ77/+Dax", - "9vgblmc6KB6LOXBnnZ76s+r6M33zUJ4IqkBQHD7pi9n4wAd8wSYQDu/2NThHNJgz/rHUrYgA07r3XN+j", - "O2RHFEf3h+wkfxhcrY9qXxOq2tHIwF2iR2fYFqeW2XNCr8i6+vAVt/LrQPniHpG0WffsPaNSaQ8sm6nS", - "v/opJ4V/9d21lro97qyDZ+wLG+vW/wNeS4/KrIkqXjq8msAIgvn1dyC5RXPzpELkzD4QbMN1/4/H9opm", - "kGrwMKm/JOkfK+fm715mBHDo3pjRPW9+UaVsnrh0hp3eA2ITDrBPgic5KYf1IKJRlIs0mkYrpTI5nUxw", - "RsfuL0/HhK8nmzeRjhbHbZ/ebeFvaf+KFhKkuJs4ySpa/YnTbtSPzAOPazRqndojCFTlp0+qUdoeQ7Pq", - "KtVJNvoffUlWlZcj5rm0LxV7e6pR8W9Pu6+7PwMAAP//WyCUh9g/AAA=", + "H4sIAAAAAAAC/+RbX3PbuBH/Khi2jxrJudyT3nK2k1EntVNH6Vzn6tFA4ErCBQJoAJStevTdO/hDkBRB", + "ipYlJ52+mSKwf3+7WOzSzwkR60xw4Fol4+dEkRWssf3zklHg+lIC1mCeMykykJqCfcvx2v6agiKSZpoK", + "noyT6QoQsfuQXTBI9DaDZJwoLSlfJrvdIJHwkFMJaTL+w1G5D6vE/E8gOtkNPPOvGutcNdl8zYDQBQWF", + "9AqQsquQWCDsmQ+TQQI8XxsWk5vZ13/dXCaD5OZ2av+8vqo8TG4+VQQoxKwL8JtIt00LOFYzmnaagabJ", + "IFkIucY6GSd5bp8b3FRQ9K8SFsk4+cuo9MvIO2VUM8q+KUtxArl2w37L0qhXCeYzteWkqdJkYW1daKUQ", + "Zkw8Ur5EmGi6AWS2qVK3uRAMMDcsj4dKU3rBGBAtZJNeeIXWIgVmIFDXzck524BUdkdMIK+LX2MgZZUu", + "SA+rvqRcv/+lFJtyDUuQRsoDyJhcFYSxUoJQrCGtAPcgWBYUWKritN07RHlTcKphfRhixY6PhlBS+gBL", + "ibfmmWENSneb0a15rRnXlNN1vp4RAySQ3Sz94sBzIaRlmgqSrw3IPJUX8tbwpI9jbHaiDEvVk2dHOHuH", + "t1ukRd6Gswb7QXDfFWMOAu0wQ2V8Ba1rPu57YohcZ7n2RM2yYQz2DznIbWtQ2beIpkivsEaPlDGUiSw3", + "BrCCSVA56xNfsROqwrzTYmVarQv4BW+ZwKm1Um7W2LyJLtuN9XbJ6sdmk/+7GG+A50oQdya/0Wl8knrl", + "NNWfpe1XBl3v4zayDm6aaJ6T76Djorh3iAiuMeXGOFWwxNRqs00AWD/rfIdtnE6G9QppcUCOmJW8no52", + "zETXT5mQ+go0pkx1ZyCwS5GWdLkEaTJjJriCZhLqW+EiCQxb2HndHIdexYzL/TMmCNbt9YR/azLR44qS", + "VYUL+g/N0IIycEl/Dmghcp7aK8ATXmfM8ns/Ho2enRFnBnG70TM8aZAcs1lQczfabrfb9TpNzVtD3Pw4", + "JGrzb3581e4887qqvUYjDi2hFJ2zYBVDDFT1ImREY6DBsKJ8lkmxlKBMklhgyiCNXoMc46mDSjesPJ5s", + "EuIVCNQxZY+U2YIyDTKiykf7wl3riMgAzbGCFJmca6sDXypsMMuddr3OIXv8ONKxQ4jyJSgjwDFyhc3m", + "IMBrMPubagNPZ/HawNXLSiPz2hzigeCwBXNSt5KiCi2o7EmseRrFzqeq7VrM4qoZIli+5s0kInhKi9Du", + "6aTLsKcoUGbtJ455UxQ/DiSUm3Kw3YYOPXFq7h3KFSxyVuQzB4sa3BpE66DaC/OKDoOKRYIo992Gv6za", + "sCP2nfqL4JTAqZYIGCg10yts+C9te0cWj4QJBenMFCxyg1kySEQGvPrMYKFnzWWSLlex3yknLE/BJmP3", + "VyzLTNK/g1J4GelIxA6fb5w+5IBoClzTBQXpjjWuqd6+vMJvKeo/U6WLukN1lwVlhRwu8b3SUihrIjnJ", + "sP9HDtKboW6Uh/JFXSqzy8SCX9BbEsNpexDFBduYuRyFhqTH32Psjatvw0XwBV3GekLm91y64qGo2gPl", + "XvVfHGxxCduKnON7Ji+xQuGq2UF4FCv9rXlyVUfKQX32weqee0BsahZGC1xLotGfaJiuFXptfeqXQYPD", + "Ywc8/lcNbHe3Wm4KSt+BPTCb5itSXPQSUDYywwXJHJgGxniJKVf9rgCut9IaHP+MRIThkytwfjMnPV8e", + "02SrarcvxwF7uTtT02D2QG8qcWfbT3s6gDNQ9/HkCLZL45HRNSAxOxsJpagG/vb19mZ2/fv07sPl9PYu", + "GSSXtzfT69+ns4/fPn+OntaWb9sE4VDCd/vSvaT/yjxfECUvzPeHA7qgzM4S2JEK3F5HFqIpyYcvkxBm", + "df2sP9CtJCtQ2uuuQG4ocRWAptpegmPrPnyZmCq08FbybngxvLD38gw4zmgyTt4PL4bvTVmP9coqOSJl", + "v0qoSPPF5WKFsM2nvk3wSLW7tmdSbGgKKUpds2Loqkwn0CQN+31XzHkJlC6mcERw7dnjLGPUtQZGfyoH", + "N5cO+43S/KGxq8ecljnYH1yMW51/uXh3Mt5ltWsZ79nOWYtYyVKkckJAqUXO2HZo/PLrxUUkEfMNZrYP", + "Yy2F5iI1q811MV+vsSnMvFFrPjHYwEtlsoAfLTrUJPdmq3fz6JmmO8NyGeuz3YGWFDbW2colHVJ4fL5F", + "VCs0uWp6+BPointrdr44mZ3Lzmq7nQMG+5q2esmvG/gT6DCANrpPrjrsa8KpIJSM/+geE3qaWiDpzW2T", + "jllogrJooo5dJVUH8qBirEP3onsb5GTVlgaVa+xQe97W4tpgqjWc3eY3CGd/JvUK54irPSRyn/JPF3pO", + "rgCO/mE3ItV59wsCMMzCfQx6X7WFYrH8t60TaJKeMy7LIX4sLoPovUMzZnIfjFU7BBtU7V8seFFkBkrx", + "secPD8ygde/YbAHAGQJ1b1TaK1Z/7fra45zhGoDaiZj9oE2rXaMDQWvAU9SWZTtpvq3m/VjU1rpTbxO2", + "9YZYx5Ea1DCKvc4V9j6ta602E3KRTBoke2UkB0ZniuR9sLg5SStSLldAvjuYkFxKI6yd7Niv3apTljo8", + "ynkRnBMS9aFjBBJuAVphheYAHIUZ1HFZ3VqjNuEaViDglT6uxioteZ70Hb0oTTjV1Obvyjwzk8IksqZT", + "/RTOqXmm/Fyf9/1MF6Pr2tT6pAnfq1sGVAeo9uO3nAH3yPRqy0nxuWqvysx/f/o2Zdn+F6/tKd6rcOLL", + "kzdFxUivu0CF7xPOmMofirHHweKc1Ucz1cG2HS0TSTVIiuNnfTEKOvP5XrCJuP7DvgancLs93x+CboWz", + "baeq5usefSbXkXtxm8kNrs6TTauTiZ8pl7pO4Jl7TA/esC1ODdFzRIPJubq7v1T69Uzx4membdY9eWcp", + "KF27wTZDpX/NExrjP/rGWgndHrfVs0fsK/tIzv9nvJe+KLJGuhjs/TTAiCbz6ycgucvmdoIoc+7+pact", + "r9c/t9orlUHps8OkOjjtj5VT8/eDyEgemloz+u/eX1Uf24nuQdiZPSA3cYB9kSLNSZhNgUwGSS5ZMk5W", + "WmdqPBrhjA79F5tDItajzbvEoMVz26d3W/hbua9PIUVa+JJPlWitl427QU8yRZOnSmm/8dOXWNkOCrQa", + "zYq+tMpSyVOq+aAvFXfJqVCpX3J297v/BgAA//8NIKB+FzgAAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/api/queryService/client.go b/api/queryService/client.go index e7bd1357..d79c20e1 100644 --- a/api/queryService/client.go +++ b/api/queryService/client.go @@ -32,7 +32,7 @@ func (s *Controllers) GetClient(ctx echo.Context, id types.UUID) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to get client: %s", err)) } - return ctx.JSON(http.StatusOK, JobClient{ + return ctx.JSON(http.StatusOK, DocClient{ Id: client.ID, Name: client.Name, CanSync: client.CanSync, diff --git a/api/queryService/client_test.go b/api/queryService/client_test.go index ff9b222b..09217d57 100644 --- a/api/queryService/client_test.go +++ b/api/queryService/client_test.go @@ -10,12 +10,16 @@ import ( queryservice "queryorchestration/api/queryService" "queryorchestration/internal/client" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + cleanversion "queryorchestration/internal/document/clean/version" + textversion "queryorchestration/internal/document/text/version" "queryorchestration/internal/serviceconfig" "github.com/go-playground/validator/v10" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/labstack/echo/v4" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" @@ -31,7 +35,12 @@ func TestCreateClient(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(cfg), + Client: client.New(cfg, &client.Services{ + Collector: collector.New(cfg, &collector.Services{ + TextVersion: textversion.New(cfg), + CleanVersion: cleanversion.New(cfg), + }), + }), }) body := queryservice.ClientCreate{ @@ -52,6 +61,14 @@ func TestCreateClient(t *testing.T) { pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(id)), ) + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( + pgxmock.NewRows([]string{"version"}). + AddRow(int32(1)), + ) + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() err = cons.CreateClient(ctx) assert.NoError(t, err) @@ -68,7 +85,12 @@ func TestGetClient(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(cfg), + Client: client.New(cfg, &client.Services{ + Collector: collector.New(cfg, &collector.Services{ + TextVersion: textversion.New(cfg), + CleanVersion: cleanversion.New(cfg), + }), + }), }) e := echo.New() @@ -89,10 +111,10 @@ func TestGetClient(t *testing.T) { assert.NoError(t, err) assert.Equal(t, http.StatusOK, rec.Code) - var res queryservice.JobClient + var res queryservice.DocClient err = json.Unmarshal(rec.Body.Bytes(), &res) assert.NoError(t, err) - assert.EqualExportedValues(t, queryservice.JobClient{ + assert.EqualExportedValues(t, queryservice.DocClient{ Id: id, Name: "client_name", CanSync: true, @@ -108,7 +130,12 @@ func TestUpdateClient(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Client: client.New(cfg), + Client: client.New(cfg, &client.Services{ + Collector: collector.New(cfg, &collector.Services{ + TextVersion: textversion.New(cfg), + CleanVersion: cleanversion.New(cfg), + }), + }), }) cs := false diff --git a/api/queryService/jobcollector.go b/api/queryService/collector.go similarity index 64% rename from api/queryService/jobcollector.go rename to api/queryService/collector.go index 7201e3e9..b16f8112 100644 --- a/api/queryService/jobcollector.go +++ b/api/queryService/collector.go @@ -4,31 +4,31 @@ import ( "fmt" "net/http" - collectorupdate "queryorchestration/internal/job/collector/update" + collectorupdate "queryorchestration/internal/collector/update" "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/oapi-codegen/runtime/types" ) -func (s *Controllers) GetJobCollectorByJobId(ctx echo.Context, jobId types.UUID) error { - coll, err := s.svc.Collector.GetByJobID(ctx.Request().Context(), jobId) +func (s *Controllers) GetCollectorByClientId(ctx echo.Context, clientId types.UUID) error { + coll, err := s.svc.Collector.GetByClientID(ctx.Request().Context(), clientId) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to get collector: %s", err)) } - fields := make([]JobCollectorField, len(coll.Fields)) + fields := make([]CollectorField, len(coll.Fields)) index := 0 for name, id := range coll.Fields { - fields[index] = JobCollectorField{ + fields[index] = CollectorField{ Name: name, QueryId: id, } index++ } - return ctx.JSON(http.StatusOK, JobCollector{ - JobId: coll.JobID, + return ctx.JSON(http.StatusOK, Collector{ + ClientId: coll.ClientID, MinimumCleanerVersion: coll.MinCleanVersion, MinimumTextVersion: coll.MinTextVersion, ActiveVersion: coll.ActiveVersion, @@ -37,8 +37,8 @@ func (s *Controllers) GetJobCollectorByJobId(ctx echo.Context, jobId types.UUID) }) } -func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UUID) error { - req := JobCollectorUpdate{} +func (s *Controllers) UpdateCollectorByClientId(ctx echo.Context, clientId types.UUID) error { + req := CollectorUpdate{} if err := ctx.Bind(&req); err != nil { return echo.NewHTTPError(http.StatusBadRequest, err) } @@ -52,8 +52,8 @@ func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UU fields = &fs } - err := s.svc.CollectorUpdate.UpdateByJobId(ctx.Request().Context(), &collectorupdate.UpdateParams{ - JobID: jobId, + err := s.svc.CollectorUpdate.UpdateByClientId(ctx.Request().Context(), &collectorupdate.UpdateParams{ + ClientID: clientId, ActiveVersion: req.ActiveVersion, MinCleanVersion: req.MinimumCleanerVersion, MinTextVersion: req.MinimumTextVersion, diff --git a/api/queryService/jobcollector_test.go b/api/queryService/collector_test.go similarity index 66% rename from api/queryService/jobcollector_test.go rename to api/queryService/collector_test.go index 3d0bcdd5..78afd875 100644 --- a/api/queryService/jobcollector_test.go +++ b/api/queryService/collector_test.go @@ -9,14 +9,14 @@ import ( "testing" queryservice "queryorchestration/api/queryService" + "queryorchestration/internal/collector" + collectorupdate "queryorchestration/internal/collector/update" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job/collector" - collectorupdate "queryorchestration/internal/job/collector/update" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -31,12 +31,12 @@ import ( "github.com/stretchr/testify/require" ) -func TestUpdateJobCollector(t *testing.T) { +func TestUpdateCollector(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) cfg := &struct { serviceconfig.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig }{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) @@ -44,17 +44,17 @@ func TestUpdateJobCollector(t *testing.T) { cfg.QueueClient = mockSQS current := collector.Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), ActiveVersion: 1, LatestVersion: 4, } av := int32(2) cv := int32(1) - body := queryservice.JobCollectorUpdate{ + body := queryservice.CollectorUpdate{ ActiveVersion: &av, MinimumCleanerVersion: &cv, - Fields: &[]queryservice.JobCollectorField{ + Fields: &[]queryservice.CollectorField{ { Name: "a", QueryId: uuid.New(), @@ -79,12 +79,12 @@ func TestUpdateJobCollector(t *testing.T) { }), }) - ctx.Set("id", current.JobID) + ctx.Set("id", current.ClientID) - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)). + pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(current.ClientID)). WillReturnRows( - pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(current.ClientID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]pgtype.UUID{database.MustToDBUUID((*body.Fields)[0].QueryId)}). WillReturnRows( @@ -92,15 +92,15 @@ func TestUpdateJobCollector(t *testing.T) { AddRow(true), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(5)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(5), *body.MinimumCleanerVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(5), *body.MinimumCleanerVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -108,19 +108,19 @@ func TestUpdateJobCollector(t *testing.T) { SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", current.JobID.String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", current.ClientID.String()) }), mock.Anything, ). Return(&sqs.SendMessageOutput{}, nil) - err = cons.UpdateJobCollectorByJobId(ctx, current.JobID) + err = cons.UpdateCollectorByClientId(ctx, current.ClientID) assert.NoError(t, err) assert.Equal(t, http.StatusOK, rec.Code) assert.Empty(t, rec.Body.String()) } -func TestGetJobCollectorByJobId(t *testing.T) { +func TestGetCollectorByclientId(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) cfg := &serviceconfig.BaseConfig{} @@ -141,30 +141,30 @@ func TestGetJobCollectorByJobId(t *testing.T) { }) coll := collector.Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), MinCleanVersion: int32(1), MinTextVersion: int32(2), } - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)). + pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(coll.ClientID)). WillReturnRows( - pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.ClientID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), ) - err = cons.GetJobCollectorByJobId(ctx, coll.JobID) + err = cons.GetCollectorByClientId(ctx, coll.ClientID) assert.NoError(t, err) assert.Equal(t, http.StatusOK, rec.Code) - var res queryservice.JobCollector + var res queryservice.Collector err = json.Unmarshal(rec.Body.Bytes(), &res) assert.NoError(t, err) - assert.EqualExportedValues(t, queryservice.JobCollector{ - JobId: coll.JobID, + assert.EqualExportedValues(t, queryservice.Collector{ + ClientId: coll.ClientID, MinimumCleanerVersion: coll.MinCleanVersion, MinimumTextVersion: coll.MinTextVersion, ActiveVersion: int32(1), LatestVersion: int32(2), - Fields: []queryservice.JobCollectorField{}, + Fields: []queryservice.CollectorField{}, }, res) } diff --git a/api/queryService/controllers.go b/api/queryService/controllers.go index 970146ab..ebb2fc37 100644 --- a/api/queryService/controllers.go +++ b/api/queryService/controllers.go @@ -2,11 +2,10 @@ package queryservice import ( "queryorchestration/internal/client" + "queryorchestration/internal/collector" + collectorupdate "queryorchestration/internal/collector/update" "queryorchestration/internal/document" "queryorchestration/internal/export" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" - collectorupdate "queryorchestration/internal/job/collector/update" "queryorchestration/internal/query" querytest "queryorchestration/internal/query/test" queryupdate "queryorchestration/internal/query/update" @@ -24,7 +23,6 @@ type Services struct { QueryUpdate *queryupdate.Service QueryTest *querytest.Service Client *client.Service - Job *job.Service Document *document.Service } diff --git a/api/queryService/documents.go b/api/queryService/documents.go index e72ab9c0..c9dd5e27 100644 --- a/api/queryService/documents.go +++ b/api/queryService/documents.go @@ -8,8 +8,8 @@ import ( "github.com/oapi-codegen/runtime/types" ) -func (s *Controllers) ListDocumentsByJobId(ctx echo.Context, jobId types.UUID) error { - documents, err := s.svc.Document.ListByJobId(ctx.Request().Context(), jobId) +func (s *Controllers) ListDocumentsByClientId(ctx echo.Context, clientId types.UUID) error { + documents, err := s.svc.Document.ListByClientId(ctx.Request().Context(), clientId) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to list documents: %s", err)) } diff --git a/api/queryService/documents_test.go b/api/queryService/documents_test.go index 965798c3..210cebc5 100644 --- a/api/queryService/documents_test.go +++ b/api/queryService/documents_test.go @@ -12,7 +12,7 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/document" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/go-playground/validator/v10" @@ -23,19 +23,19 @@ import ( "github.com/stretchr/testify/require" ) -func TestListDocumentsByJobId(t *testing.T) { +func TestListDocumentsByClientId(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) cfg := &struct { serviceconfig.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig }{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - jobId := uuid.New() + clientId := uuid.New() e := echo.New() req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) @@ -47,7 +47,7 @@ func TestListDocumentsByJobId(t *testing.T) { Document: document.New(cfg), }) - ctx.Set("id", jobId) + ctx.Set("id", clientId) doc := queryservice.ListDocuments{ { Id: uuid.New(), @@ -56,13 +56,13 @@ func TestListDocumentsByJobId(t *testing.T) { }, } - pool.ExpectQuery("name: ListDocumentsByJobId :many").WithArgs(database.MustToDBUUID(jobId)). + pool.ExpectQuery("name: ListDocumentsByClientId :many").WithArgs(database.MustToDBUUID(clientId)). WillReturnRows( pgxmock.NewRows([]string{"id", "bucket", "key"}). AddRow(database.MustToDBUUID(doc[0].Id), doc[0].Bucket, doc[0].Key), ) - err = cons.ListDocumentsByJobId(ctx, jobId) + err = cons.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Equal(t, http.StatusOK, rec.Code) diff --git a/api/queryService/job.go b/api/queryService/job.go deleted file mode 100644 index 8af9f73c..00000000 --- a/api/queryService/job.go +++ /dev/null @@ -1,69 +0,0 @@ -package queryservice - -import ( - "fmt" - "net/http" - - "queryorchestration/internal/job" - - "github.com/labstack/echo/v4" - "github.com/oapi-codegen/runtime/types" -) - -func (s *Controllers) CreateJob(ctx echo.Context) error { - req := JobCreate{} - if err := ctx.Bind(&req); err != nil { - return echo.NewHTTPError(http.StatusBadRequest, err) - } - - id, err := s.svc.Job.Create(ctx.Request().Context(), req.ClientId) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to create job: %s", err)) - } - - return ctx.JSON(http.StatusCreated, IdMessage{ - Id: id, - }) -} - -func (s *Controllers) GetJob(ctx echo.Context, id types.UUID) error { - job, err := s.svc.Job.Get(ctx.Request().Context(), id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to find job: %s", err)) - } - - return ctx.JSON(http.StatusOK, Job{ - Id: job.ID, - ClientId: job.ClientID, - CanSync: job.CanSync, - }) -} - -func (s *Controllers) GetJobStatusByJobId(ctx echo.Context, id types.UUID) error { - status, err := s.svc.Job.GetStatus(ctx.Request().Context(), id) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to find job: %s", err)) - } - - return ctx.JSON(http.StatusOK, JobStatusBody{ - JobId: id, - Status: parseJobStatus(*status), - }) -} - -func (s *Controllers) UpdateJob(ctx echo.Context, id types.UUID) error { - req := JobUpdate{} - if err := ctx.Bind(&req); err != nil { - return echo.NewHTTPError(http.StatusBadRequest, err) - } - - err := s.svc.Job.Update(ctx.Request().Context(), &job.Update{ - ID: id, - CanSync: req.CanSync, - }) - if err != nil { - return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to update job: %s", err)) - } - - return ctx.NoContent(http.StatusOK) -} diff --git a/api/queryService/job_test.go b/api/queryService/job_test.go deleted file mode 100644 index 2ac294b7..00000000 --- a/api/queryService/job_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package queryservice_test - -import ( - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - - queryservice "queryorchestration/api/queryService" - "queryorchestration/internal/client" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - cleanversion "queryorchestration/internal/document/clean/version" - textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/serviceconfig" - - "github.com/go-playground/validator/v10" - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/labstack/echo/v4" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCreateJob(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(cfg, &job.Services{ - Collector: collector.New(cfg, &collector.Services{ - CleanVersion: cleanversion.New(cfg), - TextVersion: textversion.New(cfg), - }), - }), - }) - - body := queryservice.JobCreate{ - ClientId: uuid.New(), - } - bodyBytes, err := json.Marshal(body) - assert.NoError(t, err) - - e := echo.New() - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(bodyBytes))) - req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) - rec := httptest.NewRecorder() - ctx := e.NewContext(req, rec) - - id := uuid.New() - - pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(body.ClientId)).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(id)), - ) - pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"version"}). - AddRow(int32(1)), - ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(id), int32(1)). - WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectCommit() - - err = cons.CreateJob(ctx) - assert.NoError(t, err) - assert.Equal(t, http.StatusCreated, rec.Code) - assert.Equal(t, fmt.Sprintf("{\"id\":\"%s\"}\n", id), rec.Body.String()) -} - -func TestGetJob(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(cfg, &job.Services{ - Client: client.New(cfg), - }), - }) - - e := echo.New() - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) - rec := httptest.NewRecorder() - ctx := e.NewContext(req, rec) - - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: true, - } - - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"issynced"}). - AddRow(true), - ) - - err = cons.GetJob(ctx, j.ID) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, rec.Code) - - var res queryservice.Job - err = json.Unmarshal(rec.Body.Bytes(), &res) - assert.NoError(t, err) - assert.EqualExportedValues(t, queryservice.Job{ - Id: j.ID, - ClientId: j.ClientID, - CanSync: j.CanSync, - }, res) -} - -func TestGetJobStatus(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(cfg, &job.Services{ - Client: client.New(cfg), - }), - }) - - e := echo.New() - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) - rec := httptest.NewRecorder() - ctx := e.NewContext(req, rec) - - jobId := uuid.New() - - pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows( - pgxmock.NewRows([]string{"issynced"}). - AddRow(true), - ) - - err = cons.GetJobStatusByJobId(ctx, jobId) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, rec.Code) - - var res queryservice.JobStatusBody - err = json.Unmarshal(rec.Body.Bytes(), &res) - assert.NoError(t, err) - assert.EqualExportedValues(t, queryservice.JobStatusBody{ - JobId: jobId, - Status: queryservice.INSYNC, - }, res) -} - -func TestUpdateJob(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Job: job.New(cfg, &job.Services{ - Client: client.New(cfg), - }), - }) - - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: false, - } - - ucs := !j.CanSync - body := queryservice.JobUpdate{ - CanSync: &ucs, - } - bodyBytes, err := json.Marshal(body) - assert.NoError(t, err) - - e := echo.New() - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(string(bodyBytes))) - req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) - rec := httptest.NewRecorder() - ctx := e.NewContext(req, rec) - - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - pool.ExpectBegin() - pool.ExpectExec("name: AddJobCanSync :exec").WithArgs(*body.CanSync, database.MustToDBUUID(j.ID)). - WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectCommit() - - err = cons.UpdateJob(ctx, j.ID) - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, rec.Code) - assert.Empty(t, rec.Body.String()) -} diff --git a/api/queryService/parse.go b/api/queryService/parse.go index 2b7a9685..a5343ebb 100644 --- a/api/queryService/parse.go +++ b/api/queryService/parse.go @@ -3,7 +3,7 @@ package queryservice import ( "errors" - "queryorchestration/internal/job" + "queryorchestration/internal/client" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" @@ -87,10 +87,12 @@ func parseStringToUUIDArray(sids *[]string) (*[]uuid.UUID, error) { return nil, nil } -func parseJobStatus(status job.Status) JobStatus { +func parseClientStatus(status client.Status) ClientStatus { switch status { - case job.IN_SYNC: + case client.IN_SYNC: return INSYNC + case client.NOT_SYNCING: + return NOTSYNCING } return NOTSYNCED diff --git a/api/queryService/parse_test.go b/api/queryService/parse_test.go index 83fefc32..80e3beb4 100644 --- a/api/queryService/parse_test.go +++ b/api/queryService/parse_test.go @@ -3,7 +3,7 @@ package queryservice import ( "testing" - "queryorchestration/internal/job" + "queryorchestration/internal/client" "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" @@ -127,13 +127,16 @@ func TestParseStringToUUIDArray(t *testing.T) { assert.Nil(t, out) } -func TestParseSpecJobStatus(t *testing.T) { - qt := parseJobStatus(job.IN_SYNC) +func TestParseSpecClientStatus(t *testing.T) { + qt := parseClientStatus(client.IN_SYNC) assert.Equal(t, INSYNC, qt) - qt = parseJobStatus(job.NOT_SYNCED) + qt = parseClientStatus(client.NOT_SYNCED) assert.Equal(t, NOTSYNCED, qt) - qt = parseJobStatus("invalid") + qt = parseClientStatus(client.NOT_SYNCING) + assert.Equal(t, NOTSYNCING, qt) + + qt = parseClientStatus("invalid") assert.Equal(t, NOTSYNCED, qt) } diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index bca6e5a0..19eaa0ec 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -9,12 +9,12 @@ import ( "testing" queryservice "queryorchestration/api/queryService" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" querytest "queryorchestration/internal/query/test" diff --git a/api/queryService/status.go b/api/queryService/status.go new file mode 100644 index 00000000..c295d76e --- /dev/null +++ b/api/queryService/status.go @@ -0,0 +1,21 @@ +package queryservice + +import ( + "fmt" + "net/http" + + "github.com/labstack/echo/v4" + "github.com/oapi-codegen/runtime/types" +) + +func (s *Controllers) GetStatusByClientId(ctx echo.Context, id types.UUID) error { + status, err := s.svc.Client.GetStatus(ctx.Request().Context(), id) + if err != nil { + return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Unable to find client: %s", err)) + } + + return ctx.JSON(http.StatusOK, ClientStatusBody{ + ClientId: id, + Status: parseClientStatus(status), + }) +} diff --git a/api/queryService/status_test.go b/api/queryService/status_test.go new file mode 100644 index 00000000..c05b048c --- /dev/null +++ b/api/queryService/status_test.go @@ -0,0 +1,70 @@ +package queryservice_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + queryservice "queryorchestration/api/queryService" + "queryorchestration/internal/client" + "queryorchestration/internal/collector" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + cleanversion "queryorchestration/internal/document/clean/version" + textversion "queryorchestration/internal/document/text/version" + "queryorchestration/internal/serviceconfig" + + "github.com/go-playground/validator/v10" + "github.com/google/uuid" + "github.com/labstack/echo/v4" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetClientStatus(t *testing.T) { + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ + Client: client.New(cfg, &client.Services{ + Collector: collector.New(cfg, &collector.Services{ + TextVersion: textversion.New(cfg), + CleanVersion: cleanversion.New(cfg), + }), + }), + }) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("")) + rec := httptest.NewRecorder() + ctx := e.NewContext(req, rec) + + clientId := uuid.New() + + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"id", "name", "can_sync"}). + AddRow(database.MustToDBUUID(clientId), "name", true), + ) + pool.ExpectQuery("name: IsClientSynced :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(true), + ) + + err = cons.GetStatusByClientId(ctx, clientId) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + var res queryservice.ClientStatusBody + err = json.Unmarshal(rec.Body.Bytes(), &res) + assert.NoError(t, err) + assert.EqualExportedValues(t, queryservice.ClientStatusBody{ + ClientId: clientId, + Status: queryservice.INSYNC, + }, res) +} diff --git a/api/queryVersionSyncRunner/runner_test.go b/api/queryVersionSyncRunner/runner_test.go index 4bc89b24..b3aa9119 100644 --- a/api/queryVersionSyncRunner/runner_test.go +++ b/api/queryVersionSyncRunner/runner_test.go @@ -11,7 +11,7 @@ import ( "queryorchestration/internal/database/repository" queryversionsync "queryorchestration/internal/query/versionsync" "queryorchestration/internal/server/runner" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -26,7 +26,7 @@ import ( type QuerySyncConfig struct { runner.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig } func TestQueryRunner(t *testing.T) { @@ -40,7 +40,7 @@ func TestQueryRunner(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - cfg.JobSyncURL = "/i/am/here" + cfg.ClientSyncURL = "/i/am/here" svc := queryversionsync.New(cfg) @@ -60,23 +60,23 @@ func TestQueryRunner(t *testing.T) { Body: &body, } - jobIds := []uuid.UUID{ + clientIds := []uuid.UUID{ uuid.New(), uuid.New(), } - pool.ExpectQuery("name: ListQueryJobIDs :many").WithArgs(database.MustToDBUUID(doc.ID)). + pool.ExpectQuery("name: ListQueryClientIDs :many").WithArgs(database.MustToDBUUID(doc.ID)). WillReturnRows( - pgxmock.NewRows([]string{"jobId"}). - AddRow(database.MustToDBUUID(jobIds[0])). - AddRow(database.MustToDBUUID(jobIds[1])), + pgxmock.NewRows([]string{"clientId"}). + AddRow(database.MustToDBUUID(clientIds[0])). + AddRow(database.MustToDBUUID(clientIds[1])), ) mockSQS.EXPECT(). SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", jobIds[0].String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[0].String()) }), mock.Anything, ). @@ -85,7 +85,7 @@ func TestQueryRunner(t *testing.T) { SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", jobIds[1].String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[1].String()) }), mock.Anything, ). diff --git a/cmd/jobSyncRunner/main.go b/cmd/clientSyncRunner/main.go similarity index 60% rename from cmd/jobSyncRunner/main.go rename to cmd/clientSyncRunner/main.go index 19f1c2d2..ed20b10c 100644 --- a/cmd/jobSyncRunner/main.go +++ b/cmd/clientSyncRunner/main.go @@ -1,8 +1,8 @@ -// **Listens For**: Body Containing Job ID +// **Listens For**: Body Containing Client ID // // **Action**. // -// Adds an event for each document into the DOCSYNC queue for the given Job. +// Adds an event for each document into the DOCSYNC queue for the given Client. package main import ( @@ -10,15 +10,15 @@ import ( "log/slog" "os" - jobsyncrunner "queryorchestration/api/jobSyncRunner" - jobsync "queryorchestration/internal/job/sync" + clientsyncrunner "queryorchestration/api/clientSyncRunner" + clientsync "queryorchestration/internal/client/sync" "queryorchestration/internal/server/runner" "queryorchestration/internal/serviceconfig/queue/documentsync" _ "github.com/lib/pq" ) -type JobSyncConfig struct { +type ClientSyncConfig struct { runner.BaseConfig documentsync.DocSyncConfig } @@ -26,13 +26,13 @@ type JobSyncConfig struct { func main() { ctx := context.Background() - cfg := &JobSyncConfig{} + cfg := &ClientSyncConfig{} cfg.ControllerFunc = func() runner.Controller { - svc := jobsync.New(cfg) + svc := clientsync.New(cfg) - c := jobsyncrunner.New(cfg.GetValidator(), &jobsyncrunner.Services{ - JobSync: svc, + c := clientsyncrunner.New(cfg.GetValidator(), &clientsyncrunner.Services{ + ClientSync: svc, }) return &c diff --git a/cmd/docCleanRunner/main.go b/cmd/docCleanRunner/main.go index 1edb66d2..32300cfa 100644 --- a/cmd/docCleanRunner/main.go +++ b/cmd/docCleanRunner/main.go @@ -2,7 +2,7 @@ // // **Action**. // -// If the document is not already clean according to the job collector standards, a clean is triggered. +// If the document is not already clean according to the collector standards, a clean is triggered. // // An event with the document id is pushed to the DOCTEXT queue. package main diff --git a/cmd/docInitRunner/main.go b/cmd/docInitRunner/main.go index 2f2e43f7..d2f720fa 100644 --- a/cmd/docInitRunner/main.go +++ b/cmd/docInitRunner/main.go @@ -4,7 +4,7 @@ // // Checks if the uploaded document is a duplicate. // -// Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the job. +// Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the client. // // Find reference entity. // diff --git a/cmd/docSyncRunner/main.go b/cmd/docSyncRunner/main.go index 8b156ee3..e13a8087 100644 --- a/cmd/docSyncRunner/main.go +++ b/cmd/docSyncRunner/main.go @@ -4,7 +4,7 @@ // // Checks if the document is eligible for syncing. // -// By validating: the client can_sync parameter, the job can_sync parameter. +// By validating: the client can_sync parameter. // // If eligible, send an event to the DOCCLEAN queue. package main @@ -16,12 +16,11 @@ import ( docsyncrunner "queryorchestration/api/docSyncRunner" "queryorchestration/internal/client" + "queryorchestration/internal/collector" "queryorchestration/internal/document" cleanversion "queryorchestration/internal/document/clean/version" documentsync "queryorchestration/internal/document/sync" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" "queryorchestration/internal/server/runner" documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean" @@ -39,21 +38,17 @@ func main() { cfg := &DocSyncConfig{} cfg.ControllerFunc = func() runner.Controller { - tev := textversion.New(cfg) - clv := cleanversion.New(cfg) - col := collector.New(cfg, &collector.Services{ - CleanVersion: clv, - TextVersion: tev, + coll := collector.New(cfg, &collector.Services{ + CleanVersion: cleanversion.New(cfg), + TextVersion: textversion.New(cfg), }) - cli := client.New(cfg) - jbb := job.New(cfg, &job.Services{ - Client: cli, - Collector: col, + cli := client.New(cfg, &client.Services{ + Collector: coll, }) doc := document.New(cfg) docsync := documentsync.New(cfg, &documentsync.Services{ Document: doc, - Job: jbb, + Client: cli, }) return docsyncrunner.New(cfg.GetValidator(), &docsyncrunner.Services{ diff --git a/cmd/docTextRunner/main.go b/cmd/docTextRunner/main.go index d8dd3d01..e32cd7c6 100644 --- a/cmd/docTextRunner/main.go +++ b/cmd/docTextRunner/main.go @@ -2,7 +2,7 @@ // // **Action**. // -// If the document text is not already extracted according to the job collector standards, the text is extracted. +// If the document text is not already extracted according to the collector standards, the text is extracted. // // An event with the document id is pushed to the QUERYSYNC queue. package main diff --git a/cmd/queryService/main.go b/cmd/queryService/main.go index e5c1f6ca..8ad8caa4 100644 --- a/cmd/queryService/main.go +++ b/cmd/queryService/main.go @@ -2,17 +2,15 @@ // // The entities in question: // -// - Client - A client is a single client of the Doczy project, all configurations and jobs for a client will be in reference to the respective client. +// - Client - A client is a single client of the Doczy project, all configurations and clients for a client will be in reference to the respective client. // -// - Job - A job is a single instance of requirements for a client. This maintains a set of queries and processes in relation to itself. +// - Collector - A collector specifies the required queries for export for a given client. // -// - Collector - A collector specifies the required queries for export for a given job. -// -// - Sample - A sample is a subset of documents to be used for testing. When it is active a job will only process documents in the sample. +// - Sample - A sample is a subset of documents to be used for testing. When it is active a client will only process documents in the sample. // // - Query - A query is a block of processing of data which takes an input and produces an output. // -// - Export - An export is a manually started process which outputs the results for a job to a given location. +// - Export - An export is a manually started process which outputs the results for a client to a given location. package main import ( @@ -23,19 +21,18 @@ import ( queryservice "queryorchestration/api/queryService" "queryorchestration/internal/client" + "queryorchestration/internal/collector" + collectorupdate "queryorchestration/internal/collector/update" "queryorchestration/internal/document" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" "queryorchestration/internal/export" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" - collectorupdate "queryorchestration/internal/job/collector/update" "queryorchestration/internal/query" "queryorchestration/internal/query/result" querytest "queryorchestration/internal/query/test" queryupdate "queryorchestration/internal/query/update" service "queryorchestration/internal/server/service" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" "queryorchestration/internal/serviceconfig/queue/queryversionsync" "github.com/getkin/kin-openapi/openapi3" @@ -44,7 +41,7 @@ import ( type QueryServiceConfig struct { service.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig queryversionsync.QueryVersionSyncConfig } @@ -70,9 +67,7 @@ func main() { CleanVersion: clv, TextVersion: tev, }) - cli := client.New(cfg) - jbb := job.New(cfg, &job.Services{ - Client: cli, + cli := client.New(cfg, &client.Services{ Collector: col, }) doc := document.New(cfg) @@ -93,7 +88,6 @@ func main() { QueryUpdate: qupdate, QueryTest: quetest, Client: cli, - Job: jbb, Document: doc, } diff --git a/cmd/queryVersionSyncRunner/main.go b/cmd/queryVersionSyncRunner/main.go index f775d523..4becf596 100644 --- a/cmd/queryVersionSyncRunner/main.go +++ b/cmd/queryVersionSyncRunner/main.go @@ -2,7 +2,7 @@ // // **Action**. // -// Adds an event for each job into the JOBSYNC queue for the given query, i.e. each job that contains the query in its dependency tree. +// Adds an event for each client into the CLIENTSYNC queue for the given query, i.e. each client that contains the query in its dependency tree. package main import ( @@ -13,14 +13,14 @@ import ( queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner" queryversionsync "queryorchestration/internal/query/versionsync" "queryorchestration/internal/server/runner" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" _ "github.com/lib/pq" ) type QueryVersionSyncConfig struct { runner.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig } func main() { diff --git a/database/diagram.mmd b/database/diagram.mmd index b1b5952c..2a6b943c 100644 --- a/database/diagram.mmd +++ b/database/diagram.mmd @@ -1,13 +1,11 @@ erDiagram clients ||--o{ clientCanSync : "has" - clients ||--o{ jobs : "has" - jobs ||--o{ jobCanSync : "has" - jobs ||--o{ documents : "contains" - jobs ||--o{ collectorVersions : "has" - jobs ||--o{ collectorActiveVersions : "has current" - jobs ||--o{ collectorMinCleanVersions : "has" - jobs ||--o{ collectorMinTextVersions : "has" - jobs ||--o{ collectorQueries : "maps to" + clients ||--o{ documents : "contains" + clients ||--o{ collectorVersions : "has" + clients ||--o{ collectorActiveVersions : "has current" + clients ||--o{ collectorMinCleanVersions : "has" + clients ||--o{ collectorMinTextVersions : "has" + clients ||--o{ collectorQueries : "maps to" documents ||--o{ documentEntries : "has" documents ||--o{ documentCleans : "has clean version" @@ -34,20 +32,9 @@ erDiagram boolean canSync } - jobs { - uuid id PK - uuid clientId FK - } - - jobCanSync { - uuid id PK - uuid jobId FK - boolean canSync - } - documents { uuid id PK - uuid jobId FK + uuid clientId FK text hash } @@ -108,20 +95,20 @@ erDiagram } collectorVersions { - uuid jobId FK + uuid clientId FK int id timestamp addedAt } collectorActiveVersions { uuid id PK - uuid jobId FK + uuid clientId FK int versionId FK } collectorMinCleanVersions { uuid id PK - uuid jobId FK + uuid clientId FK int versionId int addedVersion FK int removedVersion FK @@ -129,7 +116,7 @@ erDiagram collectorMinTextVersions { uuid id PK - uuid jobId FK + uuid clientId FK int versionId int addedVersion FK int removedVersion FK @@ -137,7 +124,7 @@ erDiagram collectorQueries { uuid id PK - uuid jobId FK + uuid clientId FK varchar name uuid queryId FK int addedVersion FK diff --git a/database/migrations/00000000000005_collectors.down.sql b/database/migrations/00000000000004_collectors.down.sql similarity index 100% rename from database/migrations/00000000000005_collectors.down.sql rename to database/migrations/00000000000004_collectors.down.sql diff --git a/database/migrations/00000000000005_collectors.up.sql b/database/migrations/00000000000004_collectors.up.sql similarity index 61% rename from database/migrations/00000000000005_collectors.up.sql rename to database/migrations/00000000000004_collectors.up.sql index bffa4323..e98b42ee 100644 --- a/database/migrations/00000000000005_collectors.up.sql +++ b/database/migrations/00000000000004_collectors.up.sql @@ -1,9 +1,9 @@ CREATE TABLE collectorVersions ( - jobId uuid not null, + clientId uuid not null, id int not null, addedAt timestamp not null default current_timestamp, - foreign key (jobId) references jobs(id), - primary key (id, jobId) + foreign key (clientId) references clients(id), + primary key (id, clientId) ); CREATE OR REPLACE FUNCTION setCollectorVersionNumber() @@ -12,7 +12,7 @@ BEGIN SELECT COALESCE(MAX(id), 0) + 1 INTO NEW.id FROM collectorVersions - WHERE jobId = NEW.jobId; + WHERE clientId = NEW.clientId; RETURN NEW; END; @@ -25,20 +25,20 @@ EXECUTE FUNCTION setCollectorVersionNumber(); CREATE TABLE collectorActiveVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, + clientId uuid not null, versionId int not null, - foreign key (jobId, versionId) references collectorVersions(jobId, id) + foreign key (clientId, versionId) references collectorVersions(clientId, id) ); CREATE TABLE collectorMinCleanVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, + clientId uuid not null, versionId int not null, addedVersion int not null, removedVersion int, - foreign key (jobId, addedVersion) references collectorVersions(jobId, id), - foreign key (jobId, removedVersion) references collectorVersions(jobId, id), - foreign key (jobId) references jobs(id) + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + foreign key (clientId) references clients(id) ); CREATE OR REPLACE FUNCTION removeMinCleanVersion() @@ -46,7 +46,7 @@ RETURNS TRIGGER AS $$ BEGIN UPDATE collectorMinCleanVersions SET removedVersion = NEW.addedVersion - WHERE jobId = NEW.jobId and removedVersion is null; + WHERE clientId = NEW.clientId and removedVersion is null; RETURN NEW; END; @@ -59,13 +59,13 @@ EXECUTE FUNCTION removeMinCleanVersion(); CREATE TABLE collectorMinTextVersions ( id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, + clientId uuid not null, versionId int not null, addedVersion int not null, removedVersion int, - foreign key (jobId, addedVersion) references collectorVersions(jobId, id), - foreign key (jobId, removedVersion) references collectorVersions(jobId, id), - foreign key (jobId) references jobs(id) + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + foreign key (clientId) references clients(id) ); CREATE OR REPLACE FUNCTION removeMinTextVersion() @@ -73,7 +73,7 @@ RETURNS TRIGGER AS $$ BEGIN UPDATE collectorMinTextVersions SET removedVersion = NEW.addedVersion - WHERE jobId = NEW.jobId and removedVersion is null; + WHERE clientId = NEW.clientId and removedVersion is null; RETURN NEW; END; @@ -86,14 +86,14 @@ EXECUTE FUNCTION removeMinTextVersion(); CREATE TABLE collectorQueries ( id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, + clientId uuid not null, name varchar(255) not null, queryId uuid not null, addedVersion int not null, removedVersion int, foreign key (queryId) references queries(id), - foreign key (jobId) references jobs(id), - foreign key (jobId, addedVersion) references collectorVersions(jobId, id), - foreign key (jobId, removedVersion) references collectorVersions(jobId, id), - unique (jobId, name, removedVersion) + foreign key (clientId) references clients(id), + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + unique (clientId, name, removedVersion) ); diff --git a/database/migrations/00000000000004_jobs.down.sql b/database/migrations/00000000000004_jobs.down.sql deleted file mode 100644 index b23f8f7b..00000000 --- a/database/migrations/00000000000004_jobs.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE jobs; \ No newline at end of file diff --git a/database/migrations/00000000000004_jobs.up.sql b/database/migrations/00000000000004_jobs.up.sql deleted file mode 100644 index 7e206c44..00000000 --- a/database/migrations/00000000000004_jobs.up.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE jobs ( - id uuid primary key DEFAULT uuid_generate_v7(), - clientId uuid not null, - foreign key (clientId) references clients(id) -); - -CREATE TABLE jobCanSync ( - id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, - canSync boolean not null, - foreign key (jobId) references jobs(id) -); \ No newline at end of file diff --git a/database/migrations/00000000000006_documents.down.sql b/database/migrations/00000000000005_documents.down.sql similarity index 100% rename from database/migrations/00000000000006_documents.down.sql rename to database/migrations/00000000000005_documents.down.sql diff --git a/database/migrations/00000000000006_documents.up.sql b/database/migrations/00000000000005_documents.up.sql similarity index 93% rename from database/migrations/00000000000006_documents.up.sql rename to database/migrations/00000000000005_documents.up.sql index edb66609..ead99cc6 100644 --- a/database/migrations/00000000000006_documents.up.sql +++ b/database/migrations/00000000000005_documents.up.sql @@ -1,9 +1,9 @@ CREATE TABLE documents ( id uuid primary key DEFAULT uuid_generate_v7(), - jobId uuid not null, + clientId uuid not null, hash text not null, - foreign key (jobId) references jobs(id), - unique(jobId, hash) + foreign key (clientId) references clients(id), + unique(clientId, hash) ); CREATE TABLE documentEntries ( diff --git a/database/migrations/00000000000007_results.down.sql b/database/migrations/00000000000006_results.down.sql similarity index 100% rename from database/migrations/00000000000007_results.down.sql rename to database/migrations/00000000000006_results.down.sql diff --git a/database/migrations/00000000000007_results.up.sql b/database/migrations/00000000000006_results.up.sql similarity index 100% rename from database/migrations/00000000000007_results.up.sql rename to database/migrations/00000000000006_results.up.sql diff --git a/database/migrations/00000000000101_collector_views.up.sql b/database/migrations/00000000000101_collector_views.up.sql index cb5071e5..9cc969cc 100644 --- a/database/migrations/00000000000101_collector_views.up.sql +++ b/database/migrations/00000000000101_collector_views.up.sql @@ -1,77 +1,77 @@ CREATE VIEW collectorCurrentActiveVersions as SELECT DISTINCT - j.id as jobId, + j.id as clientId, coalesce( (FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)), 0 )::int as activeVersion - FROM jobs as j - LEFT JOIN collectorActiveVersions as v on v.jobId = j.id; + FROM clients as j + LEFT JOIN collectorActiveVersions as v on v.clientId = j.id; CREATE VIEW collectorLatestVersions as SELECT - j.id as jobId, + j.id as clientId, coalesce(max(v.id), 0)::int as latestVersion - FROM jobs as j - LEFT JOIN collectorVersions as v on v.jobId = j.id + FROM clients as j + LEFT JOIN collectorVersions as v on v.clientId = j.id GROUP BY j.id; CREATE VIEW currentCollectorMinTextVersions as SELECT DISTINCT - av.jobId, + av.clientId, coalesce(ctv.versionId, 0) as minTextVersion FROM collectorCurrentActiveVersions as av - LEFT JOIN collectorMinTextVersions AS ctv ON av.jobId = ctv.jobId + LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); CREATE VIEW currentCollectorMinCleanVersions as SELECT DISTINCT - av.jobId, + av.clientId, coalesce(ctv.versionId, 0) as minCleanVersion FROM collectorCurrentActiveVersions as av - LEFT JOIN collectorMinCleanVersions AS ctv ON av.jobId = ctv.jobId + LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); CREATE VIEW currentCollectorQueries as SELECT DISTINCT - av.jobId, + av.clientId, q.name, q.queryId FROM collectorCurrentActiveVersions as av - LEFT JOIN collectorQueries AS q ON av.jobId = q.jobId + LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion); CREATE VIEW currentCollectorQueriesJSONAGG as SELECT DISTINCT - jobId, + clientId, jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields FROM currentCollectorQueries - GROUP BY jobId; + GROUP BY clientId; CREATE VIEW fullActiveCollectors AS - SELECT DISTINCT av.jobId, + SELECT DISTINCT av.clientId, ccv.minCleanVersion, ctv.minTextVersion, av.activeVersion, lv.latestVersion, q.fields FROM collectorCurrentActiveVersions as av - JOIN collectorLatestVersions as lv on lv.jobId = av.jobId - JOIN currentCollectorMinCleanVersions AS ccv ON av.jobId = ccv.jobId - JOIN currentCollectorMinTextVersions AS ctv ON av.jobId = ctv.jobId - JOIN currentCollectorQueriesJSONAGG AS q ON av.jobId = q.jobId; + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId + JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId + JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId; CREATE VIEW collectorQueryDependencyTree AS WITH RECURSIVE collectorQueryDependencyTree AS ( - SELECT cq.jobId, cq.queryId, ri.requiredIds + SELECT cq.clientId, cq.queryId, ri.requiredIds FROM currentCollectorQueries as cq JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId UNION ALL - SELECT acq.jobId, q.queryId, q.requiredIds + SELECT acq.clientId, q.queryId, q.requiredIds FROM queryCurrentRequiredIdsAGG as q JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) ) - SELECT DISTINCT ct.jobId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds + SELECT DISTINCT ct.clientId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds FROM collectorQueryDependencyTree as ct JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId JOIN queries as q on q.id = ct.queryId; diff --git a/database/migrations/00000000000102_job_views.down.sql b/database/migrations/00000000000102_job_views.down.sql index 58d361aa..6b386842 100644 --- a/database/migrations/00000000000102_job_views.down.sql +++ b/database/migrations/00000000000102_job_views.down.sql @@ -1 +1 @@ -DROP VIEW listJobDocumentIDs; +DROP VIEW listDocumentIDs; diff --git a/database/migrations/00000000000102_job_views.up.sql b/database/migrations/00000000000102_job_views.up.sql index 057bdf90..fe6b16a8 100644 --- a/database/migrations/00000000000102_job_views.up.sql +++ b/database/migrations/00000000000102_job_views.up.sql @@ -1,5 +1,5 @@ -CREATE OR REPLACE FUNCTION listJobDocumentIDs( - _jobId uuid, +CREATE OR REPLACE FUNCTION listDocumentIDs( + _clientId uuid, _batchSize INTEGER, _offset INTEGER ) @@ -7,12 +7,12 @@ RETURNS TABLE (id uuid, totalCount BIGINT) AS $$ DECLARE totalCount BIGINT := 0; BEGIN - SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE jobId = _jobId; + SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE clientId = _clientId; RETURN QUERY SELECT d.id, totalCount FROM documents as d - WHERE d.jobId = _jobId + WHERE d.clientId = _clientId ORDER BY d.id LIMIT _batchSize OFFSET _offset; @@ -31,7 +31,7 @@ WITH RankedExtractions AS ( dce.version, ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) as row_num FROM documents d - JOIN currentCollectorMinCleanVersions ccv ON d.jobId = ccv.jobId + JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId JOIN documentCleans dc ON dc.documentId = d.id JOIN documentCleanEntries dce ON dc.id = dce.cleanId AND dce.version >= ccv.minCleanVersion @@ -59,7 +59,7 @@ WITH RankedExtractions AS ( ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num FROM documents d JOIN currentCleanEntries cc on cc.documentId = d.id - JOIN currentCollectorMinTextVersions ctv ON ctv.jobId = d.jobId + JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId JOIN documentTextExtractions dte ON dte.cleanEntryId = cc.id AND dte.version >= ctv.minTextVersion ) @@ -88,17 +88,3 @@ SELECT FROM clients c LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1; -CREATE VIEW currentJobCanSync as -WITH RankedExtractions AS ( - SELECT - ccs.jobId, - ccs.canSync, - ROW_NUMBER() OVER (PARTITION BY ccs.jobId ORDER BY ccs.id DESC) as row_num - FROM jobs c - JOIN jobCanSync ccs on c.id = ccs.jobId -) -SELECT - c.id as jobId, - coalesce(r.canSync, false) as canSync -FROM jobs c -LEFT JOIN RankedExtractions r on r.jobId = c.id and r.row_num = 1; diff --git a/database/queries/client.sql b/database/queries/client.sql index 64449071..b939a2a5 100644 --- a/database/queries/client.sql +++ b/database/queries/client.sql @@ -11,4 +11,91 @@ SELECT c.id, c.name, cs.canSync UPDATE clients SET name = $1 WHERE id = $2; -- name: AddClientCanSync :exec -INSERT INTO clientCanSync (canSync, clientId) VALUES ($1, $2); \ No newline at end of file +INSERT INTO clientCanSync (canSync, clientId) VALUES ($1, $2); + +-- name: IsClientSynced :one +WITH +docs AS ( + -- Get all documents for this client + SELECT id, clientId FROM documents WHERE clientId = $1 +), +doc_clean_entries as ( + -- Documents with their current clean entries + SELECT + d.id AS document_id, + d.clientId as client_id, + cte.id AS clean_entry_id + FROM + docs d + LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id + WHERE cte.id is null or cte.id is not null and cte.fail is null +), +doc_text_entries AS ( + -- Documents with their current text entries + SELECT + d.document_id, + cte.id AS text_entry_id + FROM + doc_clean_entries d + LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id +), +required_results AS ( + -- All required document-query-version combinations + SELECT + d.document_id, + cqdt.queryId, + cqdt.queryVersion, + dte.text_entry_id + FROM + doc_clean_entries d + JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id + JOIN doc_text_entries dte ON dte.document_id = d.document_id + and dte.text_entry_id IS NOT NULL +), +existing_results AS ( + -- Valid results that exist + SELECT + rr.document_id AS document_id, + r.queryId, + r.id AS result_id + FROM + results r + JOIN required_results rr ON + r.queryId = rr.queryId AND + r.queryVersion = rr.queryVersion AND + r.textEntryId = rr.text_entry_id +), +missing_results AS ( + -- Find missing results + SELECT rr.queryId + FROM required_results rr + LEFT JOIN existing_results er on er.queryId = rr.queryId + WHERE er.result_id is null +), +dependency_check AS ( + -- Check for missing dependencies + SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId + FROM existing_results er + JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId + LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id + WHERE rd.resultId is null +) +SELECT ( + -- No documents means client is synced + NOT EXISTS (SELECT 1 FROM docs) + + OR + + -- Documents with no text entries + (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) + + and + + -- Documents with missing results + NOT EXISTS (SELECT 1 FROM missing_results) + + and + + -- Documents with missing dependencies + NOT EXISTS (SELECT 1 FROM dependency_check)) +)::bool as is_synced; diff --git a/database/queries/collector.sql b/database/queries/collector.sql index 3e68f7be..6e946f16 100644 --- a/database/queries/collector.sql +++ b/database/queries/collector.sql @@ -1,26 +1,23 @@ -- name: ListCollectorQueries :many -SELECT * FROM collectorQueryDependencyTree WHERE jobId = @jobId; +SELECT * FROM collectorQueryDependencyTree WHERE clientId = @clientId; --- name: GetCollectorByJobID :one -SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1; - --- name: GetCollector :one -SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1; +-- name: GetCollectorByClientID :one +SELECT * FROM fullActiveCollectors WHERE clientId = @clientId LIMIT 1; -- name: AddLatestCollectorVersion :one -INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id; +INSERT INTO collectorVersions (clientId) VALUES ($1) RETURNING id; -- name: SetActiveCollectorVersion :exec -INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2); +INSERT INTO collectorActiveVersions (clientId, versionId) VALUES ($1, $2); -- name: SetCollectorCleanVersion :exec -INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3); +INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3); -- name: SetCollectorTextVersion :exec -INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3); +INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3); -- name: AddCollectorQuery :exec -INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4); +INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4); -- name: RemoveCollectorQuery :exec -UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null; +UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null; diff --git a/database/queries/document.sql b/database/queries/document.sql index d81149e8..2921756a 100644 --- a/database/queries/document.sql +++ b/database/queries/document.sql @@ -1,9 +1,9 @@ -- name: GetDocument :one -SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1; +SELECT id, clientId, hash FROM documents WHERE id = $1 LIMIT 1; --- name: ListDocumentsByJobId :many +-- name: ListDocumentsByClientId :many WITH docs as ( - SELECT id from documents where jobId = @jobId + SELECT id from documents where clientId = @clientId ), entries as ( SELECT @@ -17,7 +17,7 @@ entries as ( SELECT documentId as id, bucket, key from entries WHERE rowNumber = 1; -- name: CreateDocument :one -INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id; +INSERT INTO documents (clientId, hash) VALUES ($1, $2) RETURNING id; -- name: AddDocumentEntry :exec INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3); @@ -26,4 +26,7 @@ INSERT INTO documentEntries (documentId, bucket, key) VALUES ($1, $2, $3); SELECT documentId, bucket, key FROM documentEntries WHERE documentId = $1 ORDER BY id DESC LIMIT 1; -- name: GetDocumentIDByHash :one -SELECT id FROM documents WHERE hash = $1 and jobId = $2; +SELECT id FROM documents WHERE hash = $1 and clientId = $2; + +-- name: ListDocumentIDsBatch :many +SELECT id, totalCount FROM listDocumentIDs(@clientId, @batchSize, @pageOffset); diff --git a/database/queries/job.sql b/database/queries/job.sql deleted file mode 100644 index f754c2b2..00000000 --- a/database/queries/job.sql +++ /dev/null @@ -1,101 +0,0 @@ --- name: GetJob :one -SELECT j.id, j.clientId, cs.canSync - FROM jobs as j - JOIN currentJobCanSync as cs on cs.jobId = j.id - WHERE j.id = $1; - --- name: CreateJob :one -INSERT INTO jobs (clientId) VALUES ($1) RETURNING id; - --- name: AddJobCanSync :exec -INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2); - --- name: ListJobDocumentIDsBatch :many -SELECT id, totalCount FROM listJobDocumentIDs(@jobId, @batchSize, @pageOffset); - --- name: IsJobSynced :one -WITH -docs AS ( - -- Get all documents for this job - SELECT id, jobId FROM documents WHERE jobId = $1 -), -doc_clean_entries as ( - -- Documents with their current clean entries - SELECT - d.id AS document_id, - d.jobId as job_id, - cte.id AS clean_entry_id - FROM - docs d - LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id - WHERE cte.id is null or cte.id is not null and cte.fail is null -), -doc_text_entries AS ( - -- Documents with their current text entries - SELECT - d.document_id, - cte.id AS text_entry_id - FROM - doc_clean_entries d - LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id -), -required_results AS ( - -- All required document-query-version combinations - SELECT - d.document_id, - cqdt.queryId, - cqdt.queryVersion, - dte.text_entry_id - FROM - doc_clean_entries d - JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.job_id - JOIN doc_text_entries dte ON dte.document_id = d.document_id - and dte.text_entry_id IS NOT NULL -), -existing_results AS ( - -- Valid results that exist - SELECT - rr.document_id AS document_id, - r.queryId, - r.id AS result_id - FROM - results r - JOIN required_results rr ON - r.queryId = rr.queryId AND - r.queryVersion = rr.queryVersion AND - r.textEntryId = rr.text_entry_id -), -missing_results AS ( - -- Find missing results - SELECT rr.queryId - FROM required_results rr - LEFT JOIN existing_results er on er.queryId = rr.queryId - WHERE er.result_id is null -), -dependency_check AS ( - -- Check for missing dependencies - SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId - FROM existing_results er - JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId - LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id - WHERE rd.resultId is null -) -SELECT ( - -- No documents means job is synced - NOT EXISTS (SELECT 1 FROM docs) - - OR - - -- Documents with no text entries - (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) - - and - - -- Documents with missing results - NOT EXISTS (SELECT 1 FROM missing_results) - - and - - -- Documents with missing dependencies - NOT EXISTS (SELECT 1 FROM dependency_check)) -)::bool as is_synced; diff --git a/database/queries/query.sql b/database/queries/query.sql index d9af8ea9..7eff4682 100644 --- a/database/queries/query.sql +++ b/database/queries/query.sql @@ -75,13 +75,13 @@ SELECT EXISTS ( -- name: ListQueryDirectDependentsByDocumentID :many WITH doc AS ( - SELECT id, jobId FROM documents where id = @documentId + SELECT id, clientId FROM documents where id = @documentId ) SELECT dt.queryId FROM doc as d JOIN collectorQueryDependencyTree as dt - on d.jobId = dt.jobId + on d.clientId = dt.clientId and @queryId = any(dt.requiredIds); --- name: ListQueryJobIDs :many -SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1; +-- name: ListQueryClientIDs :many +SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1; diff --git a/database/queries/result.sql b/database/queries/result.sql index 6f810eee..200343b0 100644 --- a/database/queries/result.sql +++ b/database/queries/result.sql @@ -8,7 +8,7 @@ WITH reqQueries as ( and isInVersion(@version, rq.addedVersion, rq.removedVersion) ), docs as ( - SELECT id, jobId + SELECT id, clientId FROM documents WHERE id = @documentId ), @@ -18,8 +18,8 @@ codeVersions as ( mcv.minCleanVersion, mtv.minTextVersion FROM docs as d - JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId - JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId + JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId + JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId ), latestVersions AS ( SELECT @@ -51,7 +51,7 @@ INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2); -- name: GetResultValueWithVersion :one WITH doc as ( - SELECT id, jobId + SELECT id, clientId FROM documents WHERE id = @documentId ) @@ -65,12 +65,12 @@ SELECT r.id, r.value -- name: ListUnsyncedNoDepsQueriesByDocId :many WITH docs as ( - SELECT id, jobId from documents where id = $1 + SELECT id, clientId from documents where id = $1 ), unsyncedQueries AS ( SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d - JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId + JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId JOIN currentTextEntries as cte on cte.documentId = d.id LEFT JOIN results as r on r.queryId = dt.queryId diff --git a/deployments/compose.local.yaml b/deployments/compose.local.yaml index 8697e719..23a978c7 100644 --- a/deployments/compose.local.yaml +++ b/deployments/compose.local.yaml @@ -199,7 +199,7 @@ services: - 8080 environment: LOG_LEVEL: DEBUG - JOB_SYNC_URL: ${JOB_SYNC_URL} + CLIENT_SYNC_URL: ${CLIENT_SYNC_URL} QUERY_VERSION_SYNC_URL: ${QUERY_VERSION_SYNC_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} @@ -216,9 +216,9 @@ services: networks: - server-network - job_sync_runner: + client_sync_runner: image: queryorchestration:latest - command: ["./jobSyncRunner"] + command: ["./clientSyncRunner"] depends_on: localstack: condition: service_healthy @@ -230,7 +230,7 @@ services: - 8080 environment: LOG_LEVEL: DEBUG - QUEUE_URL: ${JOB_SYNC_URL} + QUEUE_URL: ${CLIENT_SYNC_URL} DOCUMENT_SYNC_URL: ${DOCUMENT_SYNC_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} @@ -262,7 +262,7 @@ services: environment: LOG_LEVEL: DEBUG QUEUE_URL: ${QUERY_VERSION_SYNC_URL} - JOB_SYNC_URL: ${JOB_SYNC_URL} + CLIENT_SYNC_URL: ${CLIENT_SYNC_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} diff --git a/deployments/prometheus.allmetrics.yaml b/deployments/prometheus.allmetrics.yaml index 37424bc0..6a8eb6b5 100644 --- a/deployments/prometheus.allmetrics.yaml +++ b/deployments/prometheus.allmetrics.yaml @@ -4,50 +4,50 @@ global: scrape_interval: 10s scrape_configs: - - job_name: 'query_service' + - client_name: "query_service" static_configs: - targets: - - 'query_service:8080' - metrics_path: '/metrics' + - "query_service:8080" + metrics_path: "/metrics" - - job_name: 'doc_init_runner' + - client_name: "doc_init_runner" static_configs: - targets: - - 'doc_init_runner:8080' - metrics_path: '/metrics' + - "doc_init_runner:8080" + metrics_path: "/metrics" - - job_name: 'doc_clean_runner' + - client_name: "doc_clean_runner" static_configs: - targets: - - 'doc_clean_runner:8080' - metrics_path: '/metrics' + - "doc_clean_runner:8080" + metrics_path: "/metrics" - - job_name: 'doc_text_runner' + - client_name: "doc_text_runner" static_configs: - targets: - - 'doc_text_runner:8080' - metrics_path: '/metrics' + - "doc_text_runner:8080" + metrics_path: "/metrics" - - job_name: 'query_sync_runner' + - client_name: "query_sync_runner" static_configs: - targets: - - 'query_sync_runner:8080' - metrics_path: '/metrics' + - "query_sync_runner:8080" + metrics_path: "/metrics" - - job_name: 'query_runner' + - client_name: "query_runner" static_configs: - targets: - - 'query_runner:8080' - metrics_path: '/metrics' + - "query_runner:8080" + metrics_path: "/metrics" - - job_name: 'job_sync_runner' + - client_name: "client_sync_runner" static_configs: - targets: - - 'job_sync_runner:8080' - metrics_path: '/metrics' + - "client_sync_runner:8080" + metrics_path: "/metrics" - - job_name: 'query_version_sync_runner' + - client_name: "query_version_sync_runner" static_configs: - targets: - - 'query_version_sync_runner:8080' - metrics_path: '/metrics' + - "query_version_sync_runner:8080" + metrics_path: "/metrics" diff --git a/deployments/prometheus.localservice.yaml b/deployments/prometheus.localservice.yaml index 06dc9cb7..6ae42671 100644 --- a/deployments/prometheus.localservice.yaml +++ b/deployments/prometheus.localservice.yaml @@ -3,8 +3,8 @@ global: scrape_interval: 10s scrape_configs: - - job_name: 'api' + - client_name: "api" static_configs: - targets: - - 'host.docker.internal:8080' - metrics_path: '/metrics' + - "host.docker.internal:8080" + metrics_path: "/metrics" diff --git a/devbox.json b/devbox.json index 7e60e043..b9141d42 100644 --- a/devbox.json +++ b/devbox.json @@ -8,23 +8,23 @@ "AWS_SECRET_ACCESS_KEY": "test", "AWS_SESSION_TOKEN": "", "BUCKET_IN": "documentin", + "CLIENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/client_sync", "DB_NOSSL": "true", "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", "DOCUMENT_INIT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_init", "DOCUMENT_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_sync", "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", - "JOB_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/job_sync", "PGDATABASE": "query_orchestration", "PGHOST": "localhost", "PGPASSWORD": "pass", "PGPORT": "5432", "PGPORT_GENERATE": "5431", "PGUSER": "postgres", + "QNAME_CLIENT_SYNC": "client_sync", "QNAME_DOCUMENT_CLEAN": "document_clean", "QNAME_DOCUMENT_INIT": "document_init", "QNAME_DOCUMENT_SYNC": "document_sync", "QNAME_DOCUMENT_TEXT": "document_text", - "QNAME_JOB_SYNC": "job_sync", "QNAME_QUERY_RUNNER": "query_runner", "QNAME_QUERY_SYNC": "query_sync", "QNAME_QUERY_VERSION_SYNC": "query_version_sync", @@ -59,6 +59,9 @@ "export DB_URI=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${PGDATABASE}?sslmode=disable", "export DB_URI_GENERATE=postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT_GENERATE}/${PGDATABASE}?sslmode=disable", "echo 'Welcome to the DoczyAI devbox!'" - ] + ], + "scripts": { + "precommit": "task precommit" + } } } \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index fa2c5bcd..6fc81f17 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,12 +1,20 @@ # Query Orchestration +## clientSyncRunner + +**Listens For**: Body Containing Client ID + +**Action**. + +Adds an event for each document into the DOCSYNC queue for the given Client. + ## docCleanRunner **Listens For**: Body Containing Document ID. **Action**. -If the document is not already clean according to the job collector standards, a clean is triggered. +If the document is not already clean according to the collector standards, a clean is triggered. An event with the document id is pushed to the DOCTEXT queue. @@ -18,7 +26,7 @@ An event with the document id is pushed to the DOCTEXT queue. Checks if the uploaded document is a duplicate. -Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the job. +Currently whether the document content is a duplicate or not is determines using the document ETag, and check if it already exists for the client. Find reference entity. @@ -36,7 +44,7 @@ Add an event to the DOC SYNC queue. Checks if the document is eligible for syncing. -By validating: the client can_sync parameter, the job can_sync parameter. +By validating: the client can_sync parameter. If eligible, send an event to the DOCCLEAN queue. @@ -46,18 +54,10 @@ If eligible, send an event to the DOCCLEAN queue. **Action**. -If the document text is not already extracted according to the job collector standards, the text is extracted. +If the document text is not already extracted according to the collector standards, the text is extracted. An event with the document id is pushed to the QUERYSYNC queue. -## jobSyncRunner - -**Listens For**: Body Containing Job ID - -**Action**. - -Adds an event for each document into the DOCSYNC queue for the given Job. - ## metricsExample\_test This main.go serves as an example of how to use the custom Prometheus metrics package. @@ -90,17 +90,15 @@ This API currently manages all user entities to be managed. This allows users of The entities in question: -- Client - A client is a single client of the Doczy project, all configurations and jobs for a client will be in reference to the respective client. +- Client - A client is a single client of the Doczy project, all configurations and clients for a client will be in reference to the respective client. -- Job - A job is a single instance of requirements for a client. This maintains a set of queries and processes in relation to itself. +- Collector - A collector specifies the required queries for export for a given client. -- Collector - A collector specifies the required queries for export for a given job. - -- Sample - A sample is a subset of documents to be used for testing. When it is active a job will only process documents in the sample. +- Sample - A sample is a subset of documents to be used for testing. When it is active a client will only process documents in the sample. - Query - A query is a block of processing of data which takes an input and produces an output. -- Export - An export is a manually started process which outputs the results for a job to a given location. +- Export - An export is a manually started process which outputs the results for a client to a given location. ## querySyncRunner @@ -126,6 +124,6 @@ An event with the document id is pushed to the DOCTEXT queue. **Action**. -Adds an event for each job into the JOBSYNC queue for the given query, i.e. each job that contains the query in its dependency tree. +Adds an event for each client into the CLIENTSYNC queue for the given query, i.e. each client that contains the query in its dependency tree. Generated by Doczy diff --git a/internal/client/create.go b/internal/client/create.go index 172210e6..1a2efdfc 100644 --- a/internal/client/create.go +++ b/internal/client/create.go @@ -3,6 +3,7 @@ package client import ( "context" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "github.com/google/uuid" @@ -19,5 +20,12 @@ func (s *Service) Create(ctx context.Context, name string) (uuid.UUID, error) { return uuid.Nil, err } + err = s.svc.Collector.Create(ctx, &collector.CreateParams{ + ClientID: database.MustToUUID(id), + }) + if err != nil { + return uuid.Nil, err + } + return database.MustToUUID(id), nil } diff --git a/internal/client/create_test.go b/internal/client/create_test.go index 5bd0f4d9..87817f97 100644 --- a/internal/client/create_test.go +++ b/internal/client/create_test.go @@ -5,11 +5,15 @@ import ( "testing" "queryorchestration/internal/client" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + cleanversion "queryorchestration/internal/document/clean/version" + textversion "queryorchestration/internal/document/text/version" "queryorchestration/internal/serviceconfig" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,7 +28,12 @@ func TestCreate(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := client.New(cfg) + svc := client.New(cfg, &client.Services{ + Collector: collector.New(cfg, &collector.Services{ + CleanVersion: cleanversion.New(cfg), + TextVersion: textversion.New(cfg), + }), + }) name := "client_name" aid := uuid.New() @@ -34,6 +43,14 @@ func TestCreate(t *testing.T) { pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(aid)), ) + pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(aid)).WillReturnRows( + pgxmock.NewRows([]string{"version"}). + AddRow(int32(1)), + ) + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(aid), int32(1)). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectCommit() id, err := svc.Create(ctx, name) assert.NoError(t, err) diff --git a/internal/client/get_test.go b/internal/client/get_test.go index bc95c8b5..1447e1c8 100644 --- a/internal/client/get_test.go +++ b/internal/client/get_test.go @@ -25,7 +25,7 @@ func TestGet(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := client.New(cfg) + svc := client.New(cfg, &client.Services{}) id := uuid.New() diff --git a/internal/client/service.go b/internal/client/service.go index d80e4f46..4a31ed3a 100644 --- a/internal/client/service.go +++ b/internal/client/service.go @@ -6,6 +6,7 @@ import ( "regexp" "strings" + "queryorchestration/internal/collector" "queryorchestration/internal/serviceconfig" "github.com/google/uuid" @@ -70,12 +71,18 @@ func normalizeName(name *string) error { return nil } -type Service struct { - cfg serviceconfig.ConfigProvider +type Services struct { + Collector *collector.Service } -func New(cfg serviceconfig.ConfigProvider) *Service { +type Service struct { + cfg serviceconfig.ConfigProvider + svc *Services +} + +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { return &Service{ cfg, + svc, } } diff --git a/internal/client/service_test.go b/internal/client/service_test.go index b5d049f2..ef421332 100644 --- a/internal/client/service_test.go +++ b/internal/client/service_test.go @@ -19,6 +19,6 @@ func TestService(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := client.New(cfg) + svc := client.New(cfg, &client.Services{}) assert.NotNil(t, svc) } diff --git a/internal/client/status.go b/internal/client/status.go new file mode 100644 index 00000000..81a8f902 --- /dev/null +++ b/internal/client/status.go @@ -0,0 +1,35 @@ +package client + +import ( + "context" + + "queryorchestration/internal/database" + + "github.com/google/uuid" +) + +type Status string + +const ( + NOT_SYNCED Status = "not_synced" + NOT_SYNCING Status = "not_syncing" + IN_SYNC Status = "in_sync" +) + +func (s *Service) GetStatus(ctx context.Context, id uuid.UUID) (Status, error) { + client, err := s.cfg.GetDBQueries().GetClient(ctx, database.MustToDBUUID(id)) + if err != nil { + return NOT_SYNCING, err + } else if !client.Cansync { + return NOT_SYNCING, nil + } + + issynced, err := s.cfg.GetDBQueries().IsClientSynced(ctx, database.MustToDBUUID(id)) + if err != nil { + return NOT_SYNCING, err + } else if issynced { + return IN_SYNC, nil + } + + return NOT_SYNCED, nil +} diff --git a/internal/client/status_test.go b/internal/client/status_test.go new file mode 100644 index 00000000..ee7e4895 --- /dev/null +++ b/internal/client/status_test.go @@ -0,0 +1,78 @@ +package client_test + +import ( + "context" + "errors" + "testing" + + "queryorchestration/internal/client" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/serviceconfig" + + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetStatus(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := client.New(cfg, &client.Services{}) + + clientId := uuid.New() + + t.Run("is synced", func(t *testing.T) { + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"id", "name", "can_sync"}). + AddRow(database.MustToDBUUID(clientId), "name", true), + ) + pool.ExpectQuery("name: IsClientSynced :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(true), + ) + + issynced, err := svc.GetStatus(ctx, clientId) + assert.NoError(t, err) + assert.Equal(t, client.IN_SYNC, issynced) + }) + t.Run("not synced", func(t *testing.T) { + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"id", "name", "can_sync"}). + AddRow(database.MustToDBUUID(clientId), "name", true), + ) + pool.ExpectQuery("name: IsClientSynced :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"issynced"}). + AddRow(false), + ) + + issynced, err := svc.GetStatus(ctx, clientId) + assert.NoError(t, err) + assert.Equal(t, client.NOT_SYNCED, issynced) + }) + t.Run("not syncing", func(t *testing.T) { + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( + pgxmock.NewRows([]string{"id", "name", "can_sync"}). + AddRow(database.MustToDBUUID(clientId), "name", false), + ) + + issynced, err := svc.GetStatus(ctx, clientId) + assert.NoError(t, err) + assert.Equal(t, client.NOT_SYNCING, issynced) + }) + t.Run("db error", func(t *testing.T) { + errStr := "db fail" + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)). + WillReturnError(errors.New(errStr)) + + _, err = svc.GetStatus(ctx, clientId) + assert.EqualError(t, err, errStr) + }) +} diff --git a/internal/job/sync/service.go b/internal/client/sync/service.go similarity index 95% rename from internal/job/sync/service.go rename to internal/client/sync/service.go index e1be4b0d..76a1d5c4 100644 --- a/internal/job/sync/service.go +++ b/internal/client/sync/service.go @@ -1,4 +1,4 @@ -package jobsync +package clientsync import ( "queryorchestration/internal/serviceconfig" diff --git a/internal/job/sync/service_test.go b/internal/client/sync/service_test.go similarity index 90% rename from internal/job/sync/service_test.go rename to internal/client/sync/service_test.go index b85da688..6943132c 100644 --- a/internal/job/sync/service_test.go +++ b/internal/client/sync/service_test.go @@ -1,4 +1,4 @@ -package jobsync +package clientsync import ( "testing" diff --git a/internal/job/sync/sync.go b/internal/client/sync/sync.go similarity index 88% rename from internal/job/sync/sync.go rename to internal/client/sync/sync.go index 8f1dd63c..448f9c0e 100644 --- a/internal/job/sync/sync.go +++ b/internal/client/sync/sync.go @@ -1,4 +1,4 @@ -package jobsync +package clientsync import ( "context" @@ -20,10 +20,10 @@ func (s *Service) Sync(ctx context.Context, id uuid.UUID) error { dbid := database.MustToDBUUID(id) for hasMore { - ids, err := s.cfg.GetDBQueries().ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ + ids, err := s.cfg.GetDBQueries().ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ Batchsize: s.batchSize, Pageoffset: offset, - Jobid: dbid, + Clientid: dbid, }) if errors.Is(err, sql.ErrNoRows) { return nil diff --git a/internal/job/sync/sync_test.go b/internal/client/sync/sync_test.go similarity index 77% rename from internal/job/sync/sync_test.go rename to internal/client/sync/sync_test.go index f324c437..c1c3a7fa 100644 --- a/internal/job/sync/sync_test.go +++ b/internal/client/sync/sync_test.go @@ -1,4 +1,4 @@ -package jobsync +package clientsync import ( "context" @@ -20,7 +20,7 @@ import ( "github.com/stretchr/testify/require" ) -type JobSyncConfig struct { +type ClientSyncConfig struct { serviceconfig.BaseConfig documentsync.DocSyncConfig } @@ -32,7 +32,7 @@ func TestTrigger(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) - cfg := &JobSyncConfig{} + cfg := &ClientSyncConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) @@ -41,21 +41,21 @@ func TestTrigger(t *testing.T) { svc := New(cfg) - jobId := uuid.New() + clientId := uuid.New() docs := []document.Document{ { - ID: uuid.New(), - JobID: jobId, + ID: uuid.New(), + ClientID: clientId, }, { - ID: uuid.New(), - JobID: jobId, + ID: uuid.New(), + ClientID: clientId, }, } svc.batchSize = 1 total := int64(2) - pool.ExpectQuery("name: ListJobDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(jobId), svc.batchSize, int32(0)). + pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(clientId), svc.batchSize, int32(0)). WillReturnRows( pgxmock.NewRows([]string{"id", "totalCount"}). AddRow(database.MustToDBUUID(docs[0].ID), &total), @@ -69,7 +69,7 @@ func TestTrigger(t *testing.T) { mock.Anything, ). Return(&sqs.SendMessageOutput{}, nil) - pool.ExpectQuery("name: ListJobDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(jobId), svc.batchSize, int32(1)). + pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(clientId), svc.batchSize, int32(1)). WillReturnRows( pgxmock.NewRows([]string{"id", "totalCount"}). AddRow(database.MustToDBUUID(docs[1].ID), &total), @@ -84,7 +84,7 @@ func TestTrigger(t *testing.T) { ). Return(&sqs.SendMessageOutput{}, nil) - err = svc.Sync(ctx, jobId) + err = svc.Sync(ctx, clientId) assert.NoError(t, err) }) t.Run("empty batch", func(t *testing.T) { @@ -93,7 +93,7 @@ func TestTrigger(t *testing.T) { pool, err := pgxmock.NewPool() require.NoError(t, err) - cfg := &JobSyncConfig{} + cfg := &ClientSyncConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) @@ -102,15 +102,15 @@ func TestTrigger(t *testing.T) { svc := New(cfg) - jobId := uuid.New() + clientId := uuid.New() svc.batchSize = 1 - pool.ExpectQuery("name: ListJobDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(jobId), svc.batchSize, int32(0)). + pool.ExpectQuery("name: ListDocumentIDsBatch :many").WithArgs(database.MustToDBUUID(clientId), svc.batchSize, int32(0)). WillReturnRows( pgxmock.NewRows([]string{"id", "totalCount"}), ) - err = svc.Sync(ctx, jobId) + err = svc.Sync(ctx, clientId) assert.NoError(t, err) }) } diff --git a/internal/client/update_test.go b/internal/client/update_test.go index cbe89163..577c4e81 100644 --- a/internal/client/update_test.go +++ b/internal/client/update_test.go @@ -23,7 +23,7 @@ func TestUpdate(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg) + svc := New(cfg, &Services{}) c := Client{ ID: uuid.New(), @@ -120,7 +120,7 @@ func TestSubmitUpdate(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg) + svc := New(cfg, &Services{}) c := Client{ ID: uuid.New(), diff --git a/internal/job/collector/create.go b/internal/collector/create.go similarity index 90% rename from internal/job/collector/create.go rename to internal/collector/create.go index dc6b0ba6..ccf89320 100644 --- a/internal/job/collector/create.go +++ b/internal/collector/create.go @@ -13,7 +13,7 @@ import ( ) type CreateParams struct { - JobID uuid.UUID + ClientID uuid.UUID MinCleanVersion *int32 MinTextVersion *int32 Fields *map[string]uuid.UUID @@ -29,7 +29,7 @@ func (s *Service) Create(ctx context.Context, params *CreateParams) error { } type dbCreateParams struct { - JobID pgtype.UUID + ClientID pgtype.UUID MinCleanVersion *int32 MinTextVersion *int32 Fields *map[string]pgtype.UUID @@ -58,7 +58,7 @@ func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*d } return &dbCreateParams{ - JobID: database.MustToDBUUID(params.JobID), + ClientID: database.MustToDBUUID(params.ClientID), MinCleanVersion: minClean, MinTextVersion: minText, Fields: fields, @@ -100,13 +100,13 @@ func (s *Service) NormalizeFieldsToDB(ctx context.Context, ofields *map[string]u func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) error { return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { - version, err := qtx.AddLatestCollectorVersion(ctx, params.JobID) + version, err := qtx.AddLatestCollectorVersion(ctx, params.ClientID) if err != nil { return err } err = qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: version, }) if err != nil { @@ -115,7 +115,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) erro if params.MinCleanVersion != nil { err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: *params.MinCleanVersion, Addedversion: version, }) @@ -126,7 +126,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) erro if params.MinTextVersion != nil { err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: *params.MinTextVersion, Addedversion: version, }) @@ -138,7 +138,7 @@ func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) erro if params.Fields != nil { for key, field := range *params.Fields { err = qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Name: key, Queryid: field, Addedversion: version, diff --git a/internal/job/collector/create_test.go b/internal/collector/create_test.go similarity index 79% rename from internal/job/collector/create_test.go rename to internal/collector/create_test.go index 4de9aee3..90bc8cb4 100644 --- a/internal/job/collector/create_test.go +++ b/internal/collector/create_test.go @@ -4,9 +4,9 @@ import ( "context" "testing" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig" "github.com/stretchr/testify/require" @@ -31,7 +31,7 @@ func TestCreate(t *testing.T) { minCleanV := int32(1) minTextV := int32(1) create := collector.CreateParams{ - JobID: uuid.New(), + ClientID: uuid.New(), MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, Fields: &map[string]uuid.UUID{ @@ -44,17 +44,17 @@ func TestCreate(t *testing.T) { AddRow(true), ) pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(create.JobID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(create.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(create.ClientID), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(create.ClientID), int32(1), *create.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(create.JobID), int32(1), *create.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(create.ClientID), int32(1), *create.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(create.JobID), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(create.ClientID), "example_key", database.MustToDBUUID((*create.Fields)["example_key"]), int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/internal/job/collector/createprivate_test.go b/internal/collector/createprivate_test.go similarity index 90% rename from internal/job/collector/createprivate_test.go rename to internal/collector/createprivate_test.go index 852a0678..416d70f1 100644 --- a/internal/job/collector/createprivate_test.go +++ b/internal/collector/createprivate_test.go @@ -33,7 +33,7 @@ func TestGetCreateParams(t *testing.T) { minCleanV := int32(1) minTextV := int32(1) params := CreateParams{ - JobID: uuid.New(), + ClientID: uuid.New(), MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, Fields: &map[string]uuid.UUID{ @@ -49,7 +49,7 @@ func TestGetCreateParams(t *testing.T) { dbparams, err := svc.getCreateParams(ctx, ¶ms) assert.NoError(t, err) assert.EqualExportedValues(t, &dbCreateParams{ - JobID: database.MustToDBUUID(params.JobID), + ClientID: database.MustToDBUUID(params.ClientID), MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, Fields: &map[string]pgtype.UUID{ @@ -80,7 +80,7 @@ func TestSubmitCreate(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) params := dbCreateParams{ - JobID: database.MustToDBUUID(uuid.New()), + ClientID: database.MustToDBUUID(uuid.New()), MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, Fields: &map[string]pgtype.UUID{ @@ -89,17 +89,17 @@ func TestSubmitCreate(t *testing.T) { } pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(params.JobID).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(params.ClientID).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(int32(1)), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(params.JobID, int32(1)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(params.ClientID, int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(params.JobID, int32(1), *params.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(params.ClientID, int32(1), *params.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(params.JobID, int32(1), *params.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(params.ClientID, int32(1), *params.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(params.JobID, "example_key", (*params.Fields)["example_key"], int32(1)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(params.ClientID, "example_key", (*params.Fields)["example_key"], int32(1)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/internal/job/collector/get.go b/internal/collector/get.go similarity index 64% rename from internal/job/collector/get.go rename to internal/collector/get.go index 0cc72768..62a03b5d 100644 --- a/internal/job/collector/get.go +++ b/internal/collector/get.go @@ -9,17 +9,8 @@ import ( "github.com/google/uuid" ) -func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Collector, error) { - dbColl, err := s.cfg.GetDBQueries().GetCollector(ctx, database.MustToDBUUID(id)) - if err != nil { - return nil, err - } - - return parseDBCollector(dbColl) -} - -func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, error) { - dbColl, err := s.cfg.GetDBQueries().GetCollectorByJobID(ctx, database.MustToDBUUID(jobID)) +func (s *Service) GetByClientID(ctx context.Context, clientID uuid.UUID) (*Collector, error) { + dbColl, err := s.cfg.GetDBQueries().GetCollectorByClientID(ctx, database.MustToDBUUID(clientID)) if err != nil { return nil, err } diff --git a/internal/collector/get_test.go b/internal/collector/get_test.go new file mode 100644 index 00000000..a5af588f --- /dev/null +++ b/internal/collector/get_test.go @@ -0,0 +1,82 @@ +package collector_test + +import ( + "context" + "fmt" + "testing" + + "queryorchestration/internal/collector" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" + + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetByClientID(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := collector.New(cfg, &collector.Services{}) + + minCleanV := int32(2) + minTextV := int32(4) + ogc := collector.Collector{ + ClientID: uuid.New(), + MinCleanVersion: minCleanV, + MinTextVersion: minTextV, + Fields: map[string]uuid.UUID{ + "example_key": uuid.New(), + }, + } + + pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(ogc.ClientID)). + WillReturnRows( + pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(ogc.ClientID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), + ) + + coll, err := svc.GetByClientID(ctx, ogc.ClientID) + assert.NoError(t, err) + assert.EqualExportedValues(t, ogc, *coll) +} + +func TestListQueries(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + require.NoError(t, err) + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := collector.New(cfg, &collector.Services{}) + + clientId := uuid.New() + ogc := []*resultprocessor.Query{ + { + ID: uuid.New(), + Type: resultprocessor.TypeContextFull, + Version: 2, + }, + } + + pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(clientId)). + WillReturnRows( + pgxmock.NewRows([]string{"clientId", "queryId", "type", "queryVersion", "requiredIds"}). + AddRow(database.MustToDBUUID(clientId), database.MustToDBUUID(ogc[0].ID), repository.QuerytypeContextFull, int32(2), nil), + ) + + qs, err := svc.ListQueries(ctx, clientId) + assert.NoError(t, err) + assert.EqualExportedValues(t, ogc, qs) +} diff --git a/internal/job/collector/parse.go b/internal/collector/parse.go similarity index 93% rename from internal/job/collector/parse.go rename to internal/collector/parse.go index b01fe2c7..3e280b5b 100644 --- a/internal/job/collector/parse.go +++ b/internal/collector/parse.go @@ -27,7 +27,7 @@ func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) { } return &Collector{ - JobID: database.MustToUUID(c.Jobid), + ClientID: database.MustToUUID(c.Clientid), MinCleanVersion: c.Mincleanversion, MinTextVersion: c.Mintextversion, ActiveVersion: c.Activeversion, diff --git a/internal/job/collector/parse_test.go b/internal/collector/parse_test.go similarity index 81% rename from internal/job/collector/parse_test.go rename to internal/collector/parse_test.go index 066f65b5..8e36f5d0 100644 --- a/internal/job/collector/parse_test.go +++ b/internal/collector/parse_test.go @@ -19,7 +19,7 @@ func TestParseDBCollector(t *testing.T) { minCleanV := int32(1) minTextV := int32(2) ogc := Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), MinCleanVersion: minCleanV, MinTextVersion: minTextV, Fields: map[string]uuid.UUID{ @@ -27,7 +27,7 @@ func TestParseDBCollector(t *testing.T) { }, } c, err = parseDBCollector(&repository.Fullactivecollector{ - Jobid: database.MustToDBUUID(ogc.JobID), + Clientid: database.MustToDBUUID(ogc.ClientID), Mincleanversion: ogc.MinCleanVersion, Mintextversion: ogc.MinTextVersion, Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])), @@ -38,8 +38,8 @@ func TestParseDBCollector(t *testing.T) { ogc.MinCleanVersion = 0 ogc.MinTextVersion = 0 c, err = parseDBCollector(&repository.Fullactivecollector{ - Jobid: database.MustToDBUUID(ogc.JobID), - Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])), + Clientid: database.MustToDBUUID(ogc.ClientID), + Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])), }) assert.NoError(t, err) assert.EqualExportedValues(t, ogc, *c) diff --git a/internal/job/collector/service.go b/internal/collector/service.go similarity index 96% rename from internal/job/collector/service.go rename to internal/collector/service.go index 0f665833..734b18d9 100644 --- a/internal/job/collector/service.go +++ b/internal/collector/service.go @@ -9,7 +9,7 @@ import ( ) type Collector struct { - JobID uuid.UUID + ClientID uuid.UUID MinCleanVersion int32 MinTextVersion int32 ActiveVersion int32 diff --git a/internal/job/collector/service_test.go b/internal/collector/service_test.go similarity index 91% rename from internal/job/collector/service_test.go rename to internal/collector/service_test.go index 5072e7af..0ccc459c 100644 --- a/internal/job/collector/service_test.go +++ b/internal/collector/service_test.go @@ -3,8 +3,8 @@ package collector_test import ( "testing" + "queryorchestration/internal/collector" "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig" "github.com/pashagolub/pgxmock/v3" diff --git a/internal/job/collector/update/service.go b/internal/collector/update/service.go similarity index 80% rename from internal/job/collector/update/service.go rename to internal/collector/update/service.go index 36e056e4..dcb8a683 100644 --- a/internal/job/collector/update/service.go +++ b/internal/collector/update/service.go @@ -1,11 +1,11 @@ package collectorupdate import ( + collector "queryorchestration/internal/collector" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - collector "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" ) type Services struct { @@ -16,7 +16,7 @@ type Services struct { type ConfigProvider interface { serviceconfig.ConfigProvider - jobsync.ConfigProvider + clientsync.ConfigProvider } type Service struct { diff --git a/internal/job/collector/update/service_test.go b/internal/collector/update/service_test.go similarity index 92% rename from internal/job/collector/update/service_test.go rename to internal/collector/update/service_test.go index b9e1047b..686f5f71 100644 --- a/internal/job/collector/update/service_test.go +++ b/internal/collector/update/service_test.go @@ -3,8 +3,8 @@ package collectorupdate_test import ( "testing" + "queryorchestration/internal/collector" "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig" "github.com/pashagolub/pgxmock/v3" diff --git a/internal/job/collector/update/update.go b/internal/collector/update/update.go similarity index 90% rename from internal/job/collector/update/update.go rename to internal/collector/update/update.go index 4ede8c85..39ca870a 100644 --- a/internal/job/collector/update/update.go +++ b/internal/collector/update/update.go @@ -5,10 +5,10 @@ import ( "errors" "log/slog" - jobsyncrunner "queryorchestration/api/jobSyncRunner" + clientsyncrunner "queryorchestration/api/clientSyncRunner" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig/queue" "queryorchestration/internal/validation" @@ -17,15 +17,15 @@ import ( ) type UpdateParams struct { - JobID uuid.UUID + ClientID uuid.UUID ActiveVersion *int32 MinCleanVersion *int32 MinTextVersion *int32 Fields *map[string]uuid.UUID } -func (s *Service) UpdateByJobId(ctx context.Context, params *UpdateParams) error { - current, err := s.svc.Collector.GetByJobID(ctx, params.JobID) +func (s *Service) UpdateByClientId(ctx context.Context, params *UpdateParams) error { + current, err := s.svc.Collector.GetByClientID(ctx, params.ClientID) if err != nil { return err } @@ -54,15 +54,15 @@ func (s *Service) informUpdate(ctx context.Context, update *dbUpdateParams) erro } return s.cfg.SendToQueue(ctx, &queue.SendParams{ - QueueURL: s.cfg.GetJobSyncURL(), - Body: jobsyncrunner.Body{ - ID: database.MustToUUID(update.JobID), + QueueURL: s.cfg.GetClientSyncURL(), + Body: clientsyncrunner.Body{ + ID: database.MustToUUID(update.ClientID), }, }) } type dbUpdateParams struct { - JobID pgtype.UUID + ClientID pgtype.UUID ActiveVersion *int32 MinCleanVersion *int32 MinTextVersion *int32 @@ -101,7 +101,7 @@ func (s *Service) getUpdateParams(ctx context.Context, current *collector.Collec } return &dbUpdateParams{ - JobID: database.MustToDBUUID(params.JobID), + ClientID: database.MustToDBUUID(params.ClientID), ActiveVersion: params.ActiveVersion, MinCleanVersion: params.MinCleanVersion, MinTextVersion: params.MinTextVersion, @@ -192,7 +192,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector var latestVersion int32 if !onlyactive { - latestVersion, err = qtx.AddLatestCollectorVersion(ctx, params.JobID) + latestVersion, err = qtx.AddLatestCollectorVersion(ctx, params.ClientID) if err != nil { return err } @@ -200,7 +200,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.ActiveVersion != nil { err := qtx.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: *params.ActiveVersion, }) if err != nil { @@ -210,7 +210,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.MinCleanVersion != nil { err = qtx.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: *params.MinCleanVersion, Addedversion: latestVersion, }) @@ -221,7 +221,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector if params.MinTextVersion != nil { err = qtx.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Versionid: *params.MinTextVersion, Addedversion: latestVersion, }) @@ -234,7 +234,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector removeIDs := getRemoveFields(current.Fields, params.Fields) for _, field := range removeIDs { err := qtx.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Queryid: field, Removedversion: &latestVersion, }) @@ -246,7 +246,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector addIDs := getAddFields(current.Fields, params.Fields) for key, field := range addIDs { err := qtx.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: params.JobID, + Clientid: params.ClientID, Name: key, Queryid: field, Addedversion: latestVersion, @@ -257,7 +257,7 @@ func (s *Service) submitUpdate(ctx context.Context, current *collector.Collector } } - slog.Debug("job collector updated", "update", *params) + slog.Debug("client collector updated", "update", *params) return nil }) diff --git a/internal/job/collector/update/update_test.go b/internal/collector/update/update_test.go similarity index 63% rename from internal/job/collector/update/update_test.go rename to internal/collector/update/update_test.go index e8648796..c3aa5f23 100644 --- a/internal/job/collector/update/update_test.go +++ b/internal/collector/update/update_test.go @@ -5,12 +5,12 @@ import ( "fmt" "testing" + "queryorchestration/internal/collector" + collectorupdate "queryorchestration/internal/collector/update" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - collectorupdate "queryorchestration/internal/job/collector/update" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -24,7 +24,7 @@ import ( type CollectorUpdateConfig struct { serviceconfig.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig } func TestUpdate(t *testing.T) { @@ -37,39 +37,39 @@ func TestUpdate(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - cfg.JobSyncURL = "here" + cfg.ClientSyncURL = "here" svc := collectorupdate.New(cfg, &collectorupdate.Services{ Collector: collector.New(cfg, &collector.Services{}), }) current := collector.Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), ActiveVersion: 1, LatestVersion: 4, } av := int32(2) mv := int32(1) update := collectorupdate.UpdateParams{ - JobID: current.JobID, + ClientID: current.ClientID, ActiveVersion: &av, MinCleanVersion: &mv, } - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(update.JobID)). + pool.ExpectQuery("name: GetCollectorByClientID :one").WithArgs(database.MustToDBUUID(update.ClientID)). WillReturnRows( - pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + pgxmock.NewRows([]string{"clientId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(current.ClientID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectBeginTx(pgx.TxOptions{}) rv := current.LatestVersion + 1 - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(rv), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *update.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *update.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -77,12 +77,12 @@ func TestUpdate(t *testing.T) { SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.JobID.String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID.String()) }), mock.Anything, ). Return(&sqs.SendMessageOutput{}, nil) - err = svc.UpdateByJobId(ctx, &update) + err = svc.UpdateByClientId(ctx, &update) assert.NoError(t, err) } diff --git a/internal/job/collector/update/updateprivate_test.go b/internal/collector/update/updateprivate_test.go similarity index 92% rename from internal/job/collector/update/updateprivate_test.go rename to internal/collector/update/updateprivate_test.go index b5955195..327ea5cc 100644 --- a/internal/job/collector/update/updateprivate_test.go +++ b/internal/collector/update/updateprivate_test.go @@ -5,13 +5,13 @@ import ( "fmt" "testing" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job/collector" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -26,7 +26,7 @@ import ( type CollectorUpdateConfig struct { serviceconfig.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig } func TestGetUpdateParams(t *testing.T) { @@ -54,7 +54,7 @@ func TestGetUpdateParams(t *testing.T) { LatestVersion: 10, } params := UpdateParams{ - JobID: uuid.New(), + ClientID: uuid.New(), ActiveVersion: &aV, MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, @@ -71,7 +71,7 @@ func TestGetUpdateParams(t *testing.T) { dbparams, err := svc.getUpdateParams(ctx, ¤t, ¶ms) assert.NoError(t, err) assert.EqualExportedValues(t, &dbUpdateParams{ - JobID: database.MustToDBUUID(params.JobID), + ClientID: database.MustToDBUUID(params.ClientID), ActiveVersion: &aV, MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, @@ -102,7 +102,7 @@ func TestGetUpdateParams(t *testing.T) { current := collector.Collector{} params := UpdateParams{ - JobID: current.JobID, + ClientID: current.ClientID, } _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) assert.EqualError(t, err, "no changes") @@ -128,7 +128,7 @@ func TestGetUpdateParams(t *testing.T) { } version := int32(5) params := UpdateParams{ - JobID: current.JobID, + ClientID: current.ClientID, ActiveVersion: &version, } _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) @@ -155,7 +155,7 @@ func TestGetUpdateParams(t *testing.T) { }, } params := UpdateParams{ - JobID: current.JobID, + ClientID: current.ClientID, Fields: &map[string]uuid.UUID{ "example": current.Fields["example"], }, @@ -184,7 +184,7 @@ func TestSubmitUpdate(t *testing.T) { minCleanV := int32(2) minTextV := int32(4) current := collector.Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), ActiveVersion: 1, LatestVersion: 1, MinCleanVersion: 1, @@ -196,7 +196,7 @@ func TestSubmitUpdate(t *testing.T) { } aV := int32(2) params := dbUpdateParams{ - JobID: database.MustToDBUUID(current.JobID), + ClientID: database.MustToDBUUID(current.ClientID), ActiveVersion: &aV, MinCleanVersion: &minCleanV, MinTextVersion: &minTextV, @@ -210,24 +210,24 @@ func TestSubmitUpdate(t *testing.T) { pool.ExpectBeginTx(pgx.TxOptions{}) rv := int32(2) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.JobID)).WillReturnRows( + pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(current.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"version"}). AddRow(rv), ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinCleanVersion). + pool.ExpectExec("name: SetCollectorCleanVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *params.MinCleanVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), rv, *params.MinTextVersion). + pool.ExpectExec("name: SetCollectorTextVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), rv, *params.MinTextVersion). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.JobID)). + pool.ExpectExec("name: RemoveCollectorQuery :exec").WithArgs(&rv, database.MustToDBUUID(current.Fields["original_key"]), database.MustToDBUUID(current.ClientID)). WillReturnResult(pgxmock.NewResult("", 1)) pool.MatchExpectationsInOrder(false) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "example_key", (*params.Fields)["example_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "example_key", (*params.Fields)["example_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "second_key", (*params.Fields)["second_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "second_key", (*params.Fields)["second_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.JobID), "changed_key", (*params.Fields)["changed_key"], int32(2)). + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ClientID), "changed_key", (*params.Fields)["changed_key"], int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -250,7 +250,7 @@ func TestSubmitUpdate(t *testing.T) { } current := collector.Collector{ - JobID: uuid.New(), + ClientID: uuid.New(), ActiveVersion: 1, LatestVersion: 1, MinCleanVersion: 1, @@ -259,12 +259,12 @@ func TestSubmitUpdate(t *testing.T) { } av := int32(2) params := dbUpdateParams{ - JobID: database.MustToDBUUID(current.JobID), + ClientID: database.MustToDBUUID(current.ClientID), ActiveVersion: &av, } pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.JobID), int32(2)). + pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(current.ClientID), int32(2)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() @@ -625,13 +625,13 @@ func TestInformUpdate(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - cfg.JobSyncURL = "here" + cfg.ClientSyncURL = "here" svc := New(cfg, &Services{}) av := int32(2) update := dbUpdateParams{ - JobID: database.MustToDBUUID(uuid.New()), + ClientID: database.MustToDBUUID(uuid.New()), ActiveVersion: &av, } @@ -639,7 +639,7 @@ func TestInformUpdate(t *testing.T) { SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.JobID.String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", update.ClientID.String()) }), mock.Anything, ). @@ -658,12 +658,12 @@ func TestInformUpdate(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - cfg.JobSyncURL = "here" + cfg.ClientSyncURL = "here" svc := New(cfg, &Services{}) update := dbUpdateParams{ - JobID: database.MustToDBUUID(uuid.New()), + ClientID: database.MustToDBUUID(uuid.New()), } err = svc.informUpdate(ctx, &update) diff --git a/internal/database/repository/clean_test.go b/internal/database/repository/clean_test.go index f0aa1ea4..8dce52ac 100644 --- a/internal/database/repository/clean_test.go +++ b/internal/database/repository/clean_test.go @@ -32,13 +32,11 @@ func TestClean(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) hash := "example_hash" id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: hash, + Clientid: clientId, + Hash: hash, }) assert.NoError(t, err) assert.NotEmpty(t, id) @@ -131,15 +129,15 @@ func TestClean(t *testing.T) { assert.Equal(t, clean.Version, recent.Version) assert.Equal(t, clean.ID, recent.ID) - version, err := queries.AddLatestCollectorVersion(ctx, jobId) + version, err := queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: jobId, + Clientid: clientId, Versionid: version, }) assert.NoError(t, err) err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Jobid: jobId, + Clientid: clientId, Versionid: 5, Addedversion: version, }) diff --git a/internal/database/repository/client.sql.go b/internal/database/repository/client.sql.go index 86821e6a..b69ed43b 100644 --- a/internal/database/repository/client.sql.go +++ b/internal/database/repository/client.sql.go @@ -68,6 +68,188 @@ func (q *Queries) GetClient(ctx context.Context, id pgtype.UUID) (*GetClientRow, return &i, err } +const isClientSynced = `-- name: IsClientSynced :one +WITH +docs AS ( + -- Get all documents for this client + SELECT id, clientId FROM documents WHERE clientId = $1 +), +doc_clean_entries as ( + -- Documents with their current clean entries + SELECT + d.id AS document_id, + d.clientId as client_id, + cte.id AS clean_entry_id + FROM + docs d + LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id + WHERE cte.id is null or cte.id is not null and cte.fail is null +), +doc_text_entries AS ( + -- Documents with their current text entries + SELECT + d.document_id, + cte.id AS text_entry_id + FROM + doc_clean_entries d + LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id +), +required_results AS ( + -- All required document-query-version combinations + SELECT + d.document_id, + cqdt.queryId, + cqdt.queryVersion, + dte.text_entry_id + FROM + doc_clean_entries d + JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id + JOIN doc_text_entries dte ON dte.document_id = d.document_id + and dte.text_entry_id IS NOT NULL +), +existing_results AS ( + -- Valid results that exist + SELECT + rr.document_id AS document_id, + r.queryId, + r.id AS result_id + FROM + results r + JOIN required_results rr ON + r.queryId = rr.queryId AND + r.queryVersion = rr.queryVersion AND + r.textEntryId = rr.text_entry_id +), +missing_results AS ( + -- Find missing results + SELECT rr.queryId + FROM required_results rr + LEFT JOIN existing_results er on er.queryId = rr.queryId + WHERE er.result_id is null +), +dependency_check AS ( + -- Check for missing dependencies + SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId + FROM existing_results er + JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId + LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id + WHERE rd.resultId is null +) +SELECT ( + -- No documents means client is synced + NOT EXISTS (SELECT 1 FROM docs) + + OR + + -- Documents with no text entries + (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) + + and + + -- Documents with missing results + NOT EXISTS (SELECT 1 FROM missing_results) + + and + + -- Documents with missing dependencies + NOT EXISTS (SELECT 1 FROM dependency_check)) +)::bool as is_synced +` + +// IsClientSynced +// +// WITH +// docs AS ( +// -- Get all documents for this client +// SELECT id, clientId FROM documents WHERE clientId = $1 +// ), +// doc_clean_entries as ( +// -- Documents with their current clean entries +// SELECT +// d.id AS document_id, +// d.clientId as client_id, +// cte.id AS clean_entry_id +// FROM +// docs d +// LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id +// WHERE cte.id is null or cte.id is not null and cte.fail is null +// ), +// doc_text_entries AS ( +// -- Documents with their current text entries +// SELECT +// d.document_id, +// cte.id AS text_entry_id +// FROM +// doc_clean_entries d +// LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id +// ), +// required_results AS ( +// -- All required document-query-version combinations +// SELECT +// d.document_id, +// cqdt.queryId, +// cqdt.queryVersion, +// dte.text_entry_id +// FROM +// doc_clean_entries d +// JOIN collectorQueryDependencyTree cqdt ON cqdt.clientId = d.client_id +// JOIN doc_text_entries dte ON dte.document_id = d.document_id +// and dte.text_entry_id IS NOT NULL +// ), +// existing_results AS ( +// -- Valid results that exist +// SELECT +// rr.document_id AS document_id, +// r.queryId, +// r.id AS result_id +// FROM +// results r +// JOIN required_results rr ON +// r.queryId = rr.queryId AND +// r.queryVersion = rr.queryVersion AND +// r.textEntryId = rr.text_entry_id +// ), +// missing_results AS ( +// -- Find missing results +// SELECT rr.queryId +// FROM required_results rr +// LEFT JOIN existing_results er on er.queryId = rr.queryId +// WHERE er.result_id is null +// ), +// dependency_check AS ( +// -- Check for missing dependencies +// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId +// FROM existing_results er +// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId +// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id +// WHERE rd.resultId is null +// ) +// SELECT ( +// -- No documents means client is synced +// NOT EXISTS (SELECT 1 FROM docs) +// +// OR +// +// -- Documents with no text entries +// (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) +// +// and +// +// -- Documents with missing results +// NOT EXISTS (SELECT 1 FROM missing_results) +// +// and +// +// -- Documents with missing dependencies +// NOT EXISTS (SELECT 1 FROM dependency_check)) +// )::bool as is_synced +func (q *Queries) IsClientSynced(ctx context.Context, dollar_1 pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, isClientSynced, dollar_1) + var is_synced bool + err := row.Scan(&is_synced) + return is_synced, err +} + const updateClient = `-- name: UpdateClient :exec UPDATE clients SET name = $1 WHERE id = $2 ` diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go index f70576a0..a275daed 100644 --- a/internal/database/repository/collector.sql.go +++ b/internal/database/repository/collector.sql.go @@ -12,11 +12,11 @@ import ( ) const addCollectorQuery = `-- name: AddCollectorQuery :exec -INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) +INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) ` type AddCollectorQueryParams struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Name string `db:"name"` Queryid pgtype.UUID `db:"queryid"` Addedversion int32 `db:"addedversion"` @@ -24,10 +24,10 @@ type AddCollectorQueryParams struct { // AddCollectorQuery // -// INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) +// INSERT INTO collectorQueries (clientId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4) func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error { _, err := q.db.Exec(ctx, addCollectorQuery, - arg.Jobid, + arg.Clientid, arg.Name, arg.Queryid, arg.Addedversion, @@ -36,52 +36,31 @@ func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryP } const addLatestCollectorVersion = `-- name: AddLatestCollectorVersion :one -INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id +INSERT INTO collectorVersions (clientId) VALUES ($1) RETURNING id ` // AddLatestCollectorVersion // -// INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id -func (q *Queries) AddLatestCollectorVersion(ctx context.Context, jobid pgtype.UUID) (int32, error) { - row := q.db.QueryRow(ctx, addLatestCollectorVersion, jobid) +// INSERT INTO collectorVersions (clientId) VALUES ($1) RETURNING id +func (q *Queries) AddLatestCollectorVersion(ctx context.Context, clientid pgtype.UUID) (int32, error) { + row := q.db.QueryRow(ctx, addLatestCollectorVersion, clientid) var id int32 err := row.Scan(&id) return id, err } -const getCollector = `-- name: GetCollector :one -SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 +const getCollectorByClientID = `-- name: GetCollectorByClientID :one +SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1 ` -// GetCollector +// GetCollectorByClientID // -// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 -func (q *Queries) GetCollector(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) { - row := q.db.QueryRow(ctx, getCollector, jobid) +// SELECT clientid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE clientId = $1 LIMIT 1 +func (q *Queries) GetCollectorByClientID(ctx context.Context, clientid pgtype.UUID) (*Fullactivecollector, error) { + row := q.db.QueryRow(ctx, getCollectorByClientID, clientid) var i Fullactivecollector err := row.Scan( - &i.Jobid, - &i.Mincleanversion, - &i.Mintextversion, - &i.Activeversion, - &i.Latestversion, - &i.Fields, - ) - return &i, err -} - -const getCollectorByJobID = `-- name: GetCollectorByJobID :one -SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 -` - -// GetCollectorByJobID -// -// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1 -func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) { - row := q.db.QueryRow(ctx, getCollectorByJobID, jobid) - var i Fullactivecollector - err := row.Scan( - &i.Jobid, + &i.Clientid, &i.Mincleanversion, &i.Mintextversion, &i.Activeversion, @@ -92,14 +71,14 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (* } const listCollectorQueries = `-- name: ListCollectorQueries :many -SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1 +SELECT clientid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 ` // ListCollectorQueries // -// SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1 -func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ([]*Collectorquerydependencytree, error) { - rows, err := q.db.Query(ctx, listCollectorQueries, jobid) +// SELECT clientid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE clientId = $1 +func (q *Queries) ListCollectorQueries(ctx context.Context, clientid pgtype.UUID) ([]*Collectorquerydependencytree, error) { + rows, err := q.db.Query(ctx, listCollectorQueries, clientid) if err != nil { return nil, err } @@ -108,7 +87,7 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ( for rows.Next() { var i Collectorquerydependencytree if err := rows.Scan( - &i.Jobid, + &i.Clientid, &i.Queryid, &i.Type, &i.Queryversion, @@ -125,72 +104,72 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ( } const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec -UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null +UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null ` type RemoveCollectorQueryParams struct { Removedversion *int32 `db:"removedversion"` Queryid pgtype.UUID `db:"queryid"` - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` } // RemoveCollectorQuery // -// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null +// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and clientId = $3 and removedVersion is null func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error { - _, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Jobid) + _, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Clientid) return err } const setActiveCollectorVersion = `-- name: SetActiveCollectorVersion :exec -INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2) +INSERT INTO collectorActiveVersions (clientId, versionId) VALUES ($1, $2) ` type SetActiveCollectorVersionParams struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Versionid int32 `db:"versionid"` } // SetActiveCollectorVersion // -// INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2) +// INSERT INTO collectorActiveVersions (clientId, versionId) VALUES ($1, $2) func (q *Queries) SetActiveCollectorVersion(ctx context.Context, arg *SetActiveCollectorVersionParams) error { - _, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Jobid, arg.Versionid) + _, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Clientid, arg.Versionid) return err } const setCollectorCleanVersion = `-- name: SetCollectorCleanVersion :exec -INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) +INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3) ` type SetCollectorCleanVersionParams struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Addedversion int32 `db:"addedversion"` Versionid int32 `db:"versionid"` } // SetCollectorCleanVersion // -// INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) +// INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3) func (q *Queries) SetCollectorCleanVersion(ctx context.Context, arg *SetCollectorCleanVersionParams) error { - _, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Jobid, arg.Addedversion, arg.Versionid) + _, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Clientid, arg.Addedversion, arg.Versionid) return err } const setCollectorTextVersion = `-- name: SetCollectorTextVersion :exec -INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) +INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3) ` type SetCollectorTextVersionParams struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Addedversion int32 `db:"addedversion"` Versionid int32 `db:"versionid"` } // SetCollectorTextVersion // -// INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3) +// INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3) func (q *Queries) SetCollectorTextVersion(ctx context.Context, arg *SetCollectorTextVersionParams) error { - _, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Jobid, arg.Addedversion, arg.Versionid) + _, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Clientid, arg.Addedversion, arg.Versionid) return err } diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index b1ed9f74..a0d06455 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -54,39 +54,27 @@ func TestCollector(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) minCleanVersion := int32(2) minTextVersion := int32(4) - coll, err := queries.GetCollector(ctx, jobId) + coll, err := queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: 0, Mintextversion: 0, Activeversion: 0, Latestversion: 0, }, coll) - coll, err = queries.GetCollectorByJobID(ctx, jobId) - assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, - Mincleanversion: 0, - Mintextversion: 0, - Activeversion: 0, - Latestversion: 0, - }, coll) - - version, err = queries.AddLatestCollectorVersion(ctx, jobId) + version, err = queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) assert.Equal(t, int32(1), version) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: 0, Mintextversion: 0, Activeversion: 0, @@ -95,14 +83,14 @@ func TestCollector(t *testing.T) { err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 1, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: 0, Mintextversion: 0, Activeversion: 1, @@ -110,7 +98,7 @@ func TestCollector(t *testing.T) { }, coll) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonId, Addedversion: 1, Name: "example_key", @@ -118,16 +106,16 @@ func TestCollector(t *testing.T) { assert.NoError(t, err) err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Jobid: jobId, + Clientid: clientId, Addedversion: 1, Versionid: minTextVersion, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: 0, Mintextversion: minTextVersion, Activeversion: 1, @@ -136,16 +124,16 @@ func TestCollector(t *testing.T) { }, coll) err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Jobid: jobId, + Clientid: clientId, Addedversion: 1, Versionid: minCleanVersion, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: minCleanVersion, Mintextversion: minTextVersion, Activeversion: 1, @@ -153,19 +141,19 @@ func TestCollector(t *testing.T) { Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())), }, coll) - qs, err := queries.ListCollectorQueries(ctx, jobId) + qs, err := queries.ListCollectorQueries(ctx, clientId) assert.NoError(t, err) assert.Len(t, qs, 2) assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{ { - Jobid: jobId, + Clientid: clientId, Queryid: jsonId, Queryversion: 1, Type: repository.QuerytypeJsonExtractor, Requiredids: []pgtype.UUID{contextId}, }, { - Jobid: jobId, + Clientid: clientId, Queryid: contextId, Queryversion: 0, Type: repository.QuerytypeContextFull, @@ -173,12 +161,12 @@ func TestCollector(t *testing.T) { }, }, qs) - version, err = queries.AddLatestCollectorVersion(ctx, jobId) + version, err = queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) assert.Equal(t, int32(2), version) err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonId, Removedversion: &version, }) @@ -186,21 +174,21 @@ func TestCollector(t *testing.T) { err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 2, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{ - Jobid: jobId, + Clientid: clientId, Versionid: minTextVersion + 1, Addedversion: version, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: minCleanVersion, Mintextversion: minTextVersion + 1, Activeversion: 2, @@ -209,16 +197,16 @@ func TestCollector(t *testing.T) { }, coll) err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{ - Jobid: jobId, + Clientid: clientId, Versionid: minCleanVersion + 1, Addedversion: version, }) assert.NoError(t, err) - coll, err = queries.GetCollector(ctx, jobId) + coll, err = queries.GetCollectorByClientID(ctx, clientId) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Fullactivecollector{ - Jobid: jobId, + Clientid: clientId, Mincleanversion: minCleanVersion + 1, Mintextversion: minTextVersion + 1, Activeversion: 2, diff --git a/internal/database/repository/document.sql.go b/internal/database/repository/document.sql.go index 40cf092f..d256d4b7 100644 --- a/internal/database/repository/document.sql.go +++ b/internal/database/repository/document.sql.go @@ -30,35 +30,35 @@ func (q *Queries) AddDocumentEntry(ctx context.Context, arg *AddDocumentEntryPar } const createDocument = `-- name: CreateDocument :one -INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id +INSERT INTO documents (clientId, hash) VALUES ($1, $2) RETURNING id ` type CreateDocumentParams struct { - Jobid pgtype.UUID `db:"jobid"` - Hash string `db:"hash"` + Clientid pgtype.UUID `db:"clientid"` + Hash string `db:"hash"` } // CreateDocument // -// INSERT INTO documents (jobId, hash) VALUES ($1, $2) RETURNING id +// INSERT INTO documents (clientId, hash) VALUES ($1, $2) RETURNING id func (q *Queries) CreateDocument(ctx context.Context, arg *CreateDocumentParams) (pgtype.UUID, error) { - row := q.db.QueryRow(ctx, createDocument, arg.Jobid, arg.Hash) + row := q.db.QueryRow(ctx, createDocument, arg.Clientid, arg.Hash) var id pgtype.UUID err := row.Scan(&id) return id, err } const getDocument = `-- name: GetDocument :one -SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1 +SELECT id, clientId, hash FROM documents WHERE id = $1 LIMIT 1 ` // GetDocument // -// SELECT id, jobId, hash FROM documents WHERE id = $1 LIMIT 1 +// SELECT id, clientId, hash FROM documents WHERE id = $1 LIMIT 1 func (q *Queries) GetDocument(ctx context.Context, id pgtype.UUID) (*Document, error) { row := q.db.QueryRow(ctx, getDocument, id) var i Document - err := row.Scan(&i.ID, &i.Jobid, &i.Hash) + err := row.Scan(&i.ID, &i.Clientid, &i.Hash) return &i, err } @@ -83,27 +83,65 @@ func (q *Queries) GetDocumentEntry(ctx context.Context, documentid pgtype.UUID) } const getDocumentIDByHash = `-- name: GetDocumentIDByHash :one -SELECT id FROM documents WHERE hash = $1 and jobId = $2 +SELECT id FROM documents WHERE hash = $1 and clientId = $2 ` type GetDocumentIDByHashParams struct { - Hash string `db:"hash"` - Jobid pgtype.UUID `db:"jobid"` + Hash string `db:"hash"` + Clientid pgtype.UUID `db:"clientid"` } // GetDocumentIDByHash // -// SELECT id FROM documents WHERE hash = $1 and jobId = $2 +// SELECT id FROM documents WHERE hash = $1 and clientId = $2 func (q *Queries) GetDocumentIDByHash(ctx context.Context, arg *GetDocumentIDByHashParams) (pgtype.UUID, error) { - row := q.db.QueryRow(ctx, getDocumentIDByHash, arg.Hash, arg.Jobid) + row := q.db.QueryRow(ctx, getDocumentIDByHash, arg.Hash, arg.Clientid) var id pgtype.UUID err := row.Scan(&id) return id, err } -const listDocumentsByJobId = `-- name: ListDocumentsByJobId :many +const listDocumentIDsBatch = `-- name: ListDocumentIDsBatch :many +SELECT id, totalCount FROM listDocumentIDs($1, $2, $3) +` + +type ListDocumentIDsBatchParams struct { + Clientid pgtype.UUID `db:"clientid"` + Batchsize int32 `db:"batchsize"` + Pageoffset int32 `db:"pageoffset"` +} + +type ListDocumentIDsBatchRow struct { + ID pgtype.UUID `db:"id"` + Totalcount *int64 `db:"totalcount"` +} + +// ListDocumentIDsBatch +// +// SELECT id, totalCount FROM listDocumentIDs($1, $2, $3) +func (q *Queries) ListDocumentIDsBatch(ctx context.Context, arg *ListDocumentIDsBatchParams) ([]*ListDocumentIDsBatchRow, error) { + rows, err := q.db.Query(ctx, listDocumentIDsBatch, arg.Clientid, arg.Batchsize, arg.Pageoffset) + if err != nil { + return nil, err + } + defer rows.Close() + items := []*ListDocumentIDsBatchRow{} + for rows.Next() { + var i ListDocumentIDsBatchRow + if err := rows.Scan(&i.ID, &i.Totalcount); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDocumentsByClientId = `-- name: ListDocumentsByClientId :many WITH docs as ( - SELECT id from documents where jobId = $1 + SELECT id from documents where clientId = $1 ), entries as ( SELECT @@ -117,16 +155,16 @@ entries as ( SELECT documentId as id, bucket, key from entries WHERE rowNumber = 1 ` -type ListDocumentsByJobIdRow struct { +type ListDocumentsByClientIdRow struct { ID pgtype.UUID `db:"id"` Bucket string `db:"bucket"` Key string `db:"key"` } -// ListDocumentsByJobId +// ListDocumentsByClientId // // WITH docs as ( -// SELECT id from documents where jobId = $1 +// SELECT id from documents where clientId = $1 // ), // entries as ( // SELECT @@ -138,15 +176,15 @@ type ListDocumentsByJobIdRow struct { // JOIN documentEntries de on d.id = de.documentId // ) // SELECT documentId as id, bucket, key from entries WHERE rowNumber = 1 -func (q *Queries) ListDocumentsByJobId(ctx context.Context, jobid pgtype.UUID) ([]*ListDocumentsByJobIdRow, error) { - rows, err := q.db.Query(ctx, listDocumentsByJobId, jobid) +func (q *Queries) ListDocumentsByClientId(ctx context.Context, clientid pgtype.UUID) ([]*ListDocumentsByClientIdRow, error) { + rows, err := q.db.Query(ctx, listDocumentsByClientId, clientid) if err != nil { return nil, err } defer rows.Close() - items := []*ListDocumentsByJobIdRow{} + items := []*ListDocumentsByClientIdRow{} for rows.Next() { - var i ListDocumentsByJobIdRow + var i ListDocumentsByClientIdRow if err := rows.Scan(&i.ID, &i.Bucket, &i.Key); err != nil { return nil, err } diff --git a/internal/database/repository/document_test.go b/internal/database/repository/document_test.go index e4f92e2e..f43a03fe 100644 --- a/internal/database/repository/document_test.go +++ b/internal/database/repository/document_test.go @@ -32,17 +32,15 @@ func TestDocument(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - docs, err := queries.ListDocumentsByJobId(ctx, jobId) + docs, err := queries.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Len(t, docs, 0) hash := "example_hash" id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: hash, + Clientid: clientId, + Hash: hash, }) assert.NoError(t, err) assert.NotEmpty(t, id) @@ -56,15 +54,15 @@ func TestDocument(t *testing.T) { }) assert.NoError(t, err) - docs, err = queries.ListDocumentsByJobId(ctx, jobId) + docs, err = queries.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Len(t, docs, 1) assert.Equal(t, bucketone, docs[0].Bucket) assert.Equal(t, keyone, docs[0].Key) documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash_two", + Clientid: clientId, + Hash: "example_hash_two", }) assert.NoError(t, err) assert.NotEmpty(t, documentTwoID) @@ -78,23 +76,23 @@ func TestDocument(t *testing.T) { }) assert.NoError(t, err) - docs, err = queries.ListDocumentsByJobId(ctx, jobId) + docs, err = queries.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Len(t, docs, 2) - jobTwoId, err := queries.CreateJob(ctx, clientId) + clientTwoId, err := queries.CreateClient(ctx, "client two") assert.NoError(t, err) - docs, err = queries.ListDocumentsByJobId(ctx, jobId) + docs, err = queries.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Len(t, docs, 2) - docs, err = queries.ListDocumentsByJobId(ctx, jobTwoId) + docs, err = queries.ListDocumentsByClientId(ctx, clientTwoId) assert.NoError(t, err) assert.Len(t, docs, 0) documentThreeId, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobTwoId, - Hash: "example_hash", + Clientid: clientTwoId, + Hash: "example_hash", }) assert.NoError(t, err) @@ -107,10 +105,10 @@ func TestDocument(t *testing.T) { }) assert.NoError(t, err) - docs, err = queries.ListDocumentsByJobId(ctx, jobId) + docs, err = queries.ListDocumentsByClientId(ctx, clientId) assert.NoError(t, err) assert.Len(t, docs, 2) - docs, err = queries.ListDocumentsByJobId(ctx, jobTwoId) + docs, err = queries.ListDocumentsByClientId(ctx, clientTwoId) assert.NoError(t, err) assert.Len(t, docs, 1) assert.Equal(t, bucketthree, docs[0].Bucket) @@ -119,14 +117,14 @@ func TestDocument(t *testing.T) { doc, err := queries.GetDocument(ctx, id) assert.NoError(t, err) assert.EqualExportedValues(t, &repository.Document{ - ID: id, - Jobid: jobId, - Hash: hash, + ID: id, + Clientid: clientId, + Hash: hash, }, doc) docid, err := queries.GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{ - Hash: hash, - Jobid: jobId, + Hash: hash, + Clientid: clientId, }) assert.NoError(t, err) assert.EqualExportedValues(t, id, docid) diff --git a/internal/database/repository/job.sql.go b/internal/database/repository/job.sql.go deleted file mode 100644 index 92d15f57..00000000 --- a/internal/database/repository/job.sql.go +++ /dev/null @@ -1,289 +0,0 @@ -// Code generated by sqlc. DO NOT EDIT. -// versions: -// sqlc v1.27.0 -// source: job.sql - -package repository - -import ( - "context" - - "github.com/jackc/pgx/v5/pgtype" -) - -const addJobCanSync = `-- name: AddJobCanSync :exec -INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2) -` - -type AddJobCanSyncParams struct { - Cansync bool `db:"cansync"` - Jobid pgtype.UUID `db:"jobid"` -} - -// AddJobCanSync -// -// INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2) -func (q *Queries) AddJobCanSync(ctx context.Context, arg *AddJobCanSyncParams) error { - _, err := q.db.Exec(ctx, addJobCanSync, arg.Cansync, arg.Jobid) - return err -} - -const createJob = `-- name: CreateJob :one -INSERT INTO jobs (clientId) VALUES ($1) RETURNING id -` - -// CreateJob -// -// INSERT INTO jobs (clientId) VALUES ($1) RETURNING id -func (q *Queries) CreateJob(ctx context.Context, clientid pgtype.UUID) (pgtype.UUID, error) { - row := q.db.QueryRow(ctx, createJob, clientid) - var id pgtype.UUID - err := row.Scan(&id) - return id, err -} - -const getJob = `-- name: GetJob :one -SELECT j.id, j.clientId, cs.canSync - FROM jobs as j - JOIN currentJobCanSync as cs on cs.jobId = j.id - WHERE j.id = $1 -` - -type GetJobRow struct { - ID pgtype.UUID `db:"id"` - Clientid pgtype.UUID `db:"clientid"` - Cansync bool `db:"cansync"` -} - -// GetJob -// -// SELECT j.id, j.clientId, cs.canSync -// FROM jobs as j -// JOIN currentJobCanSync as cs on cs.jobId = j.id -// WHERE j.id = $1 -func (q *Queries) GetJob(ctx context.Context, id pgtype.UUID) (*GetJobRow, error) { - row := q.db.QueryRow(ctx, getJob, id) - var i GetJobRow - err := row.Scan(&i.ID, &i.Clientid, &i.Cansync) - return &i, err -} - -const isJobSynced = `-- name: IsJobSynced :one -WITH -docs AS ( - -- Get all documents for this job - SELECT id, jobId FROM documents WHERE jobId = $1 -), -doc_clean_entries as ( - -- Documents with their current clean entries - SELECT - d.id AS document_id, - d.jobId as job_id, - cte.id AS clean_entry_id - FROM - docs d - LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id - WHERE cte.id is null or cte.id is not null and cte.fail is null -), -doc_text_entries AS ( - -- Documents with their current text entries - SELECT - d.document_id, - cte.id AS text_entry_id - FROM - doc_clean_entries d - LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id -), -required_results AS ( - -- All required document-query-version combinations - SELECT - d.document_id, - cqdt.queryId, - cqdt.queryVersion, - dte.text_entry_id - FROM - doc_clean_entries d - JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.job_id - JOIN doc_text_entries dte ON dte.document_id = d.document_id - and dte.text_entry_id IS NOT NULL -), -existing_results AS ( - -- Valid results that exist - SELECT - rr.document_id AS document_id, - r.queryId, - r.id AS result_id - FROM - results r - JOIN required_results rr ON - r.queryId = rr.queryId AND - r.queryVersion = rr.queryVersion AND - r.textEntryId = rr.text_entry_id -), -missing_results AS ( - -- Find missing results - SELECT rr.queryId - FROM required_results rr - LEFT JOIN existing_results er on er.queryId = rr.queryId - WHERE er.result_id is null -), -dependency_check AS ( - -- Check for missing dependencies - SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId - FROM existing_results er - JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId - LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id - WHERE rd.resultId is null -) -SELECT ( - -- No documents means job is synced - NOT EXISTS (SELECT 1 FROM docs) - - OR - - -- Documents with no text entries - (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) - - and - - -- Documents with missing results - NOT EXISTS (SELECT 1 FROM missing_results) - - and - - -- Documents with missing dependencies - NOT EXISTS (SELECT 1 FROM dependency_check)) -)::bool as is_synced -` - -// IsJobSynced -// -// WITH -// docs AS ( -// -- Get all documents for this job -// SELECT id, jobId FROM documents WHERE jobId = $1 -// ), -// doc_clean_entries as ( -// -- Documents with their current clean entries -// SELECT -// d.id AS document_id, -// d.jobId as job_id, -// cte.id AS clean_entry_id -// FROM -// docs d -// LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id -// WHERE cte.id is null or cte.id is not null and cte.fail is null -// ), -// doc_text_entries AS ( -// -- Documents with their current text entries -// SELECT -// d.document_id, -// cte.id AS text_entry_id -// FROM -// doc_clean_entries d -// LEFT JOIN currentTextEntries cte ON cte.cleanEntryId = d.clean_entry_id -// ), -// required_results AS ( -// -- All required document-query-version combinations -// SELECT -// d.document_id, -// cqdt.queryId, -// cqdt.queryVersion, -// dte.text_entry_id -// FROM -// doc_clean_entries d -// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.job_id -// JOIN doc_text_entries dte ON dte.document_id = d.document_id -// and dte.text_entry_id IS NOT NULL -// ), -// existing_results AS ( -// -- Valid results that exist -// SELECT -// rr.document_id AS document_id, -// r.queryId, -// r.id AS result_id -// FROM -// results r -// JOIN required_results rr ON -// r.queryId = rr.queryId AND -// r.queryVersion = rr.queryVersion AND -// r.textEntryId = rr.text_entry_id -// ), -// missing_results AS ( -// -- Find missing results -// SELECT rr.queryId -// FROM required_results rr -// LEFT JOIN existing_results er on er.queryId = rr.queryId -// WHERE er.result_id is null -// ), -// dependency_check AS ( -// -- Check for missing dependencies -// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId -// FROM existing_results er -// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId -// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id -// WHERE rd.resultId is null -// ) -// SELECT ( -// -- No documents means job is synced -// NOT EXISTS (SELECT 1 FROM docs) -// -// OR -// -// -- Documents with no text entries -// (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL) -// -// and -// -// -- Documents with missing results -// NOT EXISTS (SELECT 1 FROM missing_results) -// -// and -// -// -- Documents with missing dependencies -// NOT EXISTS (SELECT 1 FROM dependency_check)) -// )::bool as is_synced -func (q *Queries) IsJobSynced(ctx context.Context, dollar_1 pgtype.UUID) (bool, error) { - row := q.db.QueryRow(ctx, isJobSynced, dollar_1) - var is_synced bool - err := row.Scan(&is_synced) - return is_synced, err -} - -const listJobDocumentIDsBatch = `-- name: ListJobDocumentIDsBatch :many -SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3) -` - -type ListJobDocumentIDsBatchParams struct { - Jobid pgtype.UUID `db:"jobid"` - Batchsize int32 `db:"batchsize"` - Pageoffset int32 `db:"pageoffset"` -} - -type ListJobDocumentIDsBatchRow struct { - ID pgtype.UUID `db:"id"` - Totalcount *int64 `db:"totalcount"` -} - -// ListJobDocumentIDsBatch -// -// SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3) -func (q *Queries) ListJobDocumentIDsBatch(ctx context.Context, arg *ListJobDocumentIDsBatchParams) ([]*ListJobDocumentIDsBatchRow, error) { - rows, err := q.db.Query(ctx, listJobDocumentIDsBatch, arg.Jobid, arg.Batchsize, arg.Pageoffset) - if err != nil { - return nil, err - } - defer rows.Close() - items := []*ListJobDocumentIDsBatchRow{} - for rows.Next() { - var i ListJobDocumentIDsBatchRow - if err := rows.Scan(&i.ID, &i.Totalcount); err != nil { - return nil, err - } - items = append(items, &i) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} diff --git a/internal/database/repository/job_test.go b/internal/database/repository/job_test.go index 4326ed63..448f303b 100644 --- a/internal/database/repository/job_test.go +++ b/internal/database/repository/job_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestJob(t *testing.T) { +func TestListClientDocumentIDs(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } @@ -30,61 +30,11 @@ func TestJob(t *testing.T) { queries := cfg.GetDBQueries() - clientId, err := queries.CreateClient(ctx, "example_client") + id, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - id, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - assert.NotEmpty(t, id) - - job, err := queries.GetJob(ctx, id) - assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetJobRow{ - ID: id, - Clientid: clientId, - Cansync: false, - }, job) - - err = queries.AddJobCanSync(ctx, &repository.AddJobCanSyncParams{ - Cansync: true, - Jobid: id, - }) - assert.NoError(t, err) - - job, err = queries.GetJob(ctx, id) - assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetJobRow{ - ID: id, - Clientid: clientId, - Cansync: true, - }, job) -} -func TestListJobDocumentIDs(t *testing.T) { - if testing.Short() { - t.Skip("Skipping long test in short mode") - } - ctx := context.Background() - - cfg := &serviceconfig.BaseConfig{} - test.SetCfgProvider(t, cfg) - cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../..")) - _, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ - Cfg: cfg, - RunMigrations: true, - }) - defer cleanup() - - queries := cfg.GetDBQueries() - - clientId, err := queries.CreateClient(ctx, "example_client") - assert.NoError(t, err) - - id, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - assert.NotEmpty(t, id) - - ids, err := queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ - Jobid: id, + ids, err := queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ + Clientid: id, Batchsize: 1, Pageoffset: 0, }) @@ -92,20 +42,20 @@ func TestListJobDocumentIDs(t *testing.T) { assert.Len(t, ids, 0) docOne, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: id, - Hash: "example_hash", + Clientid: id, + Hash: "example_hash", }) assert.NoError(t, err) - ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ - Jobid: id, + ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ + Clientid: id, Batchsize: 1, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 1) total := int64(1) - assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ + assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, @@ -113,48 +63,48 @@ func TestListJobDocumentIDs(t *testing.T) { }, ids) docTwo, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: id, - Hash: "example_hash_two", + Clientid: id, + Hash: "example_hash_two", }) assert.NoError(t, err) - ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ - Jobid: id, + ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ + Clientid: id, Batchsize: 1, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 1) total = int64(2) - assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ + assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, }, }, ids) - ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ - Jobid: id, + ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ + Clientid: id, Batchsize: 1, Pageoffset: 1, }) assert.NoError(t, err) assert.Len(t, ids, 1) - assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ + assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{ { ID: docTwo, Totalcount: &total, }, }, ids) - ids, err = queries.ListJobDocumentIDsBatch(ctx, &repository.ListJobDocumentIDsBatchParams{ - Jobid: id, + ids, err = queries.ListDocumentIDsBatch(ctx, &repository.ListDocumentIDsBatchParams{ + Clientid: id, Batchsize: 2, Pageoffset: 0, }) assert.NoError(t, err) assert.Len(t, ids, 2) - assert.ElementsMatch(t, []*repository.ListJobDocumentIDsBatchRow{ + assert.ElementsMatch(t, []*repository.ListDocumentIDsBatchRow{ { ID: docOne, Totalcount: &total, @@ -166,7 +116,7 @@ func TestListJobDocumentIDs(t *testing.T) { }, ids) } -func TestJobSync(t *testing.T) { +func TestClientSync(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } @@ -210,35 +160,33 @@ func TestJobSync(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - _, err = queries.AddLatestCollectorVersion(ctx, jobId) + _, err = queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 1, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonQueryID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) - isSynced, err := queries.IsJobSynced(ctx, jobId) + isSynced, err := queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) documentNoCleanID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_noclean", + Clientid: clientId, + Hash: "example_noclean", }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -258,17 +206,17 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash", + Clientid: clientId, + Hash: "example_hash", }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -289,7 +237,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -301,7 +249,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -316,7 +264,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -328,7 +276,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -338,7 +286,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) @@ -350,7 +298,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -362,7 +310,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -374,7 +322,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -384,7 +332,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) @@ -396,7 +344,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -411,7 +359,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -423,7 +371,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -433,7 +381,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) @@ -453,7 +401,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -465,7 +413,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -480,7 +428,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -492,7 +440,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -502,60 +450,60 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) - _, err = queries.AddLatestCollectorVersion(ctx, jobId) + _, err = queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) collversion := int32(2) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: 2, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonQueryID, Removedversion: &collversion, }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonQueryID, Addedversion: 2, Name: "second_key", }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: contextQueryID, Addedversion: 2, Name: "example_key", }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) @@ -583,7 +531,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -595,7 +543,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -607,7 +555,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -617,7 +565,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, isSynced) @@ -627,7 +575,7 @@ func TestJobSync(t *testing.T) { }) assert.NoError(t, err) - isSynced, err = queries.IsJobSynced(ctx, jobId) + isSynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, isSynced) } diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index 029f162d..5d4c40c3 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -191,23 +191,23 @@ type Clientcansync struct { type Collectoractiveversion struct { ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Versionid int32 `db:"versionid"` } type Collectorcurrentactiveversion struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Activeversion int32 `db:"activeversion"` } type Collectorlatestversion struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Latestversion int32 `db:"latestversion"` } type Collectormincleanversion struct { ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Versionid int32 `db:"versionid"` Addedversion int32 `db:"addedversion"` Removedversion *int32 `db:"removedversion"` @@ -215,7 +215,7 @@ type Collectormincleanversion struct { type Collectormintextversion struct { ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Versionid int32 `db:"versionid"` Addedversion int32 `db:"addedversion"` Removedversion *int32 `db:"removedversion"` @@ -223,7 +223,7 @@ type Collectormintextversion struct { type Collectorquery struct { ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Name string `db:"name"` Queryid pgtype.UUID `db:"queryid"` Addedversion int32 `db:"addedversion"` @@ -231,7 +231,7 @@ type Collectorquery struct { } type Collectorquerydependencytree struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Queryid pgtype.UUID `db:"queryid"` Type Querytype `db:"type"` Queryversion int32 `db:"queryversion"` @@ -239,9 +239,9 @@ type Collectorquerydependencytree struct { } type Collectorversion struct { - Jobid pgtype.UUID `db:"jobid"` - ID int32 `db:"id"` - Addedat pgtype.Timestamp `db:"addedat"` + Clientid pgtype.UUID `db:"clientid"` + ID int32 `db:"id"` + Addedat pgtype.Timestamp `db:"addedat"` } type Currentcleanentry struct { @@ -260,29 +260,24 @@ type Currentclientcansync struct { } type Currentcollectormincleanversion struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Mincleanversion int32 `db:"mincleanversion"` } type Currentcollectormintextversion struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Mintextversion int32 `db:"mintextversion"` } type Currentcollectorqueriesjsonagg struct { - Jobid pgtype.UUID `db:"jobid"` - Fields []byte `db:"fields"` + Clientid pgtype.UUID `db:"clientid"` + Fields []byte `db:"fields"` } type Currentcollectorquery struct { - Jobid pgtype.UUID `db:"jobid"` - Name *string `db:"name"` - Queryid pgtype.UUID `db:"queryid"` -} - -type Currentjobcansync struct { - Jobid pgtype.UUID `db:"jobid"` - Cansync bool `db:"cansync"` + Clientid pgtype.UUID `db:"clientid"` + Name *string `db:"name"` + Queryid pgtype.UUID `db:"queryid"` } type Currenttextentry struct { @@ -295,9 +290,9 @@ type Currenttextentry struct { } type Document struct { - ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` - Hash string `db:"hash"` + ID pgtype.UUID `db:"id"` + Clientid pgtype.UUID `db:"clientid"` + Hash string `db:"hash"` } type Documentclean struct { @@ -331,7 +326,7 @@ type Documenttextextraction struct { } type Fullactivecollector struct { - Jobid pgtype.UUID `db:"jobid"` + Clientid pgtype.UUID `db:"clientid"` Mincleanversion int32 `db:"mincleanversion"` Mintextversion int32 `db:"mintextversion"` Activeversion int32 `db:"activeversion"` @@ -348,17 +343,6 @@ type Fullactivequery struct { Requiredids []pgtype.UUID `db:"requiredids"` } -type Job struct { - ID pgtype.UUID `db:"id"` - Clientid pgtype.UUID `db:"clientid"` -} - -type Jobcansync struct { - ID pgtype.UUID `db:"id"` - Jobid pgtype.UUID `db:"jobid"` - Cansync bool `db:"cansync"` -} - type Query struct { ID pgtype.UUID `db:"id"` Type Querytype `db:"type"` diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index c8285988..c9dc4fbf 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -317,14 +317,41 @@ func (q *Queries) ListQueriesById(ctx context.Context, id []pgtype.UUID) ([]*Ful return items, nil } +const listQueryClientIDs = `-- name: ListQueryClientIDs :many +SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1 +` + +// ListQueryClientIDs +// +// SELECT clientId FROM collectorQueryDependencyTree WHERE queryId = $1 +func (q *Queries) ListQueryClientIDs(ctx context.Context, queryid pgtype.UUID) ([]pgtype.UUID, error) { + rows, err := q.db.Query(ctx, listQueryClientIDs, queryid) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.UUID{} + for rows.Next() { + var clientid pgtype.UUID + if err := rows.Scan(&clientid); err != nil { + return nil, err + } + items = append(items, clientid) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const listQueryDirectDependentsByDocumentID = `-- name: ListQueryDirectDependentsByDocumentID :many WITH doc AS ( - SELECT id, jobId FROM documents where id = $2 + SELECT id, clientId FROM documents where id = $2 ) SELECT dt.queryId FROM doc as d JOIN collectorQueryDependencyTree as dt - on d.jobId = dt.jobId + on d.clientId = dt.clientId and $1 = any(dt.requiredIds) ` @@ -336,12 +363,12 @@ type ListQueryDirectDependentsByDocumentIDParams struct { // ListQueryDirectDependentsByDocumentID // // WITH doc AS ( -// SELECT id, jobId FROM documents where id = $2 +// SELECT id, clientId FROM documents where id = $2 // ) // SELECT dt.queryId // FROM doc as d // JOIN collectorQueryDependencyTree as dt -// on d.jobId = dt.jobId +// on d.clientId = dt.clientId // and $1 = any(dt.requiredIds) func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) { rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid) @@ -363,33 +390,6 @@ func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg return items, nil } -const listQueryJobIDs = `-- name: ListQueryJobIDs :many -SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1 -` - -// ListQueryJobIDs -// -// SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1 -func (q *Queries) ListQueryJobIDs(ctx context.Context, queryid pgtype.UUID) ([]pgtype.UUID, error) { - rows, err := q.db.Query(ctx, listQueryJobIDs, queryid) - if err != nil { - return nil, err - } - defer rows.Close() - items := []pgtype.UUID{} - for rows.Next() { - var jobid pgtype.UUID - if err := rows.Scan(&jobid); err != nil { - return nil, err - } - items = append(items, jobid) - } - if err := rows.Err(); err != nil { - return nil, err - } - return items, nil -} - const removeRequiredQuery = `-- name: RemoveRequiredQuery :exec UPDATE requiredQueries SET removedVersion = $1 WHERE requiredQueryId = $2 and queryId = $3 and removedVersion is null ` diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 217d5618..117e2deb 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -251,18 +251,16 @@ func TestQueryDependencyTree(t *testing.T) { clientID, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobID, err := queries.CreateJob(ctx, clientID) - assert.NoError(t, err) - version, err := queries.AddLatestCollectorVersion(ctx, jobID) + version, err := queries.AddLatestCollectorVersion(ctx, clientID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: jobID, + Clientid: clientID, Versionid: version, }) assert.NoError(t, err) docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobID, - Hash: "sample", + Clientid: clientID, + Hash: "sample", }) assert.NoError(t, err) @@ -321,7 +319,7 @@ func TestQueryDependencyTree(t *testing.T) { assert.ElementsMatch(t, []pgtype.UUID{}, dependents) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobID, + Clientid: clientID, Name: "example_name'", Queryid: jsonQueryID, Addedversion: 1, @@ -391,7 +389,7 @@ func TestQueryDependencyTree(t *testing.T) { assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobID, + Clientid: clientID, Name: "example_name'", Queryid: secondJsonQueryID, Addedversion: 1, @@ -513,7 +511,7 @@ func TestQueriesList(t *testing.T) { }, qs) } -func TestListQueryJobs(t *testing.T) { +func TestListQueryClients(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } @@ -533,76 +531,51 @@ func TestListQueryJobs(t *testing.T) { contextID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) - jobs, err := queries.ListQueryJobIDs(ctx, contextID) + clients, err := queries.ListQueryClientIDs(ctx, contextID) assert.NoError(t, err) - assert.ElementsMatch(t, []pgtype.UUID{}, jobs) + assert.ElementsMatch(t, []pgtype.UUID{}, clients) clientOneID, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobOneID, err := queries.CreateJob(ctx, clientOneID) - assert.NoError(t, err) - versionOne, err := queries.AddLatestCollectorVersion(ctx, jobOneID) + versionOne, err := queries.AddLatestCollectorVersion(ctx, clientOneID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: jobOneID, + Clientid: clientOneID, Versionid: versionOne, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobOneID, + Clientid: clientOneID, Queryid: contextID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) - jobs, err = queries.ListQueryJobIDs(ctx, contextID) + clients, err = queries.ListQueryClientIDs(ctx, contextID) assert.NoError(t, err) - assert.ElementsMatch(t, []pgtype.UUID{jobOneID}, jobs) + assert.ElementsMatch(t, []pgtype.UUID{clientOneID}, clients) - jobTwoID, err := queries.CreateJob(ctx, clientOneID) + clientTwoID, err := queries.CreateClient(ctx, "client dos") assert.NoError(t, err) - versionTwo, err := queries.AddLatestCollectorVersion(ctx, jobTwoID) + versionTwo, err := queries.AddLatestCollectorVersion(ctx, clientTwoID) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: jobTwoID, + Clientid: clientTwoID, Versionid: versionTwo, }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobTwoID, + Clientid: clientTwoID, Queryid: contextID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) - jobs, err = queries.ListQueryJobIDs(ctx, contextID) + clients, err = queries.ListQueryClientIDs(ctx, contextID) assert.NoError(t, err) - assert.ElementsMatch(t, []pgtype.UUID{jobOneID, jobTwoID}, jobs) - - clientTwoID, err := queries.CreateClient(ctx, "example_client_two") - assert.NoError(t, err) - jobThreeID, err := queries.CreateJob(ctx, clientTwoID) - assert.NoError(t, err) - versionThree, err := queries.AddLatestCollectorVersion(ctx, jobThreeID) - assert.NoError(t, err) - err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ - Jobid: jobThreeID, - Versionid: versionThree, - }) - assert.NoError(t, err) - err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobThreeID, - Queryid: contextID, - Addedversion: 1, - Name: "example_key", - }) - assert.NoError(t, err) - - jobs, err = queries.ListQueryJobIDs(ctx, contextID) - assert.NoError(t, err) - assert.ElementsMatch(t, []pgtype.UUID{jobOneID, jobTwoID, jobThreeID}, jobs) + assert.ElementsMatch(t, []pgtype.UUID{clientOneID, clientTwoID}, clients) jsonID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.NoError(t, err) @@ -615,14 +588,14 @@ func TestListQueryJobs(t *testing.T) { }) assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobOneID, + Clientid: clientOneID, Queryid: jsonID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) - jobs, err = queries.ListQueryJobIDs(ctx, jsonID) + clients, err = queries.ListQueryClientIDs(ctx, jsonID) assert.NoError(t, err) - assert.ElementsMatch(t, []pgtype.UUID{jobOneID}, jobs) + assert.ElementsMatch(t, []pgtype.UUID{clientOneID}, clients) } diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index 8d32766b..423b847f 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -56,7 +56,7 @@ func (q *Queries) AddResultDependency(ctx context.Context, arg *AddResultDepende const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one WITH doc as ( - SELECT id, jobId + SELECT id, clientId FROM documents WHERE id = $3 ) @@ -83,7 +83,7 @@ type GetResultValueWithVersionRow struct { // GetResultValueWithVersion // // WITH doc as ( -// SELECT id, jobId +// SELECT id, clientId // FROM documents // WHERE id = $3 // ) @@ -111,7 +111,7 @@ WITH reqQueries as ( and isInVersion($2, rq.addedVersion, rq.removedVersion) ), docs as ( - SELECT id, jobId + SELECT id, clientId FROM documents WHERE id = $3 ), @@ -121,8 +121,8 @@ codeVersions as ( mcv.minCleanVersion, mtv.minTextVersion FROM docs as d - JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId - JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId + JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId + JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId ), latestVersions AS ( SELECT @@ -171,7 +171,7 @@ type ListQueryRequirementValuesRow struct { // and isInVersion($2, rq.addedVersion, rq.removedVersion) // ), // docs as ( -// SELECT id, jobId +// SELECT id, clientId // FROM documents // WHERE id = $3 // ), @@ -181,8 +181,8 @@ type ListQueryRequirementValuesRow struct { // mcv.minCleanVersion, // mtv.minTextVersion // FROM docs as d -// JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId -// JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId +// JOIN currentCollectorMinTextVersions as mtv on mtv.clientId = d.clientId +// JOIN currentCollectorMinCleanVersions as mcv on mcv.clientId = d.clientId // ), // latestVersions AS ( // SELECT @@ -232,12 +232,12 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDocId :many WITH docs as ( - SELECT id, jobId from documents where id = $1 + SELECT id, clientId from documents where id = $1 ), unsyncedQueries AS ( SELECT DISTINCT dt.queryId, dt.requiredIds from docs as d - JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId + JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId JOIN currentTextEntries as cte on cte.documentId = d.id LEFT JOIN results as r on r.queryId = dt.queryId @@ -254,12 +254,12 @@ SELECT DISTINCT queryId FROM unsyncedQueries as baseuq // ListUnsyncedNoDepsQueriesByDocId // // WITH docs as ( -// SELECT id, jobId from documents where id = $1 +// SELECT id, clientId from documents where id = $1 // ), // unsyncedQueries AS ( // SELECT DISTINCT dt.queryId, dt.requiredIds // from docs as d -// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId +// JOIN collectorQueryDependencyTree as dt on d.clientId = dt.clientId // JOIN currentTextEntries as cte on cte.documentId = d.id // LEFT JOIN results as r // on r.queryId = dt.queryId diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index 71ecd02a..b444a6e3 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -43,32 +43,30 @@ func TestResults(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - issynced, err := queries.IsJobSynced(ctx, jobId) + issynced, err := queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, issynced) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash", + Clientid: clientId, + Hash: "example_hash", }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, issynced) - version, err := queries.AddLatestCollectorVersion(ctx, jobId) + version, err := queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: version, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, issynced) @@ -90,7 +88,7 @@ func TestResults(t *testing.T) { }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, issynced) @@ -102,7 +100,7 @@ func TestResults(t *testing.T) { }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, issynced) @@ -110,14 +108,14 @@ func TestResults(t *testing.T) { assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Queryid: jsonQueryID, Addedversion: 1, Name: "example_key", }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.False(t, issynced) @@ -131,7 +129,7 @@ func TestResults(t *testing.T) { }) assert.NoError(t, err) - issynced, err = queries.IsJobSynced(ctx, jobId) + issynced, err = queries.IsClientSynced(ctx, clientId) assert.NoError(t, err) assert.True(t, issynced) @@ -168,16 +166,14 @@ func TestResultValues(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash", + Clientid: clientId, + Hash: "example_hash", }) assert.NoError(t, err) documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash_two", + Clientid: clientId, + Hash: "example_hash_two", }) assert.NoError(t, err) @@ -227,11 +223,11 @@ func TestResultValues(t *testing.T) { assert.NoError(t, err) assert.Len(t, qResults, 0) - version, err := queries.AddLatestCollectorVersion(ctx, jobId) + version, err := queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: version, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) @@ -375,23 +371,21 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) - version, err := queries.AddLatestCollectorVersion(ctx, jobId) + version, err := queries.AddLatestCollectorVersion(ctx, clientId) assert.NoError(t, err) err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{ Versionid: version, - Jobid: jobId, + Clientid: clientId, }) assert.NoError(t, err) documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash", + Clientid: clientId, + Hash: "example_hash", }) assert.NoError(t, err) documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: "example_hash_two", + Clientid: clientId, + Hash: "example_hash_two", }) assert.NoError(t, err) @@ -421,7 +415,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) { assert.NoError(t, err) err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ - Jobid: jobId, + Clientid: clientId, Name: "example_name", Queryid: jsonQueryID, Addedversion: 1, diff --git a/internal/database/repository/text_test.go b/internal/database/repository/text_test.go index bcba4ca7..be6c280e 100644 --- a/internal/database/repository/text_test.go +++ b/internal/database/repository/text_test.go @@ -33,13 +33,11 @@ func TestTextExtraction(t *testing.T) { clientId, err := queries.CreateClient(ctx, "example_client") assert.NoError(t, err) - jobId, err := queries.CreateJob(ctx, clientId) - assert.NoError(t, err) hash := "example_hash" id, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: jobId, - Hash: hash, + Clientid: clientId, + Hash: hash, }) assert.NoError(t, err) assert.NotEmpty(t, id) diff --git a/internal/document/clean/clean_test.go b/internal/document/clean/clean_test.go index e7ce19ea..f3778bea 100644 --- a/internal/document/clean/clean_test.go +++ b/internal/document/clean/clean_test.go @@ -40,9 +40,9 @@ func TestClean(t *testing.T) { } doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } inloc := document.Location{ Bucket: "bucket_name", @@ -52,8 +52,8 @@ func TestClean(t *testing.T) { pool.ExpectQuery("name: GetDocument :one").WithArgs(dbid). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(dbid, database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(dbid, database.MustToDBUUID(doc.ClientID), doc.Hash), ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). diff --git a/internal/document/clean/create_test.go b/internal/document/clean/create_test.go index 3430b9f7..73c923dd 100644 --- a/internal/document/clean/create_test.go +++ b/internal/document/clean/create_test.go @@ -55,9 +55,9 @@ func TestCreate(t *testing.T) { } doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } inloc := document.Location{ Bucket: "bucket_name", @@ -71,8 +71,8 @@ func TestCreate(t *testing.T) { ) pool.ExpectQuery("name: GetDocument :one").WithArgs(dbid). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(dbid, database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(dbid, database.MustToDBUUID(doc.ClientID), doc.Hash), ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). diff --git a/internal/document/get.go b/internal/document/get.go index 42871ca3..02566fa7 100644 --- a/internal/document/get.go +++ b/internal/document/get.go @@ -15,8 +15,8 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) { } return &Document{ - ID: database.MustToUUID(doc.ID), - JobID: database.MustToUUID(doc.Jobid), - Hash: doc.Hash, + ID: database.MustToUUID(doc.ID), + ClientID: database.MustToUUID(doc.Clientid), + Hash: doc.Hash, }, nil } diff --git a/internal/document/get_test.go b/internal/document/get_test.go index 4c3b246f..04dbada6 100644 --- a/internal/document/get_test.go +++ b/internal/document/get_test.go @@ -28,15 +28,15 @@ func TestGet(t *testing.T) { svc := document.New(cfg) doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.ClientID), doc.Hash), ) adoc, err := svc.Get(ctx, doc.ID) diff --git a/internal/document/init/create.go b/internal/document/init/create.go index 12b2f7f6..f80c9f37 100644 --- a/internal/document/init/create.go +++ b/internal/document/init/create.go @@ -19,7 +19,7 @@ import ( ) type Create struct { - JobID uuid.UUID + ClientID uuid.UUID Location document.Location Bucket string Hash string @@ -53,15 +53,15 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) { type createDocumentParams struct { ID *pgtype.UUID - JobID pgtype.UUID + ClientID pgtype.UUID Hash string Location document.Location } func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) { idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{ - Jobid: database.MustToDBUUID(doc.JobID), - Hash: doc.Hash, + Clientid: database.MustToDBUUID(doc.ClientID), + Hash: doc.Hash, }) var docID *pgtype.UUID if err != nil && !errors.Is(err, sql.ErrNoRows) { @@ -72,7 +72,7 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu return &createDocumentParams{ ID: docID, - JobID: database.MustToDBUUID(doc.JobID), + ClientID: database.MustToDBUUID(doc.ClientID), Hash: doc.Hash, Location: doc.Location, }, nil @@ -85,17 +85,17 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams var dbid pgtype.UUID if params.ID == nil { createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ - Jobid: params.JobID, - Hash: params.Hash, + Clientid: params.ClientID, + Hash: params.Hash, }) if err != nil { return err } - slog.Debug("document created", "id", createid.String(), "job", params.JobID.String()) + slog.Debug("document created", "id", createid.String(), "client", params.ClientID.String()) dbid = createid } else { - slog.Debug("document exists", "id", params.ID.String(), "job", params.JobID.String()) + slog.Debug("document exists", "id", params.ID.String(), "client", params.ClientID.String()) dbid = *params.ID } @@ -120,19 +120,14 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams return id, nil } -func (s *Service) GetJobIDFromKey(key string) (uuid.UUID, error) { - re := regexp.MustCompile(`^(.+)/(.+)/.+$`) +func (s *Service) GetClientIDFromKey(key string) (uuid.UUID, error) { + re := regexp.MustCompile(`^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/.+$`) match := re.FindStringSubmatch(key) - if match == nil || len(match) != 3 { - return uuid.Nil, fmt.Errorf("no job id match found in key") + if len(match) < 2 { + return uuid.Nil, fmt.Errorf("no client id match found in key") } - _, err := uuid.Parse(match[1]) - if err != nil { - return uuid.Nil, err - } - - id, err := uuid.Parse(match[2]) + id, err := uuid.Parse(match[1]) if err != nil { return uuid.Nil, err } diff --git a/internal/document/init/create_test.go b/internal/document/init/create_test.go index c619e2df..dccf49bc 100644 --- a/internal/document/init/create_test.go +++ b/internal/document/init/create_test.go @@ -9,7 +9,6 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" - "queryorchestration/internal/job" queuemock "queryorchestration/mocks/queue" "github.com/stretchr/testify/require" @@ -35,25 +34,22 @@ func TestCreate(t *testing.T) { svc := New(cfg) - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - } + clientId := uuid.New() doc := document.Document{ - ID: uuid.New(), - JobID: j.ID, - Hash: "example_hash", + ID: uuid.New(), + ClientID: clientId, + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", Key: "/i/am/here", } - pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows( + pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"id"}), ) pool.ExpectBegin() - pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash). + pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.ClientID), doc.Hash). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(doc.ID)), @@ -63,16 +59,16 @@ func TestCreate(t *testing.T) { pool.ExpectCommit() pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.ClientID), doc.Hash), ) - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), + AddRow(database.MustToDBUUID(clientId), database.MustToDBUUID(clientId), true), ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)).WillReturnRows( + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(clientId)).WillReturnRows( pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), + AddRow(database.MustToDBUUID(clientId), "client_name", true), ) mockSQS.EXPECT(). @@ -86,7 +82,7 @@ func TestCreate(t *testing.T) { Return(&sqs.SendMessageOutput{}, nil) id, err := svc.Create(ctx, &Create{ - JobID: doc.JobID, + ClientID: doc.ClientID, Location: location, Hash: doc.Hash, }) @@ -106,26 +102,26 @@ func TestGetCreateParams(t *testing.T) { svc := New(cfg) doc := document.Document{ - JobID: uuid.New(), - Hash: "example_hash", + ClientID: uuid.New(), + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", Key: "/i/am/here", } - pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows( + pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"id"}), ) params, err := svc.getCreateParams(ctx, &Create{ - JobID: doc.JobID, + ClientID: doc.ClientID, Location: location, Hash: doc.Hash, }) assert.NoError(t, err) assert.Equal(t, &createDocumentParams{ - JobID: database.MustToDBUUID(doc.JobID), + ClientID: database.MustToDBUUID(doc.ClientID), Hash: doc.Hash, Location: location, }, params) @@ -143,22 +139,22 @@ func TestGetCreateParamsExisting(t *testing.T) { svc := New(cfg) doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", Key: "/i/am/here", } - pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows( + pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.ClientID)).WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(doc.ID)), ) params, err := svc.getCreateParams(ctx, &Create{ - JobID: doc.JobID, + ClientID: doc.ClientID, Location: location, Hash: doc.Hash, }) @@ -166,7 +162,7 @@ func TestGetCreateParamsExisting(t *testing.T) { dbid := database.MustToDBUUID(doc.ID) assert.Equal(t, &createDocumentParams{ ID: &dbid, - JobID: database.MustToDBUUID(doc.JobID), + ClientID: database.MustToDBUUID(doc.ClientID), Hash: doc.Hash, Location: location, }, params) @@ -184,19 +180,19 @@ func TestGetCreateParamsCurrentDocErr(t *testing.T) { svc := New(cfg) doc := document.Document{ - JobID: uuid.New(), - Hash: "example_hash", + ClientID: uuid.New(), + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", Key: "/i/am/here", } - pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)). + pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.ClientID)). WillReturnError(errors.New("db err")) _, err = svc.getCreateParams(ctx, &Create{ - JobID: doc.JobID, + ClientID: doc.ClientID, Location: location, }) assert.Error(t, err) @@ -214,9 +210,9 @@ func TestSubmitCreate(t *testing.T) { svc := New(cfg) doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", @@ -224,7 +220,7 @@ func TestSubmitCreate(t *testing.T) { } pool.ExpectBegin() - pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash). + pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.ClientID), doc.Hash). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(database.MustToDBUUID(doc.ID)), @@ -234,7 +230,7 @@ func TestSubmitCreate(t *testing.T) { pool.ExpectCommit() id, err := svc.submitCreate(ctx, &createDocumentParams{ - JobID: database.MustToDBUUID(doc.JobID), + ClientID: database.MustToDBUUID(doc.ClientID), Location: location, Hash: doc.Hash, }) @@ -254,9 +250,9 @@ func TestSubmitCreateExists(t *testing.T) { svc := New(cfg) doc := document.Document{ - ID: uuid.New(), - JobID: uuid.New(), - Hash: "example_hash", + ID: uuid.New(), + ClientID: uuid.New(), + Hash: "example_hash", } location := document.Location{ Bucket: "example_bucket", @@ -271,7 +267,7 @@ func TestSubmitCreateExists(t *testing.T) { docid := database.MustToDBUUID(doc.ID) id, err := svc.submitCreate(ctx, &createDocumentParams{ ID: &docid, - JobID: database.MustToDBUUID(doc.JobID), + ClientID: database.MustToDBUUID(doc.ClientID), Location: location, Hash: doc.Hash, }) @@ -279,23 +275,27 @@ func TestSubmitCreateExists(t *testing.T) { assert.Equal(t, doc.ID, id) } -func TestGetJobIDFromKey(t *testing.T) { +func TestGetClientIDFromKey(t *testing.T) { svc := Service{} - _, err := svc.GetJobIDFromKey("") + _, err := svc.GetClientIDFromKey("") assert.Error(t, err) - _, err = svc.GetJobIDFromKey("jkhaskhweuifhwhieh") + _, err = svc.GetClientIDFromKey("jkhaskhweuifhwhieh") assert.Error(t, err) - _, err = svc.GetJobIDFromKey("aaa/bbb/ccc") + _, err = svc.GetClientIDFromKey("aaa/bbb/ccc") assert.Error(t, err) id := uuid.New() - _, err = svc.GetJobIDFromKey(fmt.Sprintf("aaa/%s/bbb", id.String())) + _, err = svc.GetClientIDFromKey(fmt.Sprintf("aaa/%s/bbb", id.String())) assert.Error(t, err) - aid, err := svc.GetJobIDFromKey(fmt.Sprintf("%s/%s/bbb", uuid.NewString(), id.String())) + aid, err := svc.GetClientIDFromKey(fmt.Sprintf("%s/%s/bbb", id.String(), uuid.NewString())) + assert.NoError(t, err) + assert.Equal(t, id, aid) + + aid, err = svc.GetClientIDFromKey(fmt.Sprintf("%s/bbb", id.String())) assert.NoError(t, err) assert.Equal(t, id, aid) } diff --git a/internal/document/list.go b/internal/document/list.go index a777eb46..21271802 100644 --- a/internal/document/list.go +++ b/internal/document/list.go @@ -14,8 +14,8 @@ type DocumentInList struct { Key string } -func (s *Service) ListByJobId(ctx context.Context, id uuid.UUID) ([]*DocumentInList, error) { - documents, err := s.cfg.GetDBQueries().ListDocumentsByJobId(ctx, database.MustToDBUUID(id)) +func (s *Service) ListByClientId(ctx context.Context, id uuid.UUID) ([]*DocumentInList, error) { + documents, err := s.cfg.GetDBQueries().ListDocumentsByClientId(ctx, database.MustToDBUUID(id)) if err != nil { return nil, err } diff --git a/internal/document/list_test.go b/internal/document/list_test.go index 49d33c40..8e88df36 100644 --- a/internal/document/list_test.go +++ b/internal/document/list_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestListByJobId(t *testing.T) { +func TestListByClientId(t *testing.T) { ctx := context.Background() pool, err := pgxmock.NewPool() @@ -26,7 +26,7 @@ func TestListByJobId(t *testing.T) { svc := document.New(cfg) - jobId := uuid.New() + clientId := uuid.New() doc := []*document.DocumentInList{ { ID: uuid.New(), @@ -35,13 +35,13 @@ func TestListByJobId(t *testing.T) { }, } - pool.ExpectQuery("name: ListDocumentsByJobId :many").WithArgs(database.MustToDBUUID(jobId)). + pool.ExpectQuery("name: ListDocumentsByClientId :many").WithArgs(database.MustToDBUUID(clientId)). WillReturnRows( pgxmock.NewRows([]string{"id", "bucket", "key"}). AddRow(database.MustToDBUUID(doc[0].ID), doc[0].Bucket, doc[0].Key), ) - adoc, err := svc.ListByJobId(ctx, jobId) + adoc, err := svc.ListByClientId(ctx, clientId) assert.NoError(t, err) assert.Equal(t, doc, adoc) } diff --git a/internal/document/service.go b/internal/document/service.go index 1db1c8c3..293e2737 100644 --- a/internal/document/service.go +++ b/internal/document/service.go @@ -12,9 +12,9 @@ type Location struct { } type Document struct { - ID uuid.UUID - JobID uuid.UUID - Hash string + ID uuid.UUID + ClientID uuid.UUID + Hash string } type Service struct { diff --git a/internal/document/sync/service.go b/internal/document/sync/service.go index c2326615..b022eadd 100644 --- a/internal/document/sync/service.go +++ b/internal/document/sync/service.go @@ -1,15 +1,15 @@ package documentsync import ( + "queryorchestration/internal/client" "queryorchestration/internal/document" - "queryorchestration/internal/job" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/queue/documentclean" ) type Services struct { Document *document.Service - Job *job.Service + Client *client.Service } type ConfigProvider interface { diff --git a/internal/document/sync/sync.go b/internal/document/sync/sync.go index 935f24b5..d32da877 100644 --- a/internal/document/sync/sync.go +++ b/internal/document/sync/sync.go @@ -16,7 +16,7 @@ func (s *Service) Sync(ctx context.Context, id uuid.UUID) error { return err } - j, err := s.svc.Job.Get(ctx, doc.JobID) + j, err := s.svc.Client.Get(ctx, doc.ClientID) if err != nil { return err } diff --git a/internal/document/sync/sync_test.go b/internal/document/sync/sync_test.go index 10f9a66f..153485b2 100644 --- a/internal/document/sync/sync_test.go +++ b/internal/document/sync/sync_test.go @@ -10,7 +10,6 @@ import ( "queryorchestration/internal/database/repository" "queryorchestration/internal/document" documentsync "queryorchestration/internal/document/sync" - "queryorchestration/internal/job" queuemock "queryorchestration/mocks/queue" "github.com/stretchr/testify/require" @@ -36,34 +35,27 @@ func TestSync(t *testing.T) { svc := documentsync.New(cfg, &documentsync.Services{ Document: document.New(cfg), - Job: job.New(cfg, &job.Services{ - Client: client.New(cfg), - }), + Client: client.New(cfg, &client.Services{}), }) - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: true, + j := client.Client{ + ID: uuid.New(), + CanSync: true, } doc := document.Document{ - ID: uuid.New(), - JobID: j.ID, - Hash: "example_hash", + ID: uuid.New(), + ClientID: j.ID, + Hash: "example_hash", } pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "hash"}). - AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash), + pgxmock.NewRows([]string{"id", "clientId", "hash"}). + AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.ClientID), doc.Hash), ) - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)).WillReturnRows( + pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), + AddRow(database.MustToDBUUID(j.ID), "client_name", true), ) mockSQS.EXPECT(). diff --git a/internal/job/collector/get_test.go b/internal/job/collector/get_test.go deleted file mode 100644 index 32e286e4..00000000 --- a/internal/job/collector/get_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package collector_test - -import ( - "context" - "fmt" - "testing" - - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - resultprocessor "queryorchestration/internal/query/result/processor" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGet(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := collector.New(cfg, &collector.Services{}) - - minCleanV := int32(2) - minTextV := int32(4) - ogc := collector.Collector{ - JobID: uuid.New(), - MinCleanVersion: minCleanV, - MinTextVersion: minTextV, - Fields: map[string]uuid.UUID{ - "example_key": uuid.New(), - }, - } - - pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), - ) - - coll, err := svc.Get(ctx, ogc.JobID) - assert.NoError(t, err) - assert.EqualExportedValues(t, ogc, *coll) -} - -func TestGetByJobID(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := collector.New(cfg, &collector.Services{}) - - minCleanV := int32(2) - minTextV := int32(4) - ogc := collector.Collector{ - JobID: uuid.New(), - MinCleanVersion: minCleanV, - MinTextVersion: minTextV, - Fields: map[string]uuid.UUID{ - "example_key": uuid.New(), - }, - } - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), - ) - - coll, err := svc.GetByJobID(ctx, ogc.JobID) - assert.NoError(t, err) - assert.EqualExportedValues(t, ogc, *coll) -} - -func TestListQueries(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := collector.New(cfg, &collector.Services{}) - - jobId := uuid.New() - ogc := []*resultprocessor.Query{ - { - ID: uuid.New(), - Type: resultprocessor.TypeContextFull, - Version: 2, - }, - } - - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(jobId)). - WillReturnRows( - pgxmock.NewRows([]string{"jobId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(database.MustToDBUUID(jobId), database.MustToDBUUID(ogc[0].ID), repository.QuerytypeContextFull, int32(2), nil), - ) - - qs, err := svc.ListQueries(ctx, jobId) - assert.NoError(t, err) - assert.EqualExportedValues(t, ogc, qs) -} diff --git a/internal/job/create.go b/internal/job/create.go deleted file mode 100644 index 924bdcde..00000000 --- a/internal/job/create.go +++ /dev/null @@ -1,28 +0,0 @@ -package job - -import ( - "context" - - "queryorchestration/internal/database" - "queryorchestration/internal/job/collector" - - "github.com/google/uuid" -) - -func (s *Service) Create(ctx context.Context, clientID uuid.UUID) (uuid.UUID, error) { - did, err := s.cfg.GetDBQueries().CreateJob(ctx, database.MustToDBUUID(clientID)) - if err != nil { - return uuid.Nil, err - } - - id := database.MustToUUID(did) - - err = s.svc.Collector.Create(ctx, &collector.CreateParams{ - JobID: id, - }) - if err != nil { - return uuid.Nil, err - } - - return id, nil -} diff --git a/internal/job/create_test.go b/internal/job/create_test.go deleted file mode 100644 index 0835a2a8..00000000 --- a/internal/job/create_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package job_test - -import ( - "context" - "testing" - - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestCreate(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{ - Collector: collector.New(cfg, &collector.Services{}), - }) - - job := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - } - - pool.ExpectQuery("name: CreateJob :one").WithArgs(database.MustToDBUUID(job.ClientID)).WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(job.ID)), - ) - pool.ExpectBeginTx(pgx.TxOptions{}) - pool.ExpectQuery("name: AddLatestCollectorVersion :one").WithArgs(database.MustToDBUUID(job.ID)).WillReturnRows( - pgxmock.NewRows([]string{"version"}). - AddRow(int32(1)), - ) - pool.ExpectExec("name: SetActiveCollectorVersion :exec").WithArgs(database.MustToDBUUID(job.ID), int32(1)). - WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectCommit() - - aid, err := svc.Create(ctx, job.ClientID) - assert.NoError(t, err) - assert.Equal(t, job.ID, aid) -} diff --git a/internal/job/get.go b/internal/job/get.go deleted file mode 100644 index c7105a74..00000000 --- a/internal/job/get.go +++ /dev/null @@ -1,32 +0,0 @@ -package job - -import ( - "context" - - "queryorchestration/internal/database" - - "github.com/google/uuid" -) - -func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Job, error) { - job, err := s.cfg.GetDBQueries().GetJob(ctx, database.MustToDBUUID(id)) - if err != nil { - return nil, err - } - - client, err := s.svc.Client.Get(ctx, database.MustToUUID(job.Clientid)) - if err != nil { - return nil, err - } - - canSync := job.Cansync - if !client.CanSync { - canSync = false - } - - return &Job{ - ID: id, - ClientID: database.MustToUUID(job.Clientid), - CanSync: canSync, - }, nil -} diff --git a/internal/job/get_test.go b/internal/job/get_test.go deleted file mode 100644 index 48113835..00000000 --- a/internal/job/get_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package job_test - -import ( - "context" - "testing" - - "queryorchestration/internal/client" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/serviceconfig" - - "github.com/stretchr/testify/require" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestGet(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{ - Collector: collector.New(cfg, &collector.Services{}), - Client: client.New(cfg), - }) - - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: true, - } - - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", false), - ) - - ajob, err := svc.Get(ctx, j.ID) - assert.NoError(t, err) - assert.EqualExportedValues(t, &job.Job{ - ID: j.ID, - ClientID: j.ClientID, - CanSync: false, - }, ajob) - - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - - ajob, err = svc.Get(ctx, j.ID) - assert.NoError(t, err) - assert.EqualExportedValues(t, &job.Job{ - ID: j.ID, - ClientID: j.ClientID, - CanSync: true, - }, ajob) -} diff --git a/internal/job/service.go b/internal/job/service.go deleted file mode 100644 index 964be363..00000000 --- a/internal/job/service.go +++ /dev/null @@ -1,32 +0,0 @@ -package job - -import ( - "queryorchestration/internal/client" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" -) - -type Job struct { - ID uuid.UUID - ClientID uuid.UUID - CanSync bool -} - -type Services struct { - Collector *collector.Service - Client *client.Service -} - -type Service struct { - cfg serviceconfig.ConfigProvider - svc *Services -} - -func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { - return &Service{ - cfg, - svc, - } -} diff --git a/internal/job/service_test.go b/internal/job/service_test.go deleted file mode 100644 index 1120f7e2..00000000 --- a/internal/job/service_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package job_test - -import ( - "testing" - - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job" - "queryorchestration/internal/serviceconfig" - - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestService(t *testing.T) { - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{}) - assert.NotNil(t, svc) -} diff --git a/internal/job/status.go b/internal/job/status.go deleted file mode 100644 index af5baca6..00000000 --- a/internal/job/status.go +++ /dev/null @@ -1,31 +0,0 @@ -package job - -import ( - "context" - - "queryorchestration/internal/database" - - "github.com/google/uuid" -) - -type Status string - -const ( - NOT_SYNCED Status = "not_synced" - IN_SYNC Status = "in_sync" -) - -func (s *Service) GetStatus(ctx context.Context, id uuid.UUID) (*Status, error) { - issynced, err := s.cfg.GetDBQueries().IsJobSynced(ctx, database.MustToDBUUID(id)) - if err != nil { - return nil, err - } - - status := NOT_SYNCED - switch issynced { - case true: - status = IN_SYNC - } - - return &status, nil -} diff --git a/internal/job/status_test.go b/internal/job/status_test.go deleted file mode 100644 index 078bcf33..00000000 --- a/internal/job/status_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package job_test - -import ( - "context" - "errors" - "testing" - - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGetStatus(t *testing.T) { - t.Run("is synced", func(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{}) - - jobId := uuid.New() - - pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows( - pgxmock.NewRows([]string{"issynced"}). - AddRow(true), - ) - - issynced, err := svc.GetStatus(ctx, jobId) - assert.NoError(t, err) - assert.Equal(t, job.IN_SYNC, *issynced) - }) - t.Run("not synced", func(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{}) - - jobId := uuid.New() - - pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)).WillReturnRows( - pgxmock.NewRows([]string{"issynced"}). - AddRow(false), - ) - - issynced, err := svc.GetStatus(ctx, jobId) - assert.NoError(t, err) - assert.Equal(t, job.NOT_SYNCED, *issynced) - }) - t.Run("db error", func(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{}) - - jobId := uuid.New() - - errStr := "db fail" - pool.ExpectQuery("name: IsJobSynced :one").WithArgs(database.MustToDBUUID(jobId)). - WillReturnError(errors.New(errStr)) - - _, err = svc.GetStatus(ctx, jobId) - assert.EqualError(t, err, errStr) - }) -} diff --git a/internal/job/update.go b/internal/job/update.go deleted file mode 100644 index 8399d1dd..00000000 --- a/internal/job/update.go +++ /dev/null @@ -1,96 +0,0 @@ -package job - -import ( - "context" - "errors" - "log/slog" - - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/validation" - - "github.com/google/uuid" -) - -type Update struct { - ID uuid.UUID - CanSync *bool -} - -func (s *Service) Update(ctx context.Context, update *Update) error { - if update == nil { - return errors.New("Update required") - } - - current, err := s.Get(ctx, update.ID) - if err != nil { - return err - } - - err = s.normalizeUpdate(ctx, current, update) - if err != nil { - return err - } - - err = s.submitUpdate(ctx, update) - if err != nil { - return err - } - - return nil -} - -func (s *Service) normalizeUpdate(ctx context.Context, current *Job, update *Update) error { - err := s.normalizeCanSync(ctx, current, update) - if err != nil { - return err - } - - if validation.AreAllPointersNilExcept(update) { - return errors.New("no changes") - } - - return nil -} - -func (s *Service) submitUpdate(ctx context.Context, update *Update) error { - err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error { - if update.CanSync != nil { - err := q.AddJobCanSync(ctx, &repository.AddJobCanSyncParams{ - Jobid: database.MustToDBUUID(update.ID), - Cansync: *update.CanSync, - }) - if err != nil { - return err - } - } - - slog.Debug("job updated", "update", *update) - - return nil - }) - - return err -} - -func (s *Service) normalizeCanSync(ctx context.Context, current *Job, update *Update) error { - if update.CanSync == nil { - return nil - } else if current.CanSync == *update.CanSync { - update.CanSync = nil - return nil - } - - if !*update.CanSync { - return nil - } - - client, err := s.svc.Client.Get(ctx, current.ClientID) - if err != nil { - return err - } else if !client.CanSync { - return errors.New("client not allowing sync") - } - - return nil -} diff --git a/internal/job/update_test.go b/internal/job/update_test.go deleted file mode 100644 index 16237f07..00000000 --- a/internal/job/update_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package job_test - -import ( - "context" - "testing" - - "queryorchestration/internal/client" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestUpdate(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := job.New(cfg, &job.Services{ - Client: client.New(cfg), - }) - - j := job.Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: true, - } - ucs := !j.CanSync - u := &job.Update{ - ID: j.ID, - CanSync: &ucs, - } - - pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows( - pgxmock.NewRows([]string{"id", "clientId", "canSync"}). - AddRow(database.MustToDBUUID(j.ID), database.MustToDBUUID(j.ClientID), j.CanSync), - ) - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - pool.ExpectBegin() - pool.ExpectExec("name: AddJobCanSync :exec").WithArgs(*u.CanSync, database.MustToDBUUID(j.ID)). - WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectCommit() - - err = svc.Update(ctx, u) - assert.NoError(t, err) -} diff --git a/internal/job/updateprivate_test.go b/internal/job/updateprivate_test.go deleted file mode 100644 index 4e7c254b..00000000 --- a/internal/job/updateprivate_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package job - -import ( - "context" - "testing" - - "queryorchestration/internal/client" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestNormalizeCanSync(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := New(cfg, &Services{ - Client: client.New(cfg), - }) - - j := Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: false, - } - - update := Update{ - ID: j.ID, - } - - err = svc.normalizeCanSync(ctx, &j, &update) - assert.NoError(t, err) - assert.Nil(t, update.CanSync) - - cs := false - update.CanSync = &cs - err = svc.normalizeCanSync(ctx, &j, &update) - assert.NoError(t, err) - assert.Nil(t, update.CanSync) - - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", false), - ) - - cs = true - update.CanSync = &cs - err = svc.normalizeCanSync(ctx, &j, &update) - assert.Error(t, err) - - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - - cs = false - update.CanSync = &cs - err = svc.normalizeCanSync(ctx, &j, &update) - assert.NoError(t, err) - assert.Nil(t, update.CanSync) - - pool.ExpectQuery("name: GetClient :one").WithArgs(database.MustToDBUUID(j.ClientID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "name", "canSync"}). - AddRow(database.MustToDBUUID(j.ClientID), "client_name", true), - ) - - cs = true - update.CanSync = &cs - err = svc.normalizeCanSync(ctx, &j, &update) - assert.NoError(t, err) - assert.True(t, *update.CanSync) - - j.CanSync = true - cs = true - update.CanSync = &cs - err = svc.normalizeCanSync(ctx, &j, &update) - assert.NoError(t, err) - assert.Nil(t, update.CanSync) -} - -func TestNormalizeUpdate(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := New(cfg, &Services{ - Client: client.New(cfg), - }) - - j := Job{ - ID: uuid.New(), - ClientID: uuid.New(), - CanSync: true, - } - - ucs := false - update := Update{ - ID: j.ID, - CanSync: &ucs, - } - - err = svc.normalizeUpdate(ctx, &j, &update) - assert.NoError(t, err) - ucs = !ucs - assert.EqualExportedValues(t, Update{ID: update.ID, CanSync: &ucs}, update) -} - -func TestSubmitUpdate(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - require.NoError(t, err) - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := Service{ - cfg: cfg, - } - - j := Job{ - ID: uuid.New(), - } - u := &Update{ - ID: j.ID, - } - - pool.ExpectBegin() - pool.ExpectCommit() - - err = svc.submitUpdate(ctx, u) - assert.NoError(t, err) - - ucs := true - u = &Update{ - ID: j.ID, - CanSync: &ucs, - } - - pool.ExpectBegin() - pool.ExpectExec("name: AddJobCanSync :exec").WithArgs(*u.CanSync, database.MustToDBUUID(j.ID)). - WillReturnResult(pgxmock.NewResult("", 1)) - pool.ExpectCommit() - - err = svc.submitUpdate(ctx, u) - assert.NoError(t, err) -} diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go index 77cf57bf..562be2da 100644 --- a/internal/query/result/processor/parse_test.go +++ b/internal/query/result/processor/parse_test.go @@ -14,7 +14,7 @@ import ( func TestParseDBCollectorQuery(t *testing.T) { dbResult := repository.Collectorquerydependencytree{ - Jobid: pgtype.UUID{}, + Clientid: pgtype.UUID{}, Queryid: pgtype.UUID{}, Requiredids: []pgtype.UUID{}, Type: repository.QuerytypeJsonExtractor, diff --git a/internal/query/test/service.go b/internal/query/test/service.go index 60cd6557..99085cc8 100644 --- a/internal/query/test/service.go +++ b/internal/query/test/service.go @@ -1,8 +1,8 @@ package querytest import ( + "queryorchestration/internal/collector" "queryorchestration/internal/document" - "queryorchestration/internal/job/collector" "queryorchestration/internal/query/result" "queryorchestration/internal/serviceconfig" ) diff --git a/internal/query/test/test_test.go b/internal/query/test/test_test.go index 6b8bdbd3..0a71555d 100644 --- a/internal/query/test/test_test.go +++ b/internal/query/test/test_test.go @@ -4,12 +4,12 @@ import ( "context" "testing" + "queryorchestration/internal/collector" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" cleanversion "queryorchestration/internal/document/clean/version" textversion "queryorchestration/internal/document/text/version" - "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" querytest "queryorchestration/internal/query/test" diff --git a/internal/query/versionsync/service.go b/internal/query/versionsync/service.go index 11b4a2b3..c5905be0 100644 --- a/internal/query/versionsync/service.go +++ b/internal/query/versionsync/service.go @@ -2,12 +2,12 @@ package queryversionsync import ( "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" ) type ConfigProvider interface { serviceconfig.ConfigProvider - jobsync.ConfigProvider + clientsync.ConfigProvider } type Service struct { diff --git a/internal/query/versionsync/sync.go b/internal/query/versionsync/sync.go index e4eaef15..7e85c939 100644 --- a/internal/query/versionsync/sync.go +++ b/internal/query/versionsync/sync.go @@ -5,7 +5,7 @@ import ( "database/sql" "errors" - jobsyncrunner "queryorchestration/api/jobSyncRunner" + clientsyncrunner "queryorchestration/api/clientSyncRunner" "queryorchestration/internal/database" "queryorchestration/internal/serviceconfig/queue" @@ -13,17 +13,17 @@ import ( ) func (s *Service) Sync(ctx context.Context, id uuid.UUID) error { - jobIds, err := s.cfg.GetDBQueries().ListQueryJobIDs(ctx, database.MustToDBUUID(id)) + clientIds, err := s.cfg.GetDBQueries().ListQueryClientIDs(ctx, database.MustToDBUUID(id)) if err != nil && errors.Is(err, sql.ErrNoRows) { return nil } else if err != nil { return err } - for _, id := range jobIds { + for _, id := range clientIds { err := s.cfg.SendToQueue(ctx, &queue.SendParams{ - QueueURL: s.cfg.GetJobSyncURL(), - Body: jobsyncrunner.Body{ + QueueURL: s.cfg.GetClientSyncURL(), + Body: clientsyncrunner.Body{ ID: database.MustToUUID(id), }, }) diff --git a/internal/query/versionsync/sync_test.go b/internal/query/versionsync/sync_test.go index c366c8f8..c2851e31 100644 --- a/internal/query/versionsync/sync_test.go +++ b/internal/query/versionsync/sync_test.go @@ -8,7 +8,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/serviceconfig" - "queryorchestration/internal/serviceconfig/queue/jobsync" + "queryorchestration/internal/serviceconfig/queue/clientsync" queuemock "queryorchestration/mocks/queue" "github.com/stretchr/testify/require" @@ -22,7 +22,7 @@ import ( type QueryVersionSyncConfig struct { serviceconfig.BaseConfig - jobsync.JobSyncConfig + clientsync.ClientSyncConfig } func TestSync(t *testing.T) { @@ -34,28 +34,28 @@ func TestSync(t *testing.T) { cfg.DBQueries = repository.New(pool) mockSQS := queuemock.NewMockSQSClient(t) cfg.QueueClient = mockSQS - cfg.JobSyncURL = "/i/am/here" + cfg.ClientSyncURL = "/i/am/here" svc := Service{cfg} queryId := uuid.New() - jobIds := []uuid.UUID{ + clientIds := []uuid.UUID{ uuid.New(), uuid.New(), } - pool.ExpectQuery("name: ListQueryJobIDs :many").WithArgs(database.MustToDBUUID(queryId)). + pool.ExpectQuery("name: ListQueryClientIDs :many").WithArgs(database.MustToDBUUID(queryId)). WillReturnRows( - pgxmock.NewRows([]string{"jobId"}). - AddRow(database.MustToDBUUID(jobIds[0])). - AddRow(database.MustToDBUUID(jobIds[1])), + pgxmock.NewRows([]string{"clientId"}). + AddRow(database.MustToDBUUID(clientIds[0])). + AddRow(database.MustToDBUUID(clientIds[1])), ) mockSQS.EXPECT(). SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", jobIds[0].String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[0].String()) }), mock.Anything, ). @@ -64,7 +64,7 @@ func TestSync(t *testing.T) { SendMessage( mock.Anything, mock.MatchedBy(func(in *sqs.SendMessageInput) bool { - return *in.QueueUrl == cfg.JobSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", jobIds[1].String()) + return *in.QueueUrl == cfg.ClientSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", clientIds[1].String()) }), mock.Anything, ). diff --git a/internal/serviceconfig/observability/prometheus/prometheus.md b/internal/serviceconfig/observability/prometheus/prometheus.md index c5174166..724a48e3 100644 --- a/internal/serviceconfig/observability/prometheus/prometheus.md +++ b/internal/serviceconfig/observability/prometheus/prometheus.md @@ -2,13 +2,13 @@ All instructions assume you are already in the `devbox shell` -To run various services and inspect prometheus metrics first run the -`task compose:up:test` to start the db and prometheus services. +To run various services and inspect prometheus metrics first run the +`task compose:up:test` to start the db and prometheus services. -Then you can run your service that exposed the metrics. +Then you can run your service that exposed the metrics. -## Testing the API service -Start the API server on the host like +## Testing the API service +Start the API server on the host like `PGPORT=5430 go run ./cmd/queryService` Once started you will be able to see the raw metrics at @@ -23,11 +23,11 @@ http://localhost:8080/swagger/index.html For example to run the query endpoint (to see query related metrics generated) go to http://localhost:8080/query -## Sample prometheus queries for the console -Duration of the /query api call (from Echo metrics) +## Sample prometheus queries for the console +Duration of the /query api call (from Echo metrics) `echo_request_duration_seconds_sum{code="200", host="localhost:8080", instance="host.docker.internal:8080", job="api", method="GET", url="/query"}` -Memory allocation rate (from Go metrics) +Memory allocation rate (from Go metrics) `rate(go_memstats_alloc_bytes_total[5m])` Time since last GC @@ -37,4 +37,4 @@ Gc overhead `rate(go_gc_duration_seconds_sum[5m])` -More examples TBD. +More examples TBD. diff --git a/internal/serviceconfig/queue/clientsync/config.go b/internal/serviceconfig/queue/clientsync/config.go new file mode 100644 index 00000000..a8267675 --- /dev/null +++ b/internal/serviceconfig/queue/clientsync/config.go @@ -0,0 +1,13 @@ +package clientsync + +type ClientSyncConfig struct { + ClientSyncURL string `env:"CLIENT_SYNC_URL,required,notEmpty"` +} + +func (c *ClientSyncConfig) GetClientSyncURL() string { + return c.ClientSyncURL +} + +type ConfigProvider interface { + GetClientSyncURL() string +} diff --git a/internal/serviceconfig/queue/clientsync/config_test.go b/internal/serviceconfig/queue/clientsync/config_test.go new file mode 100644 index 00000000..a93d6789 --- /dev/null +++ b/internal/serviceconfig/queue/clientsync/config_test.go @@ -0,0 +1,21 @@ +package clientsync_test + +import ( + "testing" + + clientsync "queryorchestration/internal/serviceconfig/queue/clientsync" + + "github.com/stretchr/testify/assert" +) + +func TestGetClientSyncURL(t *testing.T) { + cfg := clientsync.ClientSyncConfig{} + + name := cfg.GetClientSyncURL() + assert.Equal(t, "", name) + + cfg.ClientSyncURL = "name" + name = cfg.GetClientSyncURL() + assert.Equal(t, "name", name) + assert.Equal(t, cfg.ClientSyncURL, name) +} diff --git a/internal/serviceconfig/queue/jobsync/config.go b/internal/serviceconfig/queue/jobsync/config.go deleted file mode 100644 index c9f2ffa5..00000000 --- a/internal/serviceconfig/queue/jobsync/config.go +++ /dev/null @@ -1,13 +0,0 @@ -package jobsync - -type JobSyncConfig struct { - JobSyncURL string `env:"JOB_SYNC_URL,required,notEmpty"` -} - -func (c *JobSyncConfig) GetJobSyncURL() string { - return c.JobSyncURL -} - -type ConfigProvider interface { - GetJobSyncURL() string -} diff --git a/internal/serviceconfig/queue/jobsync/config_test.go b/internal/serviceconfig/queue/jobsync/config_test.go deleted file mode 100644 index bf856387..00000000 --- a/internal/serviceconfig/queue/jobsync/config_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package jobsync_test - -import ( - "testing" - - "queryorchestration/internal/serviceconfig/queue/jobsync" - - "github.com/stretchr/testify/assert" -) - -func TestGetJobSyncURL(t *testing.T) { - cfg := jobsync.JobSyncConfig{} - - name := cfg.GetJobSyncURL() - assert.Equal(t, "", name) - - cfg.JobSyncURL = "name" - name = cfg.GetJobSyncURL() - assert.Equal(t, "name", name) - assert.Equal(t, cfg.JobSyncURL, name) -} diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index 69ccc317..777b2749 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -43,7 +43,7 @@ func TestCreateServiceNetwork(t *testing.T) { conn, cleanup := CreateServiceNetwork(t, ctx, &ServiceNetworkConfig{ Name: QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "/here/there/every/where", }, }) @@ -77,7 +77,7 @@ func TestCreateRunnersAndServicesNetwork(t *testing.T) { { Name: QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "/here/there/every/where", }, }, diff --git a/internal/test/runner.go b/internal/test/runner.go index 514639ec..24efbdd1 100644 --- a/internal/test/runner.go +++ b/internal/test/runner.go @@ -4,11 +4,11 @@ import ( "context" "testing" + clientsyncrunner "queryorchestration/api/clientSyncRunner" doccleanrunner "queryorchestration/api/docCleanRunner" docinitrunner "queryorchestration/api/docInitRunner" docsyncrunner "queryorchestration/api/docSyncRunner" doctextrunner "queryorchestration/api/docTextRunner" - jobsyncrunner "queryorchestration/api/jobSyncRunner" queryrunner "queryorchestration/api/queryRunner" querysyncrunner "queryorchestration/api/querySyncRunner" queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner" @@ -27,7 +27,7 @@ const ( DocTextRunner = doctextrunner.Name QuerySyncRunner = querysyncrunner.Name QueryRunner = queryrunner.Name - JobSyncRunner = jobsyncrunner.Name + ClientSyncRunner = clientsyncrunner.Name QueryVersionSyncRunner = queryversionsyncrunner.Name ) diff --git a/internal/test/service_test.go b/internal/test/service_test.go index 0a2db0f0..8be68899 100644 --- a/internal/test/service_test.go +++ b/internal/test/service_test.go @@ -35,7 +35,7 @@ func TestCreateService(t *testing.T) { Cfg: cfg, Network: ncfg, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "/here/there/every/where", }, } diff --git a/pkg/queryService/api.gen.go b/pkg/queryService/api.gen.go index 6b1d981d..7ac9bd3f 100644 --- a/pkg/queryService/api.gen.go +++ b/pkg/queryService/api.gen.go @@ -17,6 +17,13 @@ import ( openapi_types "github.com/oapi-codegen/runtime/types" ) +// Defines values for ClientStatus. +const ( + INSYNC ClientStatus = "IN_SYNC" + NOTSYNCED ClientStatus = "NOT_SYNCED" + NOTSYNCING ClientStatus = "NOT_SYNCING" +) + // Defines values for ExportStatus. const ( Completed ExportStatus = "completed" @@ -36,12 +43,6 @@ const ( RightClosedInterval FieldFilterCondition = "right_closed_interval" ) -// Defines values for JobStatus. -const ( - INSYNC JobStatus = "IN_SYNC" - NOTSYNCED JobStatus = "NOT_SYNCED" -) - // Defines values for QueryType. const ( CONTEXTFULL QueryType = "CONTEXT_FULL" @@ -54,6 +55,18 @@ type ClientCreate struct { Name string `json:"name"` } +// ClientStatus Specifies the status of a client. +type ClientStatus string + +// ClientStatusBody defines model for ClientStatusBody. +type ClientStatusBody struct { + // ClientId The client id + ClientId openapi_types.UUID `json:"client_id"` + + // Status Specifies the status of a client. + Status ClientStatus `json:"status"` +} + // ClientUpdate defines model for ClientUpdate. type ClientUpdate struct { // CanSync If the client is allowing active syncs @@ -63,6 +76,63 @@ type ClientUpdate struct { Name *string `json:"name,omitempty"` } +// Collector Collector model. +type Collector struct { + // ActiveVersion The active version of the collector. + ActiveVersion int32 `json:"active_version"` + + // ClientId The ID of the associated client. + ClientId openapi_types.UUID `json:"client_id"` + + // Fields The fields in the collector. + Fields []CollectorField `json:"fields"` + + // LatestVersion The latest version of the collector. + LatestVersion int32 `json:"latest_version"` + + // MinimumCleanerVersion The minimum version for the document cleaner. + MinimumCleanerVersion int32 `json:"minimum_cleaner_version"` + + // MinimumTextVersion The minimum version for the text parser. + MinimumTextVersion int32 `json:"minimum_text_version"` +} + +// CollectorField The field properties for the collector. +type CollectorField struct { + // Name The output field name. + Name string `json:"name"` + + // QueryId The query id that will populate the result. + QueryId openapi_types.UUID `json:"query_id"` +} + +// CollectorUpdate Payload for updating a Collector. +type CollectorUpdate struct { + // ActiveVersion The active version of the collector. + ActiveVersion *int32 `json:"active_version,omitempty"` + + // Fields The fields in the collector. + Fields *[]CollectorField `json:"fields,omitempty"` + + // MinimumCleanerVersion The minimum version for the document cleaner. + MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"` + + // MinimumTextVersion The minimum version for the text parser. + MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"` +} + +// DocClient defines model for DocClient. +type DocClient struct { + // CanSync If the client is allowing active syncs + CanSync bool `json:"can_sync"` + + // Id The client id + Id openapi_types.UUID `json:"id"` + + // Name The client name + Name string `json:"name"` +} + // Document defines model for Document. type Document struct { // Bucket The bucket containing the document @@ -77,17 +147,17 @@ type Document struct { // ExportDetails Payload for export trigger response. type ExportDetails struct { - // JobId The job id relative to the export. - JobId openapi_types.UUID `json:"job_id"` + // ClientId The client id relative to the export. + ClientId openapi_types.UUID `json:"client_id"` // OutputLocation The location in which the export zip file will be found. OutputLocation *string `json:"output_location,omitempty"` - // Status The possible export job states. + // Status The possible export states. Status ExportStatus `json:"status"` } -// ExportStatus The possible export job states. +// ExportStatus The possible export states. type ExportStatus string // ExportTrigger Payload for triggering an export. @@ -126,100 +196,7 @@ type IdMessage struct { Id openapi_types.UUID `json:"id"` } -// Job defines model for Job. -type Job struct { - // CanSync Specifies whether the job is actively syncing - CanSync bool `json:"can_sync"` - - // ClientId The client id the job belongs to - ClientId openapi_types.UUID `json:"client_id"` - - // Id The job id - Id openapi_types.UUID `json:"id"` -} - -// JobClient defines model for JobClient. -type JobClient struct { - // CanSync If the client is allowing active syncs - CanSync bool `json:"can_sync"` - - // Id The client id - Id openapi_types.UUID `json:"id"` - - // Name The client name - Name string `json:"name"` -} - -// JobCollector JobCollector model. -type JobCollector struct { - // ActiveVersion The active version of the collector. - ActiveVersion int32 `json:"active_version"` - - // Fields The fields in the job collector. - Fields []JobCollectorField `json:"fields"` - - // JobId The ID of the associated job. - JobId openapi_types.UUID `json:"job_id"` - - // LatestVersion The latest version of the collector. - LatestVersion int32 `json:"latest_version"` - - // MinimumCleanerVersion The minimum version for the document cleaner. - MinimumCleanerVersion int32 `json:"minimum_cleaner_version"` - - // MinimumTextVersion The minimum version for the text parser. - MinimumTextVersion int32 `json:"minimum_text_version"` -} - -// JobCollectorField The field properties for the job collector. -type JobCollectorField struct { - // Name The output field name. - Name string `json:"name"` - - // QueryId The query id that will populate the result. - QueryId openapi_types.UUID `json:"query_id"` -} - -// JobCollectorUpdate Payload for updating a JobCollector. -type JobCollectorUpdate struct { - // ActiveVersion The active version of the collector. - ActiveVersion *int32 `json:"active_version,omitempty"` - - // Fields The fields in the job collector. - Fields *[]JobCollectorField `json:"fields,omitempty"` - - // MinimumCleanerVersion The minimum version for the document cleaner. - MinimumCleanerVersion *int32 `json:"minimum_cleaner_version,omitempty"` - - // MinimumTextVersion The minimum version for the text parser. - MinimumTextVersion *int32 `json:"minimum_text_version,omitempty"` -} - -// JobCreate defines model for JobCreate. -type JobCreate struct { - // ClientId The client id the job belongs to - ClientId openapi_types.UUID `json:"client_id"` -} - -// JobStatus Specifies the status of a job. -type JobStatus string - -// JobStatusBody defines model for JobStatusBody. -type JobStatusBody struct { - // JobId The job id - JobId openapi_types.UUID `json:"job_id"` - - // Status Specifies the status of a job. - Status JobStatus `json:"status"` -} - -// JobUpdate defines model for JobUpdate. -type JobUpdate struct { - // CanSync Specifies whether the job is actively syncing - CanSync *bool `json:"can_sync,omitempty"` -} - -// ListDocuments The documents in the job. +// ListDocuments The documents in the client. type ListDocuments = []Document // ListQueries defines model for ListQueries. @@ -297,14 +274,8 @@ type CreateClientJSONRequestBody = ClientCreate // UpdateClientJSONRequestBody defines body for UpdateClient for application/json ContentType. type UpdateClientJSONRequestBody = ClientUpdate -// CreateJobJSONRequestBody defines body for CreateJob for application/json ContentType. -type CreateJobJSONRequestBody = JobCreate - -// UpdateJobJSONRequestBody defines body for UpdateJob for application/json ContentType. -type UpdateJobJSONRequestBody = JobUpdate - -// UpdateJobCollectorByJobIdJSONRequestBody defines body for UpdateJobCollectorByJobId for application/json ContentType. -type UpdateJobCollectorByJobIdJSONRequestBody = JobCollectorUpdate +// UpdateCollectorByClientIdJSONRequestBody defines body for UpdateCollectorByClientId for application/json ContentType. +type UpdateCollectorByClientIdJSONRequestBody = CollectorUpdate // TriggerExportJSONRequestBody defines body for TriggerExport for application/json ContentType. type TriggerExportJSONRequestBody = ExportTrigger @@ -404,29 +375,16 @@ type ClientInterface interface { UpdateClient(ctx context.Context, id openapi_types.UUID, body UpdateClientJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // CreateJobWithBody request with any body - CreateJobWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetCollectorByClientId request + GetCollectorByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) - CreateJob(ctx context.Context, body CreateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // UpdateCollectorByClientIdWithBody request with any body + UpdateCollectorByClientIdWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetJob request - GetJob(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + UpdateCollectorByClientId(ctx context.Context, id openapi_types.UUID, body UpdateCollectorByClientIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // UpdateJobWithBody request with any body - UpdateJobWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - UpdateJob(ctx context.Context, id openapi_types.UUID, body UpdateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - - // GetJobCollectorByJobId request - GetJobCollectorByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) - - // UpdateJobCollectorByJobIdWithBody request with any body - UpdateJobCollectorByJobIdWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) - - UpdateJobCollectorByJobId(ctx context.Context, id openapi_types.UUID, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - - // ListDocumentsByJobId request - ListDocumentsByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListDocumentsByClientId request + ListDocumentsByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) // ExportState request ExportState(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -436,8 +394,8 @@ type ClientInterface interface { TriggerExport(ctx context.Context, id openapi_types.UUID, body TriggerExportJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // GetJobStatusByJobId request - GetJobStatusByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetStatusByClientId request + GetStatusByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) // ListQueries request ListQueries(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -521,8 +479,8 @@ func (c *Client) UpdateClient(ctx context.Context, id openapi_types.UUID, body U return c.Client.Do(req) } -func (c *Client) CreateJobWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewCreateJobRequestWithBody(c.Server, contentType, body) +func (c *Client) GetCollectorByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetCollectorByClientIdRequest(c.Server, id) if err != nil { return nil, err } @@ -533,8 +491,8 @@ func (c *Client) CreateJobWithBody(ctx context.Context, contentType string, body return c.Client.Do(req) } -func (c *Client) CreateJob(ctx context.Context, body CreateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewCreateJobRequest(c.Server, body) +func (c *Client) UpdateCollectorByClientIdWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUpdateCollectorByClientIdRequestWithBody(c.Server, id, contentType, body) if err != nil { return nil, err } @@ -545,8 +503,8 @@ func (c *Client) CreateJob(ctx context.Context, body CreateJobJSONRequestBody, r return c.Client.Do(req) } -func (c *Client) GetJob(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetJobRequest(c.Server, id) +func (c *Client) UpdateCollectorByClientId(ctx context.Context, id openapi_types.UUID, body UpdateCollectorByClientIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewUpdateCollectorByClientIdRequest(c.Server, id, body) if err != nil { return nil, err } @@ -557,68 +515,8 @@ func (c *Client) GetJob(ctx context.Context, id openapi_types.UUID, reqEditors . return c.Client.Do(req) } -func (c *Client) UpdateJobWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewUpdateJobRequestWithBody(c.Server, id, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) UpdateJob(ctx context.Context, id openapi_types.UUID, body UpdateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewUpdateJobRequest(c.Server, id, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) GetJobCollectorByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetJobCollectorByJobIdRequest(c.Server, id) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) UpdateJobCollectorByJobIdWithBody(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewUpdateJobCollectorByJobIdRequestWithBody(c.Server, id, contentType, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) UpdateJobCollectorByJobId(ctx context.Context, id openapi_types.UUID, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewUpdateJobCollectorByJobIdRequest(c.Server, id, body) - if err != nil { - return nil, err - } - req = req.WithContext(ctx) - if err := c.applyEditors(ctx, req, reqEditors); err != nil { - return nil, err - } - return c.Client.Do(req) -} - -func (c *Client) ListDocumentsByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewListDocumentsByJobIdRequest(c.Server, id) +func (c *Client) ListDocumentsByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewListDocumentsByClientIdRequest(c.Server, id) if err != nil { return nil, err } @@ -665,8 +563,8 @@ func (c *Client) TriggerExport(ctx context.Context, id openapi_types.UUID, body return c.Client.Do(req) } -func (c *Client) GetJobStatusByJobId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewGetJobStatusByJobIdRequest(c.Server, id) +func (c *Client) GetStatusByClientId(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetStatusByClientIdRequest(c.Server, id) if err != nil { return nil, err } @@ -894,48 +792,8 @@ func NewUpdateClientRequestWithBody(server string, id openapi_types.UUID, conten return req, nil } -// NewCreateJobRequest calls the generic CreateJob builder with application/json body -func NewCreateJobRequest(server string, body CreateJobJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewCreateJobRequestWithBody(server, "application/json", bodyReader) -} - -// NewCreateJobRequestWithBody generates requests for CreateJob with any type of body -func NewCreateJobRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { - var err error - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/job") - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("POST", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - -// NewGetJobRequest generates requests for GetJob -func NewGetJobRequest(server string, id openapi_types.UUID) (*http.Request, error) { +// NewGetCollectorByClientIdRequest generates requests for GetCollectorByClientId +func NewGetCollectorByClientIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { var err error var pathParam0 string @@ -950,7 +808,7 @@ func NewGetJobRequest(server string, id openapi_types.UUID) (*http.Request, erro return nil, err } - operationPath := fmt.Sprintf("/job/%s", pathParam0) + operationPath := fmt.Sprintf("/client/%s/collector", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -968,19 +826,19 @@ func NewGetJobRequest(server string, id openapi_types.UUID) (*http.Request, erro return req, nil } -// NewUpdateJobRequest calls the generic UpdateJob builder with application/json body -func NewUpdateJobRequest(server string, id openapi_types.UUID, body UpdateJobJSONRequestBody) (*http.Request, error) { +// NewUpdateCollectorByClientIdRequest calls the generic UpdateCollectorByClientId builder with application/json body +func NewUpdateCollectorByClientIdRequest(server string, id openapi_types.UUID, body UpdateCollectorByClientIdJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader buf, err := json.Marshal(body) if err != nil { return nil, err } bodyReader = bytes.NewReader(buf) - return NewUpdateJobRequestWithBody(server, id, "application/json", bodyReader) + return NewUpdateCollectorByClientIdRequestWithBody(server, id, "application/json", bodyReader) } -// NewUpdateJobRequestWithBody generates requests for UpdateJob with any type of body -func NewUpdateJobRequestWithBody(server string, id openapi_types.UUID, contentType string, body io.Reader) (*http.Request, error) { +// NewUpdateCollectorByClientIdRequestWithBody generates requests for UpdateCollectorByClientId with any type of body +func NewUpdateCollectorByClientIdRequestWithBody(server string, id openapi_types.UUID, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -995,7 +853,7 @@ func NewUpdateJobRequestWithBody(server string, id openapi_types.UUID, contentTy return nil, err } - operationPath := fmt.Sprintf("/job/%s", pathParam0) + operationPath := fmt.Sprintf("/client/%s/collector", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -1015,8 +873,8 @@ func NewUpdateJobRequestWithBody(server string, id openapi_types.UUID, contentTy return req, nil } -// NewGetJobCollectorByJobIdRequest generates requests for GetJobCollectorByJobId -func NewGetJobCollectorByJobIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { +// NewListDocumentsByClientIdRequest generates requests for ListDocumentsByClientId +func NewListDocumentsByClientIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { var err error var pathParam0 string @@ -1031,88 +889,7 @@ func NewGetJobCollectorByJobIdRequest(server string, id openapi_types.UUID) (*ht return nil, err } - operationPath := fmt.Sprintf("/job/%s/collector", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("GET", queryURL.String(), nil) - if err != nil { - return nil, err - } - - return req, nil -} - -// NewUpdateJobCollectorByJobIdRequest calls the generic UpdateJobCollectorByJobId builder with application/json body -func NewUpdateJobCollectorByJobIdRequest(server string, id openapi_types.UUID, body UpdateJobCollectorByJobIdJSONRequestBody) (*http.Request, error) { - var bodyReader io.Reader - buf, err := json.Marshal(body) - if err != nil { - return nil, err - } - bodyReader = bytes.NewReader(buf) - return NewUpdateJobCollectorByJobIdRequestWithBody(server, id, "application/json", bodyReader) -} - -// NewUpdateJobCollectorByJobIdRequestWithBody generates requests for UpdateJobCollectorByJobId with any type of body -func NewUpdateJobCollectorByJobIdRequestWithBody(server string, id openapi_types.UUID, contentType string, body io.Reader) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/job/%s/collector", pathParam0) - if operationPath[0] == '/' { - operationPath = "." + operationPath - } - - queryURL, err := serverURL.Parse(operationPath) - if err != nil { - return nil, err - } - - req, err := http.NewRequest("PATCH", queryURL.String(), body) - if err != nil { - return nil, err - } - - req.Header.Add("Content-Type", contentType) - - return req, nil -} - -// NewListDocumentsByJobIdRequest generates requests for ListDocumentsByJobId -func NewListDocumentsByJobIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { - var err error - - var pathParam0 string - - pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) - if err != nil { - return nil, err - } - - serverURL, err := url.Parse(server) - if err != nil { - return nil, err - } - - operationPath := fmt.Sprintf("/job/%s/documents", pathParam0) + operationPath := fmt.Sprintf("/client/%s/documents", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -1146,7 +923,7 @@ func NewExportStateRequest(server string, id openapi_types.UUID) (*http.Request, return nil, err } - operationPath := fmt.Sprintf("/job/%s/export", pathParam0) + operationPath := fmt.Sprintf("/client/%s/export", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -1191,7 +968,7 @@ func NewTriggerExportRequestWithBody(server string, id openapi_types.UUID, conte return nil, err } - operationPath := fmt.Sprintf("/job/%s/export", pathParam0) + operationPath := fmt.Sprintf("/client/%s/export", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -1211,8 +988,8 @@ func NewTriggerExportRequestWithBody(server string, id openapi_types.UUID, conte return req, nil } -// NewGetJobStatusByJobIdRequest generates requests for GetJobStatusByJobId -func NewGetJobStatusByJobIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { +// NewGetStatusByClientIdRequest generates requests for GetStatusByClientId +func NewGetStatusByClientIdRequest(server string, id openapi_types.UUID) (*http.Request, error) { var err error var pathParam0 string @@ -1227,7 +1004,7 @@ func NewGetJobStatusByJobIdRequest(server string, id openapi_types.UUID) (*http. return nil, err } - operationPath := fmt.Sprintf("/job/%s/status", pathParam0) + operationPath := fmt.Sprintf("/client/%s/status", pathParam0) if operationPath[0] == '/' { operationPath = "." + operationPath } @@ -1496,29 +1273,16 @@ type ClientWithResponsesInterface interface { UpdateClientWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateClientJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateClientResponse, error) - // CreateJobWithBodyWithResponse request with any body - CreateJobWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateJobResponse, error) + // GetCollectorByClientIdWithResponse request + GetCollectorByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetCollectorByClientIdResponse, error) - CreateJobWithResponse(ctx context.Context, body CreateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateJobResponse, error) + // UpdateCollectorByClientIdWithBodyWithResponse request with any body + UpdateCollectorByClientIdWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateCollectorByClientIdResponse, error) - // GetJobWithResponse request - GetJobWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobResponse, error) + UpdateCollectorByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateCollectorByClientIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateCollectorByClientIdResponse, error) - // UpdateJobWithBodyWithResponse request with any body - UpdateJobWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobResponse, error) - - UpdateJobWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobResponse, error) - - // GetJobCollectorByJobIdWithResponse request - GetJobCollectorByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobCollectorByJobIdResponse, error) - - // UpdateJobCollectorByJobIdWithBodyWithResponse request with any body - UpdateJobCollectorByJobIdWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) - - UpdateJobCollectorByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) - - // ListDocumentsByJobIdWithResponse request - ListDocumentsByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*ListDocumentsByJobIdResponse, error) + // ListDocumentsByClientIdWithResponse request + ListDocumentsByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*ListDocumentsByClientIdResponse, error) // ExportStateWithResponse request ExportStateWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*ExportStateResponse, error) @@ -1528,8 +1292,8 @@ type ClientWithResponsesInterface interface { TriggerExportWithResponse(ctx context.Context, id openapi_types.UUID, body TriggerExportJSONRequestBody, reqEditors ...RequestEditorFn) (*TriggerExportResponse, error) - // GetJobStatusByJobIdWithResponse request - GetJobStatusByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobStatusByJobIdResponse, error) + // GetStatusByClientIdWithResponse request + GetStatusByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetStatusByClientIdResponse, error) // ListQueriesWithResponse request ListQueriesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListQueriesResponse, error) @@ -1578,7 +1342,7 @@ func (r CreateClientResponse) StatusCode() int { type GetClientResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *JobClient + JSON200 *DocClient } // Status returns HTTPResponse.Status @@ -1618,14 +1382,14 @@ func (r UpdateClientResponse) StatusCode() int { return 0 } -type CreateJobResponse struct { +type GetCollectorByClientIdResponse struct { Body []byte HTTPResponse *http.Response - JSON201 *IdMessage + JSON200 *Collector } // Status returns HTTPResponse.Status -func (r CreateJobResponse) Status() string { +func (r GetCollectorByClientIdResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } @@ -1633,21 +1397,20 @@ func (r CreateJobResponse) Status() string { } // StatusCode returns HTTPResponse.StatusCode -func (r CreateJobResponse) StatusCode() int { +func (r GetCollectorByClientIdResponse) StatusCode() int { if r.HTTPResponse != nil { return r.HTTPResponse.StatusCode } return 0 } -type GetJobResponse struct { +type UpdateCollectorByClientIdResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *Job } // Status returns HTTPResponse.Status -func (r GetJobResponse) Status() string { +func (r UpdateCollectorByClientIdResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } @@ -1655,85 +1418,21 @@ func (r GetJobResponse) Status() string { } // StatusCode returns HTTPResponse.StatusCode -func (r GetJobResponse) StatusCode() int { +func (r UpdateCollectorByClientIdResponse) StatusCode() int { if r.HTTPResponse != nil { return r.HTTPResponse.StatusCode } return 0 } -type UpdateJobResponse struct { - Body []byte - HTTPResponse *http.Response -} - -// Status returns HTTPResponse.Status -func (r UpdateJobResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r UpdateJobResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type GetJobCollectorByJobIdResponse struct { - Body []byte - HTTPResponse *http.Response - JSON200 *JobCollector -} - -// Status returns HTTPResponse.Status -func (r GetJobCollectorByJobIdResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r GetJobCollectorByJobIdResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type UpdateJobCollectorByJobIdResponse struct { - Body []byte - HTTPResponse *http.Response -} - -// Status returns HTTPResponse.Status -func (r UpdateJobCollectorByJobIdResponse) Status() string { - if r.HTTPResponse != nil { - return r.HTTPResponse.Status - } - return http.StatusText(0) -} - -// StatusCode returns HTTPResponse.StatusCode -func (r UpdateJobCollectorByJobIdResponse) StatusCode() int { - if r.HTTPResponse != nil { - return r.HTTPResponse.StatusCode - } - return 0 -} - -type ListDocumentsByJobIdResponse struct { +type ListDocumentsByClientIdResponse struct { Body []byte HTTPResponse *http.Response JSON200 *ListDocuments } // Status returns HTTPResponse.Status -func (r ListDocumentsByJobIdResponse) Status() string { +func (r ListDocumentsByClientIdResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } @@ -1741,7 +1440,7 @@ func (r ListDocumentsByJobIdResponse) Status() string { } // StatusCode returns HTTPResponse.StatusCode -func (r ListDocumentsByJobIdResponse) StatusCode() int { +func (r ListDocumentsByClientIdResponse) StatusCode() int { if r.HTTPResponse != nil { return r.HTTPResponse.StatusCode } @@ -1792,14 +1491,14 @@ func (r TriggerExportResponse) StatusCode() int { return 0 } -type GetJobStatusByJobIdResponse struct { +type GetStatusByClientIdResponse struct { Body []byte HTTPResponse *http.Response - JSON200 *JobStatusBody + JSON200 *ClientStatusBody } // Status returns HTTPResponse.Status -func (r GetJobStatusByJobIdResponse) Status() string { +func (r GetStatusByClientIdResponse) Status() string { if r.HTTPResponse != nil { return r.HTTPResponse.Status } @@ -1807,7 +1506,7 @@ func (r GetJobStatusByJobIdResponse) Status() string { } // StatusCode returns HTTPResponse.StatusCode -func (r GetJobStatusByJobIdResponse) StatusCode() int { +func (r GetStatusByClientIdResponse) StatusCode() int { if r.HTTPResponse != nil { return r.HTTPResponse.StatusCode } @@ -1966,82 +1665,39 @@ func (c *ClientWithResponses) UpdateClientWithResponse(ctx context.Context, id o return ParseUpdateClientResponse(rsp) } -// CreateJobWithBodyWithResponse request with arbitrary body returning *CreateJobResponse -func (c *ClientWithResponses) CreateJobWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreateJobResponse, error) { - rsp, err := c.CreateJobWithBody(ctx, contentType, body, reqEditors...) +// GetCollectorByClientIdWithResponse request returning *GetCollectorByClientIdResponse +func (c *ClientWithResponses) GetCollectorByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetCollectorByClientIdResponse, error) { + rsp, err := c.GetCollectorByClientId(ctx, id, reqEditors...) if err != nil { return nil, err } - return ParseCreateJobResponse(rsp) + return ParseGetCollectorByClientIdResponse(rsp) } -func (c *ClientWithResponses) CreateJobWithResponse(ctx context.Context, body CreateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*CreateJobResponse, error) { - rsp, err := c.CreateJob(ctx, body, reqEditors...) +// UpdateCollectorByClientIdWithBodyWithResponse request with arbitrary body returning *UpdateCollectorByClientIdResponse +func (c *ClientWithResponses) UpdateCollectorByClientIdWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateCollectorByClientIdResponse, error) { + rsp, err := c.UpdateCollectorByClientIdWithBody(ctx, id, contentType, body, reqEditors...) if err != nil { return nil, err } - return ParseCreateJobResponse(rsp) + return ParseUpdateCollectorByClientIdResponse(rsp) } -// GetJobWithResponse request returning *GetJobResponse -func (c *ClientWithResponses) GetJobWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobResponse, error) { - rsp, err := c.GetJob(ctx, id, reqEditors...) +func (c *ClientWithResponses) UpdateCollectorByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateCollectorByClientIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateCollectorByClientIdResponse, error) { + rsp, err := c.UpdateCollectorByClientId(ctx, id, body, reqEditors...) if err != nil { return nil, err } - return ParseGetJobResponse(rsp) + return ParseUpdateCollectorByClientIdResponse(rsp) } -// UpdateJobWithBodyWithResponse request with arbitrary body returning *UpdateJobResponse -func (c *ClientWithResponses) UpdateJobWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobResponse, error) { - rsp, err := c.UpdateJobWithBody(ctx, id, contentType, body, reqEditors...) +// ListDocumentsByClientIdWithResponse request returning *ListDocumentsByClientIdResponse +func (c *ClientWithResponses) ListDocumentsByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*ListDocumentsByClientIdResponse, error) { + rsp, err := c.ListDocumentsByClientId(ctx, id, reqEditors...) if err != nil { return nil, err } - return ParseUpdateJobResponse(rsp) -} - -func (c *ClientWithResponses) UpdateJobWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateJobJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobResponse, error) { - rsp, err := c.UpdateJob(ctx, id, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseUpdateJobResponse(rsp) -} - -// GetJobCollectorByJobIdWithResponse request returning *GetJobCollectorByJobIdResponse -func (c *ClientWithResponses) GetJobCollectorByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobCollectorByJobIdResponse, error) { - rsp, err := c.GetJobCollectorByJobId(ctx, id, reqEditors...) - if err != nil { - return nil, err - } - return ParseGetJobCollectorByJobIdResponse(rsp) -} - -// UpdateJobCollectorByJobIdWithBodyWithResponse request with arbitrary body returning *UpdateJobCollectorByJobIdResponse -func (c *ClientWithResponses) UpdateJobCollectorByJobIdWithBodyWithResponse(ctx context.Context, id openapi_types.UUID, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) { - rsp, err := c.UpdateJobCollectorByJobIdWithBody(ctx, id, contentType, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseUpdateJobCollectorByJobIdResponse(rsp) -} - -func (c *ClientWithResponses) UpdateJobCollectorByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, body UpdateJobCollectorByJobIdJSONRequestBody, reqEditors ...RequestEditorFn) (*UpdateJobCollectorByJobIdResponse, error) { - rsp, err := c.UpdateJobCollectorByJobId(ctx, id, body, reqEditors...) - if err != nil { - return nil, err - } - return ParseUpdateJobCollectorByJobIdResponse(rsp) -} - -// ListDocumentsByJobIdWithResponse request returning *ListDocumentsByJobIdResponse -func (c *ClientWithResponses) ListDocumentsByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*ListDocumentsByJobIdResponse, error) { - rsp, err := c.ListDocumentsByJobId(ctx, id, reqEditors...) - if err != nil { - return nil, err - } - return ParseListDocumentsByJobIdResponse(rsp) + return ParseListDocumentsByClientIdResponse(rsp) } // ExportStateWithResponse request returning *ExportStateResponse @@ -2070,13 +1726,13 @@ func (c *ClientWithResponses) TriggerExportWithResponse(ctx context.Context, id return ParseTriggerExportResponse(rsp) } -// GetJobStatusByJobIdWithResponse request returning *GetJobStatusByJobIdResponse -func (c *ClientWithResponses) GetJobStatusByJobIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetJobStatusByJobIdResponse, error) { - rsp, err := c.GetJobStatusByJobId(ctx, id, reqEditors...) +// GetStatusByClientIdWithResponse request returning *GetStatusByClientIdResponse +func (c *ClientWithResponses) GetStatusByClientIdWithResponse(ctx context.Context, id openapi_types.UUID, reqEditors ...RequestEditorFn) (*GetStatusByClientIdResponse, error) { + rsp, err := c.GetStatusByClientId(ctx, id, reqEditors...) if err != nil { return nil, err } - return ParseGetJobStatusByJobIdResponse(rsp) + return ParseGetStatusByClientIdResponse(rsp) } // ListQueriesWithResponse request returning *ListQueriesResponse @@ -2189,7 +1845,7 @@ func ParseGetClientResponse(rsp *http.Response) (*GetClientResponse, error) { switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest JobClient + var dest DocClient if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -2216,48 +1872,22 @@ func ParseUpdateClientResponse(rsp *http.Response) (*UpdateClientResponse, error return response, nil } -// ParseCreateJobResponse parses an HTTP response from a CreateJobWithResponse call -func ParseCreateJobResponse(rsp *http.Response) (*CreateJobResponse, error) { +// ParseGetCollectorByClientIdResponse parses an HTTP response from a GetCollectorByClientIdWithResponse call +func ParseGetCollectorByClientIdResponse(rsp *http.Response) (*GetCollectorByClientIdResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { return nil, err } - response := &CreateJobResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: - var dest IdMessage - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON201 = &dest - - } - - return response, nil -} - -// ParseGetJobResponse parses an HTTP response from a GetJobWithResponse call -func ParseGetJobResponse(rsp *http.Response) (*GetJobResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &GetJobResponse{ + response := &GetCollectorByClientIdResponse{ Body: bodyBytes, HTTPResponse: rsp, } switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Job + var dest Collector if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } @@ -2268,15 +1898,15 @@ func ParseGetJobResponse(rsp *http.Response) (*GetJobResponse, error) { return response, nil } -// ParseUpdateJobResponse parses an HTTP response from a UpdateJobWithResponse call -func ParseUpdateJobResponse(rsp *http.Response) (*UpdateJobResponse, error) { +// ParseUpdateCollectorByClientIdResponse parses an HTTP response from a UpdateCollectorByClientIdWithResponse call +func ParseUpdateCollectorByClientIdResponse(rsp *http.Response) (*UpdateCollectorByClientIdResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { return nil, err } - response := &UpdateJobResponse{ + response := &UpdateCollectorByClientIdResponse{ Body: bodyBytes, HTTPResponse: rsp, } @@ -2284,57 +1914,15 @@ func ParseUpdateJobResponse(rsp *http.Response) (*UpdateJobResponse, error) { return response, nil } -// ParseGetJobCollectorByJobIdResponse parses an HTTP response from a GetJobCollectorByJobIdWithResponse call -func ParseGetJobCollectorByJobIdResponse(rsp *http.Response) (*GetJobCollectorByJobIdResponse, error) { +// ParseListDocumentsByClientIdResponse parses an HTTP response from a ListDocumentsByClientIdWithResponse call +func ParseListDocumentsByClientIdResponse(rsp *http.Response) (*ListDocumentsByClientIdResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { return nil, err } - response := &GetJobCollectorByJobIdResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - switch { - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest JobCollector - if err := json.Unmarshal(bodyBytes, &dest); err != nil { - return nil, err - } - response.JSON200 = &dest - - } - - return response, nil -} - -// ParseUpdateJobCollectorByJobIdResponse parses an HTTP response from a UpdateJobCollectorByJobIdWithResponse call -func ParseUpdateJobCollectorByJobIdResponse(rsp *http.Response) (*UpdateJobCollectorByJobIdResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &UpdateJobCollectorByJobIdResponse{ - Body: bodyBytes, - HTTPResponse: rsp, - } - - return response, nil -} - -// ParseListDocumentsByJobIdResponse parses an HTTP response from a ListDocumentsByJobIdWithResponse call -func ParseListDocumentsByJobIdResponse(rsp *http.Response) (*ListDocumentsByJobIdResponse, error) { - bodyBytes, err := io.ReadAll(rsp.Body) - defer func() { _ = rsp.Body.Close() }() - if err != nil { - return nil, err - } - - response := &ListDocumentsByJobIdResponse{ + response := &ListDocumentsByClientIdResponse{ Body: bodyBytes, HTTPResponse: rsp, } @@ -2404,22 +1992,22 @@ func ParseTriggerExportResponse(rsp *http.Response) (*TriggerExportResponse, err return response, nil } -// ParseGetJobStatusByJobIdResponse parses an HTTP response from a GetJobStatusByJobIdWithResponse call -func ParseGetJobStatusByJobIdResponse(rsp *http.Response) (*GetJobStatusByJobIdResponse, error) { +// ParseGetStatusByClientIdResponse parses an HTTP response from a GetStatusByClientIdWithResponse call +func ParseGetStatusByClientIdResponse(rsp *http.Response) (*GetStatusByClientIdResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { return nil, err } - response := &GetJobStatusByJobIdResponse{ + response := &GetStatusByClientIdResponse{ Body: bodyBytes, HTTPResponse: rsp, } switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest JobStatusBody + var dest ClientStatusBody if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index 4d4d09e5..c18d32fe 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -8,18 +8,15 @@ vars: OUT_DIR: out includes: - deps: - dir: "{{.CONTEXT}}" - taskfile: dependencies.yml test: dir: "{{.CONTEXT}}" taskfile: tests.yml docker: - dir: "{{.CONTEXT}}" + dir: "." taskfile: docker.yml openapi: dir: "{{.CONTEXT}}" - taskfile: openapi.yml + taskfile: openapi-scripts.yml compose: dir: "{{.CONTEXT}}" taskfile: local-deployments.yml @@ -28,75 +25,71 @@ includes: taskfile: database.yml tasks: - init: - cmds: - - touch .env fullsuite: cmds: - - task: deps:tidy - - task: generate:clean - - task: build:nogen - - task: lint:nogen - - task: test:unit:nogen - - task: test:endtoend:cmd:nocache + - task: build + - task: lint + - task: test:unit + - task: test:endtoend + precommit: + cmds: + - task: lint:short + deps:tidy: + - go mod tidy + - go mod vendor build: - deps: - - generate - cmds: - - task: build:nogen - build:nogen: cmds: + - task: deps:tidy + - task: generate - task: docker:build generate:clean: + run: once deps: - db:generate:clean - openapi:generate - test:mocks:generate - docs:generate + cmds: + - go generate ./... generate: + run: once deps: - db:generate - openapi:generate - test:mocks:generate - docs:generate + cmds: + - go generate ./... lint: deps: - generate - cmds: - - task: lint:nogen - lint:nogen: cmds: - task: go:lint - - task: db:lint - task: yaml:lint - task: json:lint - task: docker:lint - - task: compose:lint - - task: openapi:lint - lint:fix: - cmds: - - task: go:lint:fix - task: openapi:lint - task: db:lint - - task: yaml:lint - - task: json:lint:fix - - task: docker:lint - task: compose:lint + lint:short: + deps: + - go:lint + - yaml:lint + - json:lint + - docker:lint + - openapi:lint + - db:lint + - compose:lint:short go:lint: cmds: - - golangci-lint run - go:lint:fix: - cmds: - - golangci-lint run --fix + - golangci-lint run --fix {{.CLI_ARGS}} yaml:lint: cmds: - - yamllint . -s + - yamllint . -s {{.CLI_ARGS}} json:lint: cmds: - - jsonlint devbox.json -q -s - json:lint:fix: - cmds: - - jsonlint devbox.json -q -s -i + - jsonlint devbox.json -q -s -i {{.CLI_ARGS}} docs:generate: + run: once cmds: - gomarkdoc --template-file file=docs/templates/root.gotxt ./cmd/... diff --git a/scripts/database.yml b/scripts/database.yml index 708f0a4b..a2f4a1b8 100644 --- a/scripts/database.yml +++ b/scripts/database.yml @@ -8,6 +8,7 @@ vars: tasks: generate: + run: once deps: - mig cmds: @@ -15,7 +16,7 @@ tasks: generate:clean: cmds: - task compose:clean:generate - - task db:generate + - task: generate lint: cmds: - task compose:up:generate diff --git a/scripts/dependencies.yml b/scripts/dependencies.yml deleted file mode 100644 index f3f35287..00000000 --- a/scripts/dependencies.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# https://taskfile.dev - -version: '3' - -tasks: - tidy:novendor: - cmds: - - go mod tidy - tidy: - deps: - - tidy:novendor - cmds: - - go mod vendor diff --git a/scripts/docker.yml b/scripts/docker.yml index ac7179e4..f5dd9dda 100644 --- a/scripts/docker.yml +++ b/scripts/docker.yml @@ -1,15 +1,16 @@ --- # https://taskfile.dev -version: '3' +version: "3" vars: - DOCKERFILE: "build/Dockerfile" + DOCKERFILE: "{{.CONTEXT}}/build/Dockerfile" tasks: lint: cmds: - hadolint {{.DOCKERFILE}} -t none build: + run: once cmds: - - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} . + - docker build -t {{.IMAGE_NAME}} -f {{.DOCKERFILE}} {{.CONTEXT}} diff --git a/scripts/local-deployments.yml b/scripts/local-deployments.yml index 46fd546c..bcc0fe8c 100644 --- a/scripts/local-deployments.yml +++ b/scripts/local-deployments.yml @@ -36,6 +36,7 @@ tasks: vars: COMPOSE_FILE: "{{.TEST_COMPOSE_FILE}}" up:generate: + run: once cmds: - task: up:cmd vars: @@ -62,7 +63,7 @@ tasks: - aws sqs create-queue --queue-name $QNAME_DOCUMENT_TEXT - aws sqs create-queue --queue-name $QNAME_QUERY_SYNC - aws sqs create-queue --queue-name $QNAME_QUERY_RUNNER - - aws sqs create-queue --queue-name $QNAME_JOB_SYNC + - aws sqs create-queue --queue-name $QNAME_CLIENT_SYNC - aws sqs create-queue --queue-name $QNAME_QUERY_VERSION_SYNC - | aws s3api put-bucket-notification-configuration \ @@ -93,6 +94,7 @@ tasks: - task: down:test - docker volume rm -f deployments_test-db-data clean:generate: + run: once cmds: - task: down:generate - docker volume rm -f deployments_generate-db-data @@ -109,3 +111,8 @@ tasks: - | docker compose -f {{.GENERATE_COMPOSE_FILE}} config -q \ --resolve-image-digests + lint:short: + cmds: + - docker compose -f {{.LOCAL_COMPOSE_FILE}} config -q + - docker compose -f {{.TEST_COMPOSE_FILE}} config -q + - docker compose -f {{.GENERATE_COMPOSE_FILE}} config -q diff --git a/scripts/openapi-scripts.yml b/scripts/openapi-scripts.yml new file mode 100644 index 00000000..b7385373 --- /dev/null +++ b/scripts/openapi-scripts.yml @@ -0,0 +1,32 @@ +--- +# https://taskfile.dev + +version: "3" + +tasks: + lint: + cmds: + - vacuum lint ./serviceAPIs/queryService.yaml -b -d -e --no-clip + generate: + - | + generateGo() { + mkdir -p $2 + oapi-codegen \ + -generate $4 \ + -config .openapi.yml \ + -package $1 \ + -o $2/api.gen.go \ + $3 + } + + for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do + if [[ -f "$file" ]]; then + service=$(basename "$file") + service=${service%.*} + service_lower=${service,,} + generateGo $service_lower ./pkg/$service $file \ + "models,client" + generateGo $service_lower ./api/$service $file \ + "models,embedded-spec,echo-server" + fi + done diff --git a/scripts/openapi.yml b/scripts/openapi.yml deleted file mode 100644 index 6001f51c..00000000 --- a/scripts/openapi.yml +++ /dev/null @@ -1,53 +0,0 @@ ---- -# https://taskfile.dev - -version: "3" - -tasks: - lint: - cmds: - - vacuum lint ./serviceAPIs/queryService.yaml -d - generate: - deps: - - generate:server - - generate:client - - generate:metrics:stubs - generate:client: - cmds: - - | - for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do - if [[ -f "$file" ]]; then - service=$(basename "$file") - service=${service%.*} - service_lower=${service,,} - outputloc=./pkg/$service - mkdir -p $outputloc - oapi-codegen \ - -generate "models,client" \ - -config .openapi.yml \ - -package $service_lower \ - -o $outputloc/api.gen.go \ - $file - fi - done - generate:server: - cmds: - - | - for file in serviceAPIs/*.yml serviceAPIs/*.yaml; do - if [[ -f "$file" ]]; then - service=$(basename "$file") - service=${service%.*} - service_lower=${service,,} - outputloc=./api/$service - mkdir -p $outputloc - oapi-codegen \ - -generate "models,embedded-spec,echo-server" \ - -config .openapi.yml \ - -package $service_lower \ - -o $outputloc/api.gen.go \ - $file - fi - done - generate:metrics:stubs: - cmds: - - go generate ./... diff --git a/scripts/tests.yml b/scripts/tests.yml index 9d5b82f9..0a70b4d0 100644 --- a/scripts/tests.yml +++ b/scripts/tests.yml @@ -12,42 +12,42 @@ vars: # yamllint disable-line rule:line-length EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go" +includes: + docker: + dir: "{{.CONTEXT}}" + taskfile: docker.yml + internal: true + tasks: mocks:generate: + run: once cmds: - rm -rf mocks/* - go tool mockery - unit: - cmds: - - task: unit:nocoverage - - task: unit:getcoverage - unit:short: - deps: - - mocks:generate - cmds: - - go test -short {{.INTERNAL}} {{.API}} {{.PKG}} {{.CMD}} - unit:nogen: - cmds: - - task: unit:cmd - - task: unit:getcoverage unit:cmd: internal: true + vars: + SKIP_GENERATE: false cmds: + - task: mocks:generate - mkdir -p {{.OUT_DIR}} - - | - go test {{.INTERNAL}} {{.API}} \ - -coverpkg={{.INTERNAL}},{{.API}} \ - -coverprofile={{.COVERAGE_FILE}} - unit:nocoverage: - deps: - - mocks:generate + - go test {{.INTERNAL}} {{.API}} {{.CLI_ARGS}} + unit:short: cmds: - task: unit:cmd - unit:getcoverage: + vars: + CLI_ARGS: -short + SKIP_GENERATE: true + unit: vars: TMP_FILE: "{{.OUT_DIR}}/coverage.tmp" FILTER_COVERAGE_FILE: "{{.OUT_DIR}}/filtered_coverage.out" cmds: + - task: unit:cmd + vars: + CLI_ARGS: | + -coverpkg={{.INTERNAL}},{{.API}} \ + -coverprofile={{.COVERAGE_FILE}} - | grep -v -E "{{.EXCLUDED_FILES}}" {{.COVERAGE_FILE}} \ > {{.FILTER_COVERAGE_FILE}} @@ -87,13 +87,7 @@ tasks: echo "✅ All coverage checks passed!" silent: true endtoend: - cmds: - - task generate - - task docker:build - - task: endtoend:cmd:nocache - endtoend:cmd:nocache: + deps: + - docker:build cmds: - go test -count=1 -v ./test/... - endtoend:cmd:nobuild: - cmds: - - go test -v ./test/... diff --git a/serviceAPIs/queryService.yaml b/serviceAPIs/queryService.yaml index a4d11916..33791caa 100644 --- a/serviceAPIs/queryService.yaml +++ b/serviceAPIs/queryService.yaml @@ -10,12 +10,10 @@ servers: tags: - name: ClientService description: Operations related to clients - - name: JobService - description: Operations related to jobs - - name: JobCollectorService - description: Operations related to job collectors - - name: JobDocumentsService - description: Operations related to job documents + - name: CollectorService + description: Operations related to collectors + - name: DocumentsService + description: Operations related to documents - name: QueryService description: Operations related to queries - name: ExportService @@ -66,7 +64,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/JobClient" + $ref: "#/components/schemas/DocClient" "400": description: Invalid request parameters. patch: @@ -198,30 +196,7 @@ paths: "400": description: Invalid request body. - /job: - post: - operationId: createJob - tags: - - JobService - summary: Create a new job - description: Creates a new job with the provided details. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/JobCreate" - responses: - "201": - description: Job created successfully. - content: - application/json: - schema: - $ref: "#/components/schemas/IdMessage" - "400": - description: Invalid request body. - - /job/{id}: + /client/{id}/status: parameters: - in: path name: id @@ -229,66 +204,24 @@ paths: schema: type: string format: uuid - description: The job ID. + description: The client id. get: - operationId: getJob + operationId: getStatusByClientId tags: - - JobService - summary: Get a job by ID - description: Retrieves a specific job by its ID. + - ClientService + summary: Get client sync status + description: Retrieves the sync status by its client ID. responses: "200": - description: Job details. + description: Client status details. content: application/json: schema: - $ref: "#/components/schemas/Job" - "400": - description: Invalid request parameters. - patch: - operationId: updateJob - tags: - - JobService - summary: Update a job - description: Updates an existing job with new details. - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/JobUpdate" - responses: - "204": - description: Job updated successfully. - "400": - description: Invalid request body. - - /job/{id}/status: - parameters: - - in: path - name: id - required: true - schema: - type: string - format: uuid - description: The job ID for the collector. - get: - operationId: getJobStatusByJobId - tags: - - JobCollectorService - summary: Get job status by job ID - description: Retrieves the job status by its job ID. - responses: - "200": - description: Job status details. - content: - application/json: - schema: - $ref: "#/components/schemas/JobStatusBody" + $ref: "#/components/schemas/ClientStatusBody" "400": description: Invalid request parameters. - /job/{id}/collector: + /client/{id}/collector: parameters: - in: path name: id @@ -296,41 +229,41 @@ paths: schema: type: string format: uuid - description: The job ID for the collector. + description: The client ID for the collector. get: - operationId: getJobCollectorByJobId + operationId: getCollectorByClientId tags: - - JobCollectorService - summary: Get a job collector by ID - description: Retrieves a specific job collector by its ID. + - CollectorService + summary: Get a collector by client ID + description: Retrieves a specific collector by its client ID. responses: "200": - description: Job collector details. + description: Collector details. content: application/json: schema: - $ref: "#/components/schemas/JobCollector" + $ref: "#/components/schemas/Collector" "400": description: Invalid body. patch: - operationId: updateJobCollectorByJobId + operationId: updateCollectorByClientId tags: - - JobCollectorService - summary: Update a job collector - description: Updates an existing job collector with new details. + - CollectorService + summary: Update a collector + description: Updates an existing collector with new details. requestBody: required: true content: application/json: schema: - $ref: "#/components/schemas/JobCollectorUpdate" + $ref: "#/components/schemas/CollectorUpdate" responses: "204": - description: Job collector updated successfully. + description: Collector updated successfully. "400": description: Invalid request body. - /job/{id}/documents: + /client/{id}/documents: parameters: - in: path name: id @@ -338,16 +271,16 @@ paths: schema: type: string format: uuid - description: The job ID for the documents. + description: The client ID for the documents. get: - operationId: listDocumentsByJobId + operationId: listDocumentsByClientId tags: - - JobDocumentsService - summary: List the documents for a job - description: Retrieves the list of documents by the job ID. + - DocumentsService + summary: List the documents for a client + description: Retrieves the list of documents by the client ID. responses: "200": - description: Job documents list. + description: Client documents list. content: application/json: schema: @@ -355,7 +288,7 @@ paths: "400": description: Invalid request body. - /job/{id}/export: + /client/{id}/export: parameters: - in: path name: id @@ -511,7 +444,7 @@ components: required: - value - JobClient: + DocClient: type: object properties: id: @@ -529,65 +462,30 @@ components: - name - can_sync - JobStatus: + ClientStatus: type: string enum: - IN_SYNC - NOT_SYNCED - description: Specifies the status of a job. + - NOT_SYNCING + description: Specifies the status of a client. - Job: + ClientStatusBody: type: object properties: - id: - type: string - format: uuid - description: The job id client_id: type: string format: uuid - description: The client id the job belongs to - can_sync: - type: boolean - description: Specifies whether the job is actively syncing - required: - - id - - client_id - - can_sync - - JobStatusBody: - type: object - properties: - job_id: - type: string - format: uuid - description: The job id + description: The client id status: - $ref: "#/components/schemas/JobStatus" - required: - - job_id - - status - - JobCreate: - type: object - properties: - client_id: - type: string - format: uuid - description: The client id the job belongs to + $ref: "#/components/schemas/ClientStatus" required: - client_id - - JobUpdate: - type: object - properties: - can_sync: - type: boolean - description: Specifies whether the job is actively syncing + - status ListDocuments: type: array - description: The documents in the job. + description: The documents in the client. items: $ref: "#/components/schemas/Document" @@ -638,13 +536,13 @@ components: required: - id - JobCollector: + Collector: type: object properties: - job_id: + client_id: type: string format: uuid - description: The ID of the associated job. + description: The ID of the associated client. active_version: type: integer format: int32 @@ -663,19 +561,19 @@ components: description: The minimum version for the text parser. fields: type: array - description: The fields in the job collector. + description: The fields in the collector. items: - $ref: "#/components/schemas/JobCollectorField" - description: JobCollector model. + $ref: "#/components/schemas/CollectorField" + description: Collector model. required: - - job_id + - client_id - fields - minimum_cleaner_version - minimum_text_version - latest_version - active_version - JobCollectorUpdate: + CollectorUpdate: type: object properties: minimum_cleaner_version: @@ -692,14 +590,14 @@ components: description: The active version of the collector. fields: type: array - description: The fields in the job collector. + description: The fields in the collector. items: - $ref: "#/components/schemas/JobCollectorField" - description: Payload for updating a JobCollector. + $ref: "#/components/schemas/CollectorField" + description: Payload for updating a Collector. - JobCollectorField: + CollectorField: type: object - description: The field properties for the job collector. + description: The field properties for the collector. properties: name: type: string @@ -770,22 +668,23 @@ components: - completed - in_progress - failed - description: The possible export job states. + description: The possible export states. ExportDetails: type: object properties: - job_id: + client_id: type: string format: uuid - description: The job id relative to the export. + description: The client id relative to the export. status: $ref: "#/components/schemas/ExportStatus" output_location: type: string description: The location in which the export zip file will be found. - example: s3://{bucket}/{clientid}/{jobid}/{timestamp}_{exportid}.zip + example: | + s3://{bucket_name}/{external_client_id}/yyyymmdd/{export_id}.csv description: Payload for export trigger response. required: - - job_id + - client_id - status diff --git a/test/process_test.go b/test/process_test.go index 7a9e7bf7..75e6b24e 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -54,7 +54,7 @@ func TestProcess(t *testing.T) { doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner) querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner) queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner) - jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner) + clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunner) queryversionsyncurl := test.CreateQueue(t, ctx, cfg, test.QueryVersionSyncRunner) bucketName := "docinitbucket" @@ -123,8 +123,8 @@ func TestProcess(t *testing.T) { }, }, { - Name: test.JobSyncRunner, - QueueURL: &jobsyncurl, + Name: test.ClientSyncRunner, + QueueURL: &clientsyncurl, Env: map[string]string{ "DOCUMENT_SYNC_URL": docsyncurl, }, @@ -133,7 +133,7 @@ func TestProcess(t *testing.T) { Name: test.QueryVersionSyncRunner, QueueURL: &queryversionsyncurl, Env: map[string]string{ - "JOB_SYNC_URL": jobsyncurl, + "CLIENT_SYNC_URL": clientsyncurl, }, }, }, @@ -141,7 +141,7 @@ func TestProcess(t *testing.T) { { Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": jobsyncurl, + "CLIENT_SYNC_URL": clientsyncurl, "QUERY_VERSION_SYNC_URL": queryversionsyncurl, }, }, @@ -156,20 +156,12 @@ func TestProcess(t *testing.T) { Name: "example_name", }) assert.NoError(t, err) - jobRes, err := qService.CreateJobWithResponse(ctx, queryservice.JobCreate{ - ClientId: clientRes.JSON201.Id, - }) - assert.NoError(t, err) canSync := true _, err = qService.UpdateClientWithResponse(ctx, clientRes.JSON201.Id, queryservice.ClientUpdate{ CanSync: &canSync, }) assert.NoError(t, err) - _, err = qService.UpdateJobWithResponse(ctx, jobRes.JSON201.Id, queryservice.JobUpdate{ - CanSync: &canSync, - }) - assert.NoError(t, err) contextQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ Type: queryservice.CONTEXTFULL, @@ -183,9 +175,9 @@ func TestProcess(t *testing.T) { }) assert.NoError(t, err) newActiveVersion := int32(2) - _, err = qService.UpdateJobCollectorByJobIdWithResponse(ctx, jobRes.JSON201.Id, queryservice.JobCollectorUpdate{ + _, err = qService.UpdateCollectorByClientIdWithResponse(ctx, clientRes.JSON201.Id, queryservice.CollectorUpdate{ ActiveVersion: &newActiveVersion, - Fields: &[]queryservice.JobCollectorField{ + Fields: &[]queryservice.CollectorField{ { Name: "JSON_QUERY", QueryId: jsonQueryRes.JSON201.Id, @@ -194,9 +186,9 @@ func TestProcess(t *testing.T) { }) assert.NoError(t, err) - WaitForJobStatus(t, ctx, qService, jobRes.JSON201.Id, queryservice.INSYNC) + WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC) - location := fmt.Sprintf("%s/%s/%s", clientRes.JSON201.Id, jobRes.JSON201.Id, "object_name") + location := fmt.Sprintf("%s/%s", clientRes.JSON201.Id, "object_name") body := strings.NewReader(pdfHelloWorld) _, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{ Bucket: &bucketName, @@ -205,10 +197,10 @@ func TestProcess(t *testing.T) { }) assert.NoError(t, err) - WaitForJobStatus(t, ctx, qService, jobRes.JSON201.Id, queryservice.NOTSYNCED) - WaitForJobStatus(t, ctx, qService, jobRes.JSON201.Id, queryservice.INSYNC) + WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.NOTSYNCED) + WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC) - docs, err := qService.ListDocumentsByJobIdWithResponse(ctx, jobRes.JSON201.Id) + docs, err := qService.ListDocumentsByClientIdWithResponse(ctx, clientRes.JSON201.Id) assert.NoError(t, err) assert.Len(t, *docs.JSON200, 1) doc := (*docs.JSON200)[0] @@ -231,8 +223,8 @@ func TestProcess(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, res) - WaitForJobStatus(t, ctx, qService, jobRes.JSON201.Id, queryservice.NOTSYNCED) - WaitForJobStatus(t, ctx, qService, jobRes.JSON201.Id, queryservice.INSYNC) + WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.NOTSYNCED) + WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC) testRes, err = qService.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{ QueryVersion: 2, @@ -242,7 +234,7 @@ func TestProcess(t *testing.T) { assert.Equal(t, "valuetwo", testRes.JSON200.Value) } -func WaitForJobStatus(t testing.TB, ctx context.Context, service *queryservice.ClientWithResponses, id types.UUID, status queryservice.JobStatus) { +func WaitForClientStatus(t testing.TB, ctx context.Context, service *queryservice.ClientWithResponses, id types.UUID, status queryservice.ClientStatus) { t.Helper() timeout := time.After(30 * time.Second) @@ -252,9 +244,9 @@ func WaitForJobStatus(t testing.TB, ctx context.Context, service *queryservice.C for { select { case <-timeout: - require.NoError(t, fmt.Errorf("Timeout waiting for job status to become %s", status)) + require.NoError(t, fmt.Errorf("Timeout waiting for client status to become %s", status)) case <-ticker.C: - jRes, err := service.GetJobStatusByJobIdWithResponse(ctx, id) + jRes, err := service.GetStatusByClientIdWithResponse(ctx, id) if err != nil { assert.NoError(t, err) } diff --git a/test/queryService/accessory_test.go b/test/queryService/accessory_test.go index 872ec2ca..f971a243 100644 --- a/test/queryService/accessory_test.go +++ b/test/queryService/accessory_test.go @@ -17,7 +17,7 @@ func TestQueryServiceAccessories(t *testing.T) { c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "iamthere", }, }) diff --git a/test/queryService/client_test.go b/test/queryService/client_test.go index b2bf7a2c..7c51bb05 100644 --- a/test/queryService/client_test.go +++ b/test/queryService/client_test.go @@ -16,7 +16,7 @@ func TestClient(t *testing.T) { c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "iamthere", }, }) diff --git a/test/queryService/jobcollectorservice_test.go b/test/queryService/collectorservice_test.go similarity index 77% rename from test/queryService/jobcollectorservice_test.go rename to test/queryService/collectorservice_test.go index 3447087c..736743d8 100644 --- a/test/queryService/jobcollectorservice_test.go +++ b/test/queryService/collectorservice_test.go @@ -22,7 +22,7 @@ type CollectorConfig struct { queue.QueueConfig } -func TestJobCollectorService(t *testing.T) { +func TestCollectorService(t *testing.T) { ctx := context.Background() cfg := &CollectorConfig{} @@ -40,14 +40,14 @@ func TestJobCollectorService(t *testing.T) { err := cfg.SetQueueClient(ctx) assert.NoError(t, err) - jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner) + clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunner) c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ Cfg: cfg, Network: network, Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": jobsyncurl, + "CLIENT_SYNC_URL": clientsyncurl, "QUERY_VERSION_SYNC_URL": "iamthere", }, }) @@ -75,15 +75,11 @@ func TestJobCollectorService(t *testing.T) { Name: "example_name", }) assert.NoError(t, err) - jobRes, err := client.CreateJobWithResponse(ctx, queryservice.JobCreate{ - ClientId: clientRes.JSON201.Id, - }) - assert.NoError(t, err) - id := jobRes.JSON201.Id + id := clientRes.JSON201.Id - collRes, err := client.GetJobCollectorByJobIdWithResponse(ctx, id) + collRes, err := client.GetCollectorByClientIdWithResponse(ctx, id) assert.NoError(t, err) - assert.Equal(t, id, collRes.JSON200.JobId) + assert.Equal(t, id, collRes.JSON200.ClientId) assert.Equal(t, int32(1), collRes.JSON200.ActiveVersion) assert.Equal(t, int32(1), collRes.JSON200.LatestVersion) assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion) @@ -91,13 +87,13 @@ func TestJobCollectorService(t *testing.T) { assert.Len(t, collRes.JSON200.Fields, 0) av := int32(2) - fields := []queryservice.JobCollectorField{ + fields := []queryservice.CollectorField{ { Name: "json", QueryId: jsonRes.JSON201.Id, }, } - uRes, err := client.UpdateJobCollectorByJobIdWithResponse(ctx, id, queryservice.JobCollectorUpdate{ + uRes, err := client.UpdateCollectorByClientIdWithResponse(ctx, id, queryservice.CollectorUpdate{ ActiveVersion: &av, Fields: &fields, }) @@ -105,11 +101,11 @@ func TestJobCollectorService(t *testing.T) { log.Print(string(uRes.Body)) assert.Equal(t, 200, uRes.StatusCode()) - test.AssertMessageBody(t, ctx, cfg, jobsyncurl, regexp.MustCompile(`{"id":".+"}`)) + test.AssertMessageBody(t, ctx, cfg, clientsyncurl, regexp.MustCompile(`{"id":".+"}`)) - collRes, err = client.GetJobCollectorByJobIdWithResponse(ctx, id) + collRes, err = client.GetCollectorByClientIdWithResponse(ctx, id) assert.NoError(t, err) - assert.Equal(t, id, collRes.JSON200.JobId) + assert.Equal(t, id, collRes.JSON200.ClientId) assert.Equal(t, int32(2), collRes.JSON200.ActiveVersion) assert.Equal(t, int32(2), collRes.JSON200.LatestVersion) assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion) diff --git a/test/queryService/exportservice_test.go b/test/queryService/exportservice_test.go index ef3f9139..a4b45a65 100644 --- a/test/queryService/exportservice_test.go +++ b/test/queryService/exportservice_test.go @@ -17,7 +17,7 @@ func TestExportService(t *testing.T) { c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": "iamthere", }, }) diff --git a/test/queryService/job_test.go b/test/queryService/job_test.go deleted file mode 100644 index dee72a57..00000000 --- a/test/queryService/job_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package endtoend - -import ( - "context" - "testing" - - "queryorchestration/internal/test" - queryservice "queryorchestration/pkg/queryService" - - "github.com/stretchr/testify/assert" -) - -func TestJob(t *testing.T) { - ctx := context.Background() - - c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ - Name: test.QueryService, - Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", - "QUERY_VERSION_SYNC_URL": "iamthere", - }, - }) - defer cleanup() - - client, err := queryservice.NewClientWithResponses(c.URI) - assert.NoError(t, err) - - clientRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{ - Name: "example_name", - }) - assert.NoError(t, err) - idRes, err := client.CreateJobWithResponse(ctx, queryservice.JobCreate{ - ClientId: clientRes.JSON201.Id, - }) - assert.NoError(t, err) - assert.NotNil(t, idRes) - assert.NotNil(t, idRes.JSON201) - assert.NotNil(t, idRes.JSON201.Id) - id := idRes.JSON201.Id - - jobRes, err := client.GetJobWithResponse(ctx, id) - assert.NoError(t, err) - assert.Equal(t, id, jobRes.JSON200.Id) - assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId) - assert.False(t, jobRes.JSON200.CanSync) - - updateCanSync := !jobRes.JSON200.CanSync - updateRes, err := client.UpdateJobWithResponse(ctx, id, queryservice.JobUpdate{ - CanSync: &updateCanSync, - }) - assert.NoError(t, err) - assert.NotNil(t, updateRes) - - jobRes, err = client.GetJobWithResponse(ctx, id) - assert.NoError(t, err) - assert.Equal(t, id, jobRes.JSON200.Id) - assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId) - assert.False(t, jobRes.JSON200.CanSync) - - jobStatusRes, err := client.GetJobStatusByJobIdWithResponse(ctx, id) - assert.NoError(t, err) - assert.Equal(t, queryservice.INSYNC, jobStatusRes.JSON200.Status) -} diff --git a/test/queryService/queryservice_test.go b/test/queryService/queryservice_test.go index bbbfe9f1..7863c73d 100644 --- a/test/queryService/queryservice_test.go +++ b/test/queryService/queryservice_test.go @@ -46,7 +46,7 @@ func TestQueryService(t *testing.T) { Network: network, Name: test.QueryService, Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", + "CLIENT_SYNC_URL": "/i/am/here", "QUERY_VERSION_SYNC_URL": queryversionsyncurl, }, })