From 71f9802e1abbc8135b5bde135264a55588cb6b67 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Tue, 11 Feb 2025 15:22:59 +0000 Subject: [PATCH] Merged in feature/splitqueryrunning (pull request #57) Split Query Running + Debugging Full Flow * completedquerysyncrunner * spliitinglogic * synccomplete * informdependents * only push same collector * deps * livetesting * foundissue * some issues resolved * activeupdate * collectorupdatefixes * fix dbquesries * tests * tests * pollingdebug --- .dockerignore | 15 +- api/docCleanRunner/runner.go | 6 +- api/docCleanRunner/runner_test.go | 2 +- api/docTextRunner/runner.go | 6 +- api/docTextRunner/runner_test.go | 6 +- api/queryRunner/runner.go | 11 +- api/queryRunner/runner_test.go | 103 ++++- api/queryService/controllers.go | 2 + api/queryService/jobcollector.go | 5 +- api/queryService/jobcollector_test.go | 28 +- api/queryService/query.go | 4 +- api/queryService/query_test.go | 25 +- api/querySyncRunner/runner.go | 45 +- api/querySyncRunner/runner_test.go | 100 +++++ build/Dockerfile | 7 +- cmd/queryRunner/main.go | 31 +- cmd/queryService/main.go | 11 +- cmd/querySyncRunner/main.go | 55 +++ .../00000000000101_collector_views.up.sql | 27 +- database/queries/clean.sql | 14 +- database/queries/query.sql | 11 +- database/queries/result.sql | 102 +++-- database/queries/text.sql | 14 +- deployments/compose.local.yaml | 32 +- devbox.json | 2 +- internal/client/update.go | 6 +- internal/database/repository/clean.sql.go | 36 +- internal/database/repository/clean_test.go | 26 +- internal/database/repository/collector.sql.go | 5 +- .../database/repository/collector_test.go | 2 + internal/database/repository/document_test.go | 13 + internal/database/repository/models.go | 1 + internal/database/repository/query.sql.go | 44 ++ internal/database/repository/query_test.go | 121 ++++++ internal/database/repository/result.sql.go | 247 ++++++----- internal/database/repository/result_test.go | 256 ++++++----- internal/database/repository/text.sql.go | 36 +- internal/database/repository/text_test.go | 26 +- internal/document/clean/clean.go | 3 + internal/document/clean/create.go | 4 +- internal/document/clean/create_test.go | 2 +- internal/document/init/create.go | 6 +- internal/document/text/create.go | 7 +- internal/document/text/create_test.go | 6 +- internal/document/text/extract_test.go | 4 +- internal/document/version.go | 13 - internal/document/version_test.go | 12 - internal/job/collector/create.go | 34 ++ internal/job/collector/createprivate_test.go | 42 ++ internal/job/collector/update.go | 77 ++-- internal/job/collector/updateprivate_test.go | 407 ++++++++++++------ internal/job/update.go | 6 +- internal/query/create_test.go | 12 +- internal/query/createprivate_test.go | 13 +- internal/query/get_test.go | 4 +- internal/query/list_test.go | 4 +- internal/query/normalize.go | 4 +- internal/query/normalize_test.go | 121 ++++-- internal/query/result/get_test.go | 2 +- internal/query/result/list.go | 52 --- internal/query/result/list_test.go | 67 --- internal/query/result/process.go | 75 +++- internal/query/result/process_test.go | 100 +++-- internal/query/result/service.go | 13 +- internal/query/result/service_test.go | 2 +- internal/query/result/set.go | 45 -- internal/query/result/set/service.go | 30 ++ internal/query/result/set/service_test.go | 21 + internal/query/result/set/set.go | 94 ++++ internal/query/result/set/set_test.go | 277 ++++++++++++ internal/query/result/set_test.go | 66 --- internal/query/result/sync.go | 18 - internal/query/result/sync_test.go | 48 --- internal/query/service.go | 22 +- internal/query/service_test.go | 2 +- internal/query/sync.go | 163 ------- internal/query/sync/service.go | 28 ++ internal/query/sync/service_test.go | 29 ++ internal/query/sync/sync.go | 35 ++ internal/query/sync/sync_test.go | 88 ++++ internal/query/sync_test.go | 304 ------------- internal/query/test/service.go | 26 ++ internal/query/test/service_test.go | 23 + internal/query/{ => test}/test.go | 31 +- internal/query/{ => test}/test_test.go | 17 +- .../query/types/contextFull/process_test.go | 3 +- internal/query/types/contextFull/service.go | 3 +- internal/query/types/jsonExtractor/creator.go | 4 + .../query/types/jsonExtractor/creator_test.go | 4 +- internal/query/update.go | 58 ++- internal/query/update_test.go | 2 +- internal/query/updateprivate_test.go | 297 ++++++++++--- internal/server/runner/poll.go | 2 +- internal/serviceconfig/common.go | 6 +- internal/serviceconfig/logger/config.go | 30 +- internal/serviceconfig/logger/config_test.go | 20 + internal/serviceconfig/queue/delete.go | 3 + internal/serviceconfig/queue/query/config.go | 13 + .../serviceconfig/queue/query/config_test.go | 20 + internal/serviceconfig/queue/send.go | 3 + internal/test/ecosystem_test.go | 10 +- internal/test/runner_test.go | 3 + internal/validation/validation.go | 59 ++- internal/validation/validation_test.go | 132 ++++++ scripts/Taskfile.yml | 2 +- .../{compose.yml => local-deployments.yml} | 6 +- test/process_test.go | 49 ++- test/queryService/queryservice_test.go | 8 +- 108 files changed, 3013 insertions(+), 1666 deletions(-) create mode 100644 api/querySyncRunner/runner_test.go create mode 100644 cmd/querySyncRunner/main.go delete mode 100644 internal/query/result/list.go delete mode 100644 internal/query/result/list_test.go delete mode 100644 internal/query/result/set.go create mode 100644 internal/query/result/set/service.go create mode 100644 internal/query/result/set/service_test.go create mode 100644 internal/query/result/set/set.go create mode 100644 internal/query/result/set/set_test.go delete mode 100644 internal/query/result/set_test.go delete mode 100644 internal/query/result/sync.go delete mode 100644 internal/query/result/sync_test.go delete mode 100644 internal/query/sync.go create mode 100644 internal/query/sync/service.go create mode 100644 internal/query/sync/service_test.go create mode 100644 internal/query/sync/sync.go create mode 100644 internal/query/sync/sync_test.go delete mode 100644 internal/query/sync_test.go create mode 100644 internal/query/test/service.go create mode 100644 internal/query/test/service_test.go rename internal/query/{ => test}/test.go (50%) rename internal/query/{ => test}/test_test.go (84%) create mode 100644 internal/serviceconfig/queue/query/config.go create mode 100644 internal/serviceconfig/queue/query/config_test.go rename scripts/{compose.yml => local-deployments.yml} (92%) diff --git a/.dockerignore b/.dockerignore index 8a63cadd..8ebc1df2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,11 +39,20 @@ database/queries/ deployments/ mocks/ out/ -scripts/ serviceAPIs/ test/ **/*_test.go Taskfile.yml -sqlc.yml +scripts/ +.gitattributes +.gitignore +.golangci.yml +.mockery.yml +.openapi.yml +.yamllint.yml devbox.* -.*.yml \ No newline at end of file +sqlc.yml +vaccum.conf.yaml +.*.yml +**/*.cast +**/*.http \ No newline at end of file diff --git a/api/docCleanRunner/runner.go b/api/docCleanRunner/runner.go index fc8a1b02..2c81292f 100644 --- a/api/docCleanRunner/runner.go +++ b/api/docCleanRunner/runner.go @@ -29,12 +29,12 @@ func New(validator *validator.Validate, svc *Services) Runner { } } -type Create struct { +type Body struct { ID uuid.UUID `json:"id" validate:"required,uuid"` } func (s Runner) Process(ctx context.Context, req *types.Message) error { - var body Create + var body Body err := json.Unmarshal([]byte(*req.Body), &body) if err != nil { return err @@ -45,7 +45,7 @@ func (s Runner) Process(ctx context.Context, req *types.Message) error { return err } - err = s.svc.Clean.Create(ctx, body.ID) + err = s.svc.Clean.Clean(ctx, body.ID) if err != nil { return err } diff --git a/api/docCleanRunner/runner_test.go b/api/docCleanRunner/runner_test.go index 75f154e5..f94535ce 100644 --- a/api/docCleanRunner/runner_test.go +++ b/api/docCleanRunner/runner_test.go @@ -55,7 +55,7 @@ func TestDocCleanRunner(t *testing.T) { }) assert.NotNil(t, runner) - doc := doccleanrunner.Create{ + doc := doccleanrunner.Body{ ID: uuid.New(), } bodyBytes, err := json.Marshal(doc) diff --git a/api/docTextRunner/runner.go b/api/docTextRunner/runner.go index f9216429..93b92656 100644 --- a/api/docTextRunner/runner.go +++ b/api/docTextRunner/runner.go @@ -29,12 +29,12 @@ func New(validator *validator.Validate, svc *Services) Runner { } } -type Create struct { +type Body struct { ID uuid.UUID `json:"id" validate:"required,uuid"` } func (s Runner) Process(ctx context.Context, req *types.Message) error { - var body Create + var body Body err := json.Unmarshal([]byte(*req.Body), &body) if err != nil { return err @@ -45,7 +45,7 @@ func (s Runner) Process(ctx context.Context, req *types.Message) error { return err } - err = s.svc.Text.Create(ctx, body.ID) + err = s.svc.Text.Extract(ctx, body.ID) if err != nil { return err } diff --git a/api/docTextRunner/runner_test.go b/api/docTextRunner/runner_test.go index a67f15a5..e56c065e 100644 --- a/api/docTextRunner/runner_test.go +++ b/api/docTextRunner/runner_test.go @@ -55,7 +55,7 @@ func TestDocCleanRunner(t *testing.T) { }) assert.NotNil(t, runner) - doc := doctextrunner.Create{ + doc := doctextrunner.Body{ ID: uuid.New(), } bodyBytes, err := json.Marshal(doc) @@ -75,8 +75,8 @@ func TestDocCleanRunner(t *testing.T) { AddRow(false), ) pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(doc.ID)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key), + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(doc.ID), inloc.Bucket, inloc.Key, int32(1)), ) pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), int32(1), inloc.Bucket, inloc.Key). WillReturnResult(pgxmock.NewResult("", 1)) diff --git a/api/queryRunner/runner.go b/api/queryRunner/runner.go index 772357b8..90b8b844 100644 --- a/api/queryRunner/runner.go +++ b/api/queryRunner/runner.go @@ -3,7 +3,7 @@ package queryrunner import ( "context" "encoding/json" - "queryorchestration/internal/query" + resultset "queryorchestration/internal/query/result/set" "github.com/go-playground/validator/v10" @@ -13,7 +13,7 @@ import ( const Name = "queryRunner" type Services struct { - Query *query.Service + ResultSet *resultset.Service } type Runner struct { @@ -29,7 +29,7 @@ func New(validator *validator.Validate, svc *Services) Runner { } func (s *Runner) Process(ctx context.Context, req *types.Message) error { - var body query.Document + var body resultset.Set err := json.Unmarshal([]byte(*req.Body), &body) if err != nil { return err @@ -40,7 +40,10 @@ func (s *Runner) Process(ctx context.Context, req *types.Message) error { return err } - err = s.svc.Query.Sync(ctx, &body) + err = s.svc.ResultSet.Set(ctx, &resultset.Set{ + DocumentID: body.DocumentID, + QueryID: body.QueryID, + }) if err != nil { return err } diff --git a/api/queryRunner/runner_test.go b/api/queryRunner/runner_test.go index 142f7b15..507ce8b6 100644 --- a/api/queryRunner/runner_test.go +++ b/api/queryRunner/runner_test.go @@ -3,23 +3,34 @@ package queryrunner_test import ( "context" "encoding/json" - controllers "queryorchestration/api/queryRunner" + "fmt" + queryrunner "queryorchestration/api/queryRunner" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" - "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" - "queryorchestration/internal/serviceconfig" + resultprocessor "queryorchestration/internal/query/result/processor" + resultset "queryorchestration/internal/query/result/set" + "queryorchestration/internal/server/runner" + queryc "queryorchestration/internal/serviceconfig/queue/query" + queuemock "queryorchestration/mocks/queue" "testing" + "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/go-playground/validator/v10" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) +type QueryConfig struct { + runner.BaseConfig + queryc.QueryConfig +} + func TestQueryRunner(t *testing.T) { ctx := context.Background() @@ -28,26 +39,27 @@ func TestQueryRunner(t *testing.T) { t.Fatalf("failed to open pgxmock database: %v", err) } - cfg := &serviceconfig.BaseConfig{} + cfg := &QueryConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" - svc := query.New(cfg, &query.Services{ - Result: result.New(cfg), - Collector: collector.New(cfg, &collector.Services{}), - Document: document.New(cfg), - }) - - runner := controllers.New(validator.New(), &controllers.Services{ - Query: svc, + que := query.New(cfg) + runner := queryrunner.New(validator.New(), &queryrunner.Services{ + ResultSet: resultset.New(cfg, &resultset.Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }), }) assert.NotNil(t, runner) - doc := query.Document{ - ID: uuid.New(), - JobID: uuid.New(), - CleanVersion: 1, - TextVersion: 1, + doc := resultset.Set{ + DocumentID: uuid.New(), + QueryID: uuid.New(), } bodyBytes, err := json.Marshal(doc) assert.NoError(t, err) @@ -56,10 +68,61 @@ func TestQueryRunner(t *testing.T) { Body: &body, } - pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)). + qcfg := "{\"path\":\"examplekey\"}" + query := &resultprocessor.Query{ + ID: doc.QueryID, + Version: 2, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + Config: &qcfg, + } + params := &resultset.Set{ + DocumentID: doc.DocumentID, + } + + pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}), + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), ) + pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), + ) + pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), + ) + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte(qcfg)), + ) + pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(doc.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"queryId"}). + AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0])), + ) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), (*query.RequiredQueryIDs)[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) err = runner.Process(ctx, msg) assert.NoError(t, err) diff --git a/api/queryService/controllers.go b/api/queryService/controllers.go index 38077a8c..9af89044 100644 --- a/api/queryService/controllers.go +++ b/api/queryService/controllers.go @@ -6,6 +6,7 @@ import ( "queryorchestration/internal/job" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + querytest "queryorchestration/internal/query/test" "github.com/go-playground/validator/v10" ) @@ -18,6 +19,7 @@ type Services struct { Query *query.Service Client *client.Service Job *job.Service + QueryTest *querytest.Service } type Controllers struct { diff --git a/api/queryService/jobcollector.go b/api/queryService/jobcollector.go index ab697200..13f0431e 100644 --- a/api/queryService/jobcollector.go +++ b/api/queryService/jobcollector.go @@ -44,10 +44,11 @@ func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UU var fields *map[string]uuid.UUID if req.Fields != nil { - fields := &map[string]uuid.UUID{} + fs := map[string]uuid.UUID{} for _, field := range *req.Fields { - (*fields)[field.Name] = field.QueryId + fs[field.Name] = field.QueryId } + fields = &fs } err := s.svc.Collector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{ diff --git a/api/queryService/jobcollector_test.go b/api/queryService/jobcollector_test.go index eb2f40fb..c1841df3 100644 --- a/api/queryService/jobcollector_test.go +++ b/api/queryService/jobcollector_test.go @@ -16,6 +16,7 @@ import ( "github.com/go-playground/validator/v10" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/labstack/echo/v4" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" @@ -30,9 +31,22 @@ func TestUpdateJobCollector(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) + current := collector.Collector{ + ID: uuid.New(), + JobID: uuid.New(), + ActiveVersion: 1, + LatestVersion: 4, + } + av := int32(2) body := queryservice.JobCollectorUpdate{ ActiveVersion: &av, + Fields: &[]queryservice.JobCollectorField{ + { + Name: "a", + QueryId: uuid.New(), + }, + }, } bodyBytes, err := json.Marshal(body) assert.NoError(t, err) @@ -49,13 +63,6 @@ func TestUpdateJobCollector(t *testing.T) { }), }) - current := collector.Collector{ - ID: uuid.New(), - JobID: uuid.New(), - ActiveVersion: 1, - LatestVersion: 4, - } - ctx.Set("id", current.JobID) pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)). @@ -63,7 +70,14 @@ func TestUpdateJobCollector(t *testing.T) { pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]pgtype.UUID{database.MustToDBUUID((*body.Fields)[0].QueryId)}). + WillReturnRows( + pgxmock.NewRows([]string{"exist"}). + AddRow(true), + ) pool.ExpectBeginTx(pgx.TxOptions{}) + pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)). + WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectExec("name: UpdateCollector :exec").WithArgs(int32(5), int32(2), database.MustToDBUUID(current.ID)). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() diff --git a/api/queryService/query.go b/api/queryService/query.go index 2c1a67cf..ee2aaf06 100644 --- a/api/queryService/query.go +++ b/api/queryService/query.go @@ -3,7 +3,7 @@ package queryservice import ( "fmt" "net/http" - "queryorchestration/internal/query" + "queryorchestration/internal/query/result" resultprocessor "queryorchestration/internal/query/result/processor" "github.com/labstack/echo/v4" @@ -90,7 +90,7 @@ func (s *Controllers) TestQuery(ctx echo.Context, id types.UUID) error { return echo.NewHTTPError(http.StatusBadRequest, err) } - value, err := s.svc.Query.Test(ctx.Request().Context(), query.Test{ + value, err := s.svc.QueryTest.Test(ctx.Request().Context(), result.Process{ QueryID: id, QueryVersion: req.QueryVersion, DocumentID: req.DocumentId, diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index f2a06e30..3c861e78 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -12,6 +12,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + querytest "queryorchestration/internal/query/test" "queryorchestration/internal/serviceconfig" "strings" "testing" @@ -35,7 +36,7 @@ func TestCreateQuery(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(cfg, &query.Services{}), + Query: query.New(cfg), }) body := queryservice.QueryCreate{ @@ -75,7 +76,7 @@ func TestListQueries(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(cfg, &query.Services{}), + Query: query.New(cfg), }) e := echo.New() @@ -118,7 +119,7 @@ func TestGetQuery(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(cfg, &query.Services{}), + Query: query.New(cfg), }) e := echo.New() @@ -160,7 +161,7 @@ func TestUpdateQuery(t *testing.T) { cfg.DBQueries = repository.New(pool) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(cfg, &query.Services{}), + Query: query.New(cfg), }) av := int32(2) @@ -209,12 +210,16 @@ func TestTestQuery(t *testing.T) { col := collector.New(cfg, &collector.Services{ Document: docsvc, }) + que := query.New(cfg) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ Collector: col, - Query: query.New(cfg, &query.Services{ + Query: que, + QueryTest: querytest.New(cfg, &querytest.Services{ Document: docsvc, Collector: col, - Result: result.New(cfg), + Result: result.New(cfg, &result.Services{ + Query: que, + }), }), }) @@ -227,7 +232,7 @@ func TestTestQuery(t *testing.T) { JobID: coll.JobID, Hash: "example_hash", } - params := &query.Test{ + params := &result.Process{ QueryID: uuid.New(), DocumentID: doc.ID, QueryVersion: int32(3), @@ -260,10 +265,10 @@ func TestTestQuery(t *testing.T) { pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion). + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull), + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"), ) pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( pgxmock.NewRows([]string{"id", "config"}). diff --git a/api/querySyncRunner/runner.go b/api/querySyncRunner/runner.go index a9f12f0a..ac7f7c7b 100644 --- a/api/querySyncRunner/runner.go +++ b/api/querySyncRunner/runner.go @@ -1,11 +1,54 @@ package querysyncrunner import ( + "context" + "encoding/json" + querysync "queryorchestration/internal/query/sync" + + "github.com/go-playground/validator/v10" "github.com/google/uuid" + + "github.com/aws/aws-sdk-go-v2/service/sqs/types" ) const Name = "querySyncRunner" -type Create struct { +type Services struct { + QuerySync *querysync.Service +} + +type Runner struct { + validator *validator.Validate + svc *Services +} + +func New(validator *validator.Validate, svc *Services) Runner { + return Runner{ + validator: validator, + svc: svc, + } +} + +type Body struct { ID uuid.UUID `json:"id" validate:"required,uuid"` } + +func (s *Runner) Process(ctx context.Context, req *types.Message) error { + var body Body + err := json.Unmarshal([]byte(*req.Body), &body) + if err != nil { + return err + } + + err = s.validator.Struct(body) + if err != nil { + return err + } + + err = s.svc.QuerySync.Sync(ctx, body.ID) + if err != nil { + return err + } + + return nil +} diff --git a/api/querySyncRunner/runner_test.go b/api/querySyncRunner/runner_test.go new file mode 100644 index 00000000..ba7bbe72 --- /dev/null +++ b/api/querySyncRunner/runner_test.go @@ -0,0 +1,100 @@ +package querysyncrunner_test + +import ( + "context" + "encoding/json" + "fmt" + querysyncrunner "queryorchestration/api/querySyncRunner" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query" + "queryorchestration/internal/query/result" + resultset "queryorchestration/internal/query/result/set" + querysync "queryorchestration/internal/query/sync" + "queryorchestration/internal/server/runner" + queryc "queryorchestration/internal/serviceconfig/queue/query" + queuemock "queryorchestration/mocks/queue" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/go-playground/validator/v10" + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type QuerySyncConfig struct { + runner.BaseConfig + queryc.QueryConfig +} + +func TestQueryRunner(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + + cfg := &QuerySyncConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + + que := query.New(cfg) + svc := querysync.New(cfg, &querysync.Services{ + ResultSet: resultset.New(cfg, &resultset.Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }), + }) + + runner := querysyncrunner.New(validator.New(), &querysyncrunner.Services{ + QuerySync: svc, + }) + assert.NotNil(t, runner) + + doc := querysyncrunner.Body{ + ID: uuid.New(), + } + bodyBytes, err := json.Marshal(doc) + assert.NoError(t, err) + body := string(bodyBytes) + msg := &types.Message{ + Body: &body, + } + + qs := []uuid.UUID{ + uuid.New(), + } + + pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(database.MustToDBUUID(doc.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"isextracted"}). + AddRow(true), + ) + pool.ExpectQuery("name: ListUnsyncedNoDepsQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(qs[0])), + ) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", doc.ID.String(), qs[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + err = runner.Process(ctx, msg) + assert.NoError(t, err) +} diff --git a/build/Dockerfile b/build/Dockerfile index 584aa899..0e870a10 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -6,10 +6,9 @@ FROM golang:${GO_VERSION} AS build WORKDIR /app -RUN --mount=type=cache,target=/go/pkg/mod/ \ - --mount=type=bind,source=go.sum,target=go.sum \ - --mount=type=bind,source=go.mod,target=go.mod \ - go mod download -x +COPY go.mod . +COPY go.sum . +COPY vendor/ vendor/ ARG TARGETARCH diff --git a/cmd/queryRunner/main.go b/cmd/queryRunner/main.go index c6f81ec2..6063ab17 100644 --- a/cmd/queryRunner/main.go +++ b/cmd/queryRunner/main.go @@ -4,35 +4,38 @@ import ( "context" "log/slog" "os" - controllers "queryorchestration/api/queryRunner" - "queryorchestration/internal/document" - "queryorchestration/internal/job/collector" + queryrunner "queryorchestration/api/queryRunner" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + resultset "queryorchestration/internal/query/result/set" "queryorchestration/internal/server/runner" + queryc "queryorchestration/internal/serviceconfig/queue/query" _ "github.com/lib/pq" ) +type QueryConfig struct { + runner.BaseConfig + queryc.QueryConfig +} + func main() { ctx := context.Background() - cfg := &runner.BaseConfig{} + cfg := &QueryConfig{} cfg.ControllerFunc = func() runner.Controller { - doc := document.New(cfg) - col := collector.New(cfg, &collector.Services{ - Document: doc, + que := query.New(cfg) + res := result.New(cfg, &result.Services{ + Query: que, }) - res := result.New(cfg) - svc := query.New(cfg, &query.Services{ - Result: res, - Collector: col, - Document: doc, + resset := resultset.New(cfg, &resultset.Services{ + Result: res, + Query: que, }) - c := controllers.New(cfg.GetValidator(), &controllers.Services{ - Query: svc, + c := queryrunner.New(cfg.GetValidator(), &queryrunner.Services{ + ResultSet: resset, }) return &c diff --git a/cmd/queryService/main.go b/cmd/queryService/main.go index ff9e79ff..734c9efa 100644 --- a/cmd/queryService/main.go +++ b/cmd/queryService/main.go @@ -13,6 +13,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + querytest "queryorchestration/internal/query/test" service "queryorchestration/internal/server/service" "github.com/getkin/kin-openapi/openapi3" @@ -26,14 +27,17 @@ func main() { cfg.RegisterHandlersFunc = func() (*openapi3.T, error) { exp := export.New() - res := result.New(cfg) + que := query.New(cfg) + res := result.New(cfg, &result.Services{ + Query: que, + }) doc := document.New(cfg) col := collector.New(cfg, &collector.Services{ Document: doc, }) - que := query.New(cfg, &query.Services{ - Result: res, + quetest := querytest.New(cfg, &querytest.Services{ Collector: col, + Result: res, Document: doc, }) cli := client.New(cfg) @@ -48,6 +52,7 @@ func main() { Query: que, Client: cli, Job: jbb, + QueryTest: quetest, } cons := queryservice.NewControllers(cfg.GetValidator(), services) diff --git a/cmd/querySyncRunner/main.go b/cmd/querySyncRunner/main.go new file mode 100644 index 00000000..24709894 --- /dev/null +++ b/cmd/querySyncRunner/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "log/slog" + "os" + querysyncrunner "queryorchestration/api/querySyncRunner" + "queryorchestration/internal/query" + "queryorchestration/internal/query/result" + resultset "queryorchestration/internal/query/result/set" + querysync "queryorchestration/internal/query/sync" + "queryorchestration/internal/server/runner" + queryc "queryorchestration/internal/serviceconfig/queue/query" + + _ "github.com/lib/pq" +) + +type QuerySyncConfig struct { + runner.BaseConfig + queryc.QueryConfig +} + +func main() { + ctx := context.Background() + + cfg := &QuerySyncConfig{} + + cfg.ControllerFunc = func() runner.Controller { + que := query.New(cfg) + res := result.New(cfg, &result.Services{ + Query: que, + }) + resset := resultset.New(cfg, &resultset.Services{ + Query: que, + Result: res, + }) + svc := querysync.New(cfg, &querysync.Services{ + ResultSet: resset, + }) + + c := querysyncrunner.New(cfg.GetValidator(), &querysyncrunner.Services{ + QuerySync: svc, + }) + + return &c + } + + server, err := runner.New(ctx, cfg) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + server.Listen(ctx) +} diff --git a/database/migrations/00000000000101_collector_views.up.sql b/database/migrations/00000000000101_collector_views.up.sql index b9726495..c434ba11 100644 --- a/database/migrations/00000000000101_collector_views.up.sql +++ b/database/migrations/00000000000101_collector_views.up.sql @@ -11,23 +11,24 @@ CREATE VIEW fullActiveCollectors AS GROUP BY c.id, c.jobId, cv.minCleanVersion, cv.minTextVersion; CREATE VIEW collectorQueryDependencyTree AS - WITH RECURSIVE collectorQueryDependencyTree AS ( - SELECT cq.id as collectorId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion - FROM fullActiveQueries as aqc - LEFT JOIN ( - SELECT DISTINCT c.id, array_agg(q.queryId) as queryIds - FROM collectors as c - LEFT JOIN collectorQueries as q ON c.id = q.collectorId - AND c.activeVersion >= q.addedVersion - and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1) - GROUP BY c.id - ) as cq ON aqc.id = any(cq.queryIds) + WITH RECURSIVE + coll as ( + SELECT DISTINCT c.id, c.jobId, q.queryId + FROM collectors as c + LEFT JOIN collectorQueries as q ON c.id = q.collectorId + AND c.activeVersion >= q.addedVersion + and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1) + ), + collectorQueryDependencyTree AS ( + SELECT cq.id as collectorId, cq.jobId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion + FROM coll as cq + JOIN fullActiveQueries as aqc on cq.queryId = aqc.id UNION ALL - SELECT acq.collectorId, q.id as queryId, q.type, q.requiredIds, q.activeVersion + SELECT acq.collectorId, acq.jobId, q.id as queryId, q.type, q.requiredIds, q.activeVersion FROM fullActiveQueries as q JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds) ) - SELECT collectorId, queryId, type, activeVersion as queryVersion, requiredIds + SELECT DISTINCT collectorId, jobId, queryId, type, activeVersion as queryVersion, requiredIds FROM collectorQueryDependencyTree; \ No newline at end of file diff --git a/database/queries/clean.sql b/database/queries/clean.sql index aa915c11..5c8f2801 100644 --- a/database/queries/clean.sql +++ b/database/queries/clean.sql @@ -3,19 +3,19 @@ SELECT EXISTS( SELECT 1 FROM documentCleans AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) ); -- name: AddDocumentCleanEntry :exec INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4); -- name: GetDocumentCleanEntry :one -SELECT dc.documentId, dc.bucket, dc.key +SELECT dc.documentId, dc.bucket, dc.key, dc.version FROM documentCleans AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) ORDER BY d.id DESC LIMIT 1; \ No newline at end of file diff --git a/database/queries/query.sql b/database/queries/query.sql index 82110895..072c4792 100644 --- a/database/queries/query.sql +++ b/database/queries/query.sql @@ -48,4 +48,13 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist -- name: IsQueryInDependencyTree :one SELECT EXISTS ( SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1) -); \ No newline at end of file +); + +-- name: ListQueryDirectDependentsByDocumentID :many +WITH doc AS ( + SELECT id, jobId FROM documents where id = $2 +) +SELECT dt.queryId + FROM collectorQueryDependencyTree as dt + JOIN doc as d on d.jobId = dt.jobId + where $1 = any(dt.requiredIds); \ No newline at end of file diff --git a/database/queries/result.sql b/database/queries/result.sql index dc17bc57..beabb39c 100644 --- a/database/queries/result.sql +++ b/database/queries/result.sql @@ -1,49 +1,77 @@ -- name: ListQueryRequirementValues :many -WITH latest_versions AS ( - SELECT - r.queryId, - MAX(r.cleanVersion) as max_clean_version, - MAX(r.textVersion) as max_text_version - FROM results r - WHERE r.documentId = $3 - AND r.queryVersion = $2 - GROUP BY r.queryId -) -SELECT rq.requiredQueryId as queryId, r.value, q.type +WITH reqQueries as ( + SELECT q.id as queryId, q.activeVersion, q.type FROM requiredQueries as rq - JOIN results as r on r.queryId = rq.requiredQueryId JOIN queries as q on q.id = rq.requiredQueryId - JOIN latest_versions lv ON lv.queryId = r.queryId - WHERE rq.queryId = $1 and r.documentId = $3 - and $2 >= rq.addedVersion - and $2 < COALESCE(rq.removedVersion, $2 + 1) - and r.queryVersion = $2 - and r.cleanVersion >= $4 and r.textVersion >= $5 - AND r.cleanVersion = lv.max_clean_version - AND r.textVersion = lv.max_text_version; + WHERE rq.queryId = @queryId + and @version >= rq.addedVersion + and (rq.removedVersion is null or @version < rq.removedVersion) +), +codeVersions as ( + SELECT + d.id as documentId, + coalesce(ccv.minCleanVersion, 1) as minCleanVersion, + coalesce(ccv.minTextVersion, 1) as minTextVersion + FROM documents as d + LEFT JOIN collectors as c on c.jobId = d.jobId + LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId + and c.activeVersion >= ccv.addedVersion + and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion) + WHERE d.id = @documentId + LIMIT 1 +), +latestVersions AS ( + SELECT + rq.queryId, + rq.type, + r.queryVersion, + r.cleanVersion, + r.textVersion, + ROW_NUMBER() OVER ( + PARTITION BY rq.queryId + ORDER BY r.cleanVersion DESC, r.textVersion DESC + ) as rowNumber + FROM reqQueries as rq + LEFT JOIN results as r ON rq.queryId = r.queryId + and r.documentId = @documentId + JOIN codeVersions as ccv on ccv.documentId = r.documentId + WHERE r.documentId = @documentId + and r.queryVersion = rq.activeVersion + and r.cleanVersion >= ccv.minCleanVersion + and r.textVersion >= ccv.minTextVersion +) +SELECT lv.queryId, lv.type, r.value +FROM latestVersions as lv +JOIN results as r ON r.queryId = lv.queryId + and r.documentId = @documentId + and r.queryVersion = lv.queryVersion + and r.cleanVersion = lv.cleanVersion + and r.textVersion = lv.textVersion + and lv.rowNumber = 1; --- name: SetResult :one -INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id; +-- name: SetResult :exec +INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6); -- name: GetResultValueWithVersion :one SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5; --- name: ListUnsyncedQueriesByDocId :many -WITH RECURSIVE unsyncedQueries AS ( - SELECT dt.queryId, dt.requiredIds - from documents as d - JOIN fullActiveCollectors as c on d.jobId = c.jobId - JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId +-- name: ListUnsyncedNoDepsQueriesByDocId :many +WITH docs as ( + SELECT id, jobId from documents where id = $1 +), +unsyncedQueries AS ( + SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id + from docs as d + JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId + JOIN fullActiveCollectors as c on c.id = dt.collectorId LEFT JOIN results as r on r.queryId = dt.queryId + and r.documentId = d.id and r.queryVersion = dt.queryVersion and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion - where d.id = $1 and r.value is null - - UNION - - SELECT DISTINCT dt.queryId, dt.requiredIds - FROM unsyncedQueries as u - JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) + where r.value is null ) -SELECT * FROM fullActiveQueries -WHERE id in (SELECT queryId FROM unsyncedQueries); \ No newline at end of file +SELECT DISTINCT queryId FROM unsyncedQueries as baseuq + WHERE NOT EXISTS ( + SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds) + ); + \ No newline at end of file diff --git a/database/queries/text.sql b/database/queries/text.sql index 5b53294b..72e324dc 100644 --- a/database/queries/text.sql +++ b/database/queries/text.sql @@ -3,19 +3,19 @@ SELECT EXISTS( SELECT 1 FROM documentTextExtractions AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) ); -- name: AddDocumentTextEntry :exec INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4); -- name: GetDocumentTextEntry :one -SELECT dc.documentId, dc.bucket, dc.key +SELECT dc.documentId, dc.bucket, dc.key, dc.version FROM documentTextExtractions AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) ORDER BY d.id DESC LIMIT 1; \ No newline at end of file diff --git a/deployments/compose.local.yaml b/deployments/compose.local.yaml index b8ca4092..38bb37e9 100644 --- a/deployments/compose.local.yaml +++ b/deployments/compose.local.yaml @@ -7,6 +7,7 @@ services: - db - localstack environment: + LOG_LEVEL: DEBUG QUEUE_URL: ${DOCUMENT_INIT_URL} DOCUMENT_CLEAN_URL: ${DOCUMENT_CLEAN_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} @@ -30,6 +31,7 @@ services: - db - localstack environment: + LOG_LEVEL: DEBUG QUEUE_URL: ${DOCUMENT_CLEAN_URL} DOCUMENT_TEXT_URL: ${DOCUMENT_TEXT_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} @@ -53,6 +55,7 @@ services: - db - localstack environment: + LOG_LEVEL: DEBUG QUEUE_URL: ${DOCUMENT_TEXT_URL} QUERY_SYNC_URL: ${QUERY_SYNC_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} @@ -69,6 +72,30 @@ services: AWS_S3_USE_PATH_STYLE: true networks: - server-network + query_sync_runner: + image: queryorchestration:latest + command: ["./querySyncRunner"] + depends_on: + - db + - localstack + environment: + LOG_LEVEL: DEBUG + QUEUE_URL: ${QUERY_SYNC_URL} + QUERY_URL: ${QUERY_URL} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} + AWS_REGION: ${AWS_REGION} + DB_USER: ${DB_USER} + DB_PASS: ${DB_PASS} + DB_HOST: db + DB_PORT: 5432 + DB_NAME: ${DB_NAME} + DB_NOSSL: ${DB_NOSSL} + AWS_ENDPOINT_URL: "http://localstack:4566" + AWS_S3_USE_PATH_STYLE: true + networks: + - server-network query_runner: image: queryorchestration:latest command: ["./queryRunner"] @@ -76,7 +103,9 @@ services: - db - localstack environment: - QUEUE_URL: ${QUERY_RUNNER_URL} + LOG_LEVEL: DEBUG + QUEUE_URL: ${QUERY_URL} + QUERY_URL: ${QUERY_URL} AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} @@ -102,6 +131,7 @@ services: expose: - 8080 environment: + LOG_LEVEL: DEBUG AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN} diff --git a/devbox.json b/devbox.json index 5e981ffe..1adab88e 100644 --- a/devbox.json +++ b/devbox.json @@ -49,7 +49,7 @@ "DOCUMENT_CLEAN_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_clean", "DOCUMENT_TEXT_URL": "http://localstack:4566/queue/us-east-1/000000000000/document_text", "QUERY_SYNC_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_sync", - "QUERY_RUNNER_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", + "QUERY_URL": "http://localstack:4566/queue/us-east-1/000000000000/query_runner", "BUCKET_IN": "documentin" }, "env_from": ".env" diff --git a/internal/client/update.go b/internal/client/update.go index 2ac03511..0c1006c9 100644 --- a/internal/client/update.go +++ b/internal/client/update.go @@ -3,8 +3,10 @@ package client import ( "context" "errors" + "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/validation" "github.com/google/uuid" ) @@ -58,6 +60,8 @@ func (s *Service) submitUpdate(ctx context.Context, entity *Update) error { } } + slog.Debug("client updated", "update", *entity) + return nil }) } @@ -74,7 +78,7 @@ func (s *Service) normalizeUpdateParams(current *Client, entity *Update) error { current.normalizeCanSyncUpdate(&entity.CanSync) - if entity.Name == nil && entity.CanSync == nil { + if validation.AreAllPointersNilExcept(entity) { return errors.New("no updates presented") } diff --git a/internal/database/repository/clean.sql.go b/internal/database/repository/clean.sql.go index 2c53430f..8f3339af 100644 --- a/internal/database/repository/clean.sql.go +++ b/internal/database/repository/clean.sql.go @@ -36,12 +36,12 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle } const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one -SELECT dc.documentId, dc.bucket, dc.key +SELECT dc.documentId, dc.bucket, dc.key, dc.version FROM documentCleans AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) ORDER BY d.id DESC LIMIT 1 ` @@ -49,21 +49,27 @@ type GetDocumentCleanEntryRow struct { Documentid pgtype.UUID `db:"documentid"` Bucket string `db:"bucket"` Key string `db:"key"` + Version int32 `db:"version"` } // GetDocumentCleanEntry // -// SELECT dc.documentId, dc.bucket, dc.key +// SELECT dc.documentId, dc.bucket, dc.key, dc.version // FROM documentCleans AS dc // JOIN documents AS d ON d.id = dc.documentId -// JOIN collectors as c ON d.jobId = c.jobId -// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId -// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion +// LEFT JOIN collectors as c ON d.jobId = c.jobId +// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId +// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) // ORDER BY d.id DESC LIMIT 1 func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) { row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid) var i GetDocumentCleanEntryRow - err := row.Scan(&i.Documentid, &i.Bucket, &i.Key) + err := row.Scan( + &i.Documentid, + &i.Bucket, + &i.Key, + &i.Version, + ) return &i, err } @@ -72,9 +78,9 @@ SELECT EXISTS( SELECT 1 FROM documentCleans AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) ) ` @@ -84,9 +90,9 @@ SELECT EXISTS( // SELECT 1 // FROM documentCleans AS dc // JOIN documents AS d ON d.id = dc.documentId -// JOIN collectors as c ON d.jobId = c.jobId -// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId -// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion +// LEFT JOIN collectors as c ON d.jobId = c.jobId +// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId +// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1) // ) func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) { row := q.db.QueryRow(ctx, isDocumentClean, documentid) diff --git a/internal/database/repository/clean_test.go b/internal/database/repository/clean_test.go index 4bc56281..5609990a 100644 --- a/internal/database/repository/clean_test.go +++ b/internal/database/repository/clean_test.go @@ -42,20 +42,13 @@ func TestClean(t *testing.T) { assert.NoError(t, err) assert.NotEmpty(t, id) - collId, err := queries.CreateCollector(ctx, jobId) - assert.NoError(t, err) - err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{ - Collectorid: collId, - Addedversion: 1, - Mincleanversion: 2, - Mintextversion: 1, - }) - assert.NoError(t, err) - isclean, err := queries.IsDocumentClean(ctx, id) assert.NoError(t, err) assert.False(t, isclean) + _, err = queries.GetDocumentCleanEntry(ctx, id) + assert.Error(t, err) + bucket := "example_bucket" key := "example_key" err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{ @@ -66,18 +59,6 @@ func TestClean(t *testing.T) { }) assert.NoError(t, err) - isclean, err = queries.IsDocumentClean(ctx, id) - assert.NoError(t, err) - assert.False(t, isclean) - - err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{ - Collectorid: collId, - Addedversion: 1, - Mincleanversion: 1, - Mintextversion: 1, - }) - assert.NoError(t, err) - isclean, err = queries.IsDocumentClean(ctx, id) assert.NoError(t, err) assert.True(t, isclean) @@ -88,5 +69,6 @@ func TestClean(t *testing.T) { Documentid: id, Bucket: bucket, Key: key, + Version: 1, }, clean) } diff --git a/internal/database/repository/collector.sql.go b/internal/database/repository/collector.sql.go index 37b63584..1c885b29 100644 --- a/internal/database/repository/collector.sql.go +++ b/internal/database/repository/collector.sql.go @@ -118,12 +118,12 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (* } const listCollectorQueries = `-- name: ListCollectorQueries :many -SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 +SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 ` // ListCollectorQueries // -// SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 +// SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1 func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) { rows, err := q.db.Query(ctx, listCollectorQueries, collectorid) if err != nil { @@ -135,6 +135,7 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U var i Collectorquerydependencytree if err := rows.Scan( &i.Collectorid, + &i.Jobid, &i.Queryid, &i.Type, &i.Queryversion, diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index 76be778b..7b607027 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -109,6 +109,7 @@ func TestCollector(t *testing.T) { assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{ { Collectorid: collId, + Jobid: jobId, Queryid: jsonId, Queryversion: 1, Type: repository.QuerytypeJsonExtractor, @@ -116,6 +117,7 @@ func TestCollector(t *testing.T) { }, { Collectorid: collId, + Jobid: jobId, Queryid: contextId, Queryversion: 1, Type: repository.QuerytypeContextFull, diff --git a/internal/database/repository/document_test.go b/internal/database/repository/document_test.go index 9e65c4c7..2accd499 100644 --- a/internal/database/repository/document_test.go +++ b/internal/database/repository/document_test.go @@ -41,6 +41,19 @@ func TestDocument(t *testing.T) { }) assert.NoError(t, err) assert.NotEmpty(t, id) + documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash_two", + }) + assert.NoError(t, err) + assert.NotEmpty(t, documentTwoID) + jobTwoId, err := queries.CreateJob(ctx, clientId) + assert.NoError(t, err) + _, err = queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobTwoId, + Hash: "example_hash", + }) + assert.NoError(t, err) doc, err := queries.GetDocument(ctx, id) assert.NoError(t, err) diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index 3542b779..348c4ffe 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -101,6 +101,7 @@ type Collectorquery struct { type Collectorquerydependencytree struct { Collectorid pgtype.UUID `db:"collectorid"` + Jobid pgtype.UUID `db:"jobid"` Queryid pgtype.UUID `db:"queryid"` Type Querytype `db:"type"` Queryversion int32 `db:"queryversion"` diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index 18ee3b39..fbc335a3 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -268,6 +268,50 @@ func (q *Queries) ListQueriesById(ctx context.Context, id []pgtype.UUID) ([]*Ful return items, nil } +const listQueryDirectDependentsByDocumentID = `-- name: ListQueryDirectDependentsByDocumentID :many +WITH doc AS ( + SELECT id, jobId FROM documents where id = $2 +) +SELECT dt.queryId + FROM collectorQueryDependencyTree as dt + JOIN doc as d on d.jobId = dt.jobId + where $1 = any(dt.requiredIds) +` + +type ListQueryDirectDependentsByDocumentIDParams struct { + Requiredids pgtype.UUID `db:"requiredids"` + ID pgtype.UUID `db:"id"` +} + +// ListQueryDirectDependentsByDocumentID +// +// WITH doc AS ( +// SELECT id, jobId FROM documents where id = $2 +// ) +// SELECT dt.queryId +// FROM collectorQueryDependencyTree as dt +// JOIN doc as d on d.jobId = dt.jobId +// where $1 = any(dt.requiredIds) +func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) { + rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Requiredids, arg.ID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []pgtype.UUID{} + for rows.Next() { + var queryid pgtype.UUID + if err := rows.Scan(&queryid); err != nil { + return nil, err + } + items = append(items, queryid) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const removeQueryConfig = `-- name: RemoveQueryConfig :exec UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null ` diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index fbdbabe5..daaf89bc 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -183,11 +183,38 @@ func TestQueryDependencyTree(t *testing.T) { queries := cfg.GetDBQueries() + clientID, err := queries.CreateClient(ctx, "example_client") + assert.NoError(t, err) + jobID, err := queries.CreateJob(ctx, clientID) + assert.NoError(t, err) + collID, err := queries.CreateCollector(ctx, jobID) + assert.NoError(t, err) + docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobID, + Hash: "sample", + }) + assert.NoError(t, err) + contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) + dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.NoError(t, err) + + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: jsonQueryID, Requiredqueryid: contextQueryID, @@ -195,8 +222,56 @@ func TestQueryDependencyTree(t *testing.T) { }) assert.NoError(t, err) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Collectorid: collID, + Name: "example_name'", + Queryid: jsonQueryID, + Addedversion: 1, + }) + assert.NoError(t, err) + + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) + secondJsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) assert.NoError(t, err) + + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: secondJsonQueryID, Requiredqueryid: jsonQueryID, @@ -204,6 +279,52 @@ func TestQueryDependencyTree(t *testing.T) { }) assert.NoError(t, err) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: secondJsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Collectorid: collID, + Name: "example_name'", + Queryid: secondJsonQueryID, + Addedversion: 1, + }) + assert.NoError(t, err) + + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: secondJsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: jsonQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents) + dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: docID, + Requiredids: contextQueryID, + }) + assert.NoError(t, err) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents) + isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ Requiredqueryid: jsonQueryID, ID: []pgtype.UUID{contextQueryID}, diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index b71594df..044d17d1 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -45,76 +45,122 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV } const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many -WITH latest_versions AS ( - SELECT - r.queryId, - MAX(r.cleanVersion) as max_clean_version, - MAX(r.textVersion) as max_text_version - FROM results r - WHERE r.documentId = $3 - AND r.queryVersion = $2 - GROUP BY r.queryId -) -SELECT rq.requiredQueryId as queryId, r.value, q.type +WITH reqQueries as ( + SELECT q.id as queryId, q.activeVersion, q.type FROM requiredQueries as rq - JOIN results as r on r.queryId = rq.requiredQueryId JOIN queries as q on q.id = rq.requiredQueryId - JOIN latest_versions lv ON lv.queryId = r.queryId - WHERE rq.queryId = $1 and r.documentId = $3 - and $2 >= rq.addedVersion - and $2 < COALESCE(rq.removedVersion, $2 + 1) - and r.queryVersion = $2 - and r.cleanVersion >= $4 and r.textVersion >= $5 - AND r.cleanVersion = lv.max_clean_version - AND r.textVersion = lv.max_text_version + WHERE rq.queryId = $2 + and $3 >= rq.addedVersion + and (rq.removedVersion is null or $3 < rq.removedVersion) +), +codeVersions as ( + SELECT + d.id as documentId, + coalesce(ccv.minCleanVersion, 1) as minCleanVersion, + coalesce(ccv.minTextVersion, 1) as minTextVersion + FROM documents as d + LEFT JOIN collectors as c on c.jobId = d.jobId + LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId + and c.activeVersion >= ccv.addedVersion + and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion) + WHERE d.id = $1 + LIMIT 1 +), +latestVersions AS ( + SELECT + rq.queryId, + rq.type, + r.queryVersion, + r.cleanVersion, + r.textVersion, + ROW_NUMBER() OVER ( + PARTITION BY rq.queryId + ORDER BY r.cleanVersion DESC, r.textVersion DESC + ) as rowNumber + FROM reqQueries as rq + LEFT JOIN results as r ON rq.queryId = r.queryId + and r.documentId = $1 + JOIN codeVersions as ccv on ccv.documentId = r.documentId + WHERE r.documentId = $1 + and r.queryVersion = rq.activeVersion + and r.cleanVersion >= ccv.minCleanVersion + and r.textVersion >= ccv.minTextVersion +) +SELECT lv.queryId, lv.type, r.value +FROM latestVersions as lv +JOIN results as r ON r.queryId = lv.queryId + and r.documentId = $1 + and r.queryVersion = lv.queryVersion + and r.cleanVersion = lv.cleanVersion + and r.textVersion = lv.textVersion + and lv.rowNumber = 1 ` type ListQueryRequirementValuesParams struct { - Queryid pgtype.UUID `db:"queryid"` - Addedversion int32 `db:"addedversion"` - Documentid pgtype.UUID `db:"documentid"` - Cleanversion int32 `db:"cleanversion"` - Textversion int32 `db:"textversion"` + Documentid pgtype.UUID `db:"documentid"` + Queryid pgtype.UUID `db:"queryid"` + Version int32 `db:"version"` } type ListQueryRequirementValuesRow struct { Queryid pgtype.UUID `db:"queryid"` - Value string `db:"value"` Type Querytype `db:"type"` + Value string `db:"value"` } // ListQueryRequirementValues // -// WITH latest_versions AS ( -// SELECT -// r.queryId, -// MAX(r.cleanVersion) as max_clean_version, -// MAX(r.textVersion) as max_text_version -// FROM results r -// WHERE r.documentId = $3 -// AND r.queryVersion = $2 -// GROUP BY r.queryId -// ) -// SELECT rq.requiredQueryId as queryId, r.value, q.type +// WITH reqQueries as ( +// SELECT q.id as queryId, q.activeVersion, q.type // FROM requiredQueries as rq -// JOIN results as r on r.queryId = rq.requiredQueryId // JOIN queries as q on q.id = rq.requiredQueryId -// JOIN latest_versions lv ON lv.queryId = r.queryId -// WHERE rq.queryId = $1 and r.documentId = $3 -// and $2 >= rq.addedVersion -// and $2 < COALESCE(rq.removedVersion, $2 + 1) -// and r.queryVersion = $2 -// and r.cleanVersion >= $4 and r.textVersion >= $5 -// AND r.cleanVersion = lv.max_clean_version -// AND r.textVersion = lv.max_text_version +// WHERE rq.queryId = $2 +// and $3 >= rq.addedVersion +// and (rq.removedVersion is null or $3 < rq.removedVersion) +// ), +// codeVersions as ( +// SELECT +// d.id as documentId, +// coalesce(ccv.minCleanVersion, 1) as minCleanVersion, +// coalesce(ccv.minTextVersion, 1) as minTextVersion +// FROM documents as d +// LEFT JOIN collectors as c on c.jobId = d.jobId +// LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId +// and c.activeVersion >= ccv.addedVersion +// and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion) +// WHERE d.id = $1 +// LIMIT 1 +// ), +// latestVersions AS ( +// SELECT +// rq.queryId, +// rq.type, +// r.queryVersion, +// r.cleanVersion, +// r.textVersion, +// ROW_NUMBER() OVER ( +// PARTITION BY rq.queryId +// ORDER BY r.cleanVersion DESC, r.textVersion DESC +// ) as rowNumber +// FROM reqQueries as rq +// LEFT JOIN results as r ON rq.queryId = r.queryId +// and r.documentId = $1 +// JOIN codeVersions as ccv on ccv.documentId = r.documentId +// WHERE r.documentId = $1 +// and r.queryVersion = rq.activeVersion +// and r.cleanVersion >= ccv.minCleanVersion +// and r.textVersion >= ccv.minTextVersion +// ) +// SELECT lv.queryId, lv.type, r.value +// FROM latestVersions as lv +// JOIN results as r ON r.queryId = lv.queryId +// and r.documentId = $1 +// and r.queryVersion = lv.queryVersion +// and r.cleanVersion = lv.cleanVersion +// and r.textVersion = lv.textVersion +// and lv.rowNumber = 1 func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) { - rows, err := q.db.Query(ctx, listQueryRequirementValues, - arg.Queryid, - arg.Addedversion, - arg.Documentid, - arg.Cleanversion, - arg.Textversion, - ) + rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version) if err != nil { return nil, err } @@ -122,7 +168,7 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery items := []*ListQueryRequirementValuesRow{} for rows.Next() { var i ListQueryRequirementValuesRow - if err := rows.Scan(&i.Queryid, &i.Value, &i.Type); err != nil { + if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil { return nil, err } items = append(items, &i) @@ -133,67 +179,60 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery return items, nil } -const listUnsyncedQueriesByDocId = `-- name: ListUnsyncedQueriesByDocId :many -WITH RECURSIVE unsyncedQueries AS ( - SELECT dt.queryId, dt.requiredIds - from documents as d - JOIN fullActiveCollectors as c on d.jobId = c.jobId - JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId +const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDocId :many +WITH docs as ( + SELECT id, jobId from documents where id = $1 +), +unsyncedQueries AS ( + SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id + from docs as d + JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId + JOIN fullActiveCollectors as c on c.id = dt.collectorId LEFT JOIN results as r on r.queryId = dt.queryId + and r.documentId = d.id and r.queryVersion = dt.queryVersion and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion - where d.id = $1 and r.value is null - - UNION - - SELECT DISTINCT dt.queryId, dt.requiredIds - FROM unsyncedQueries as u - JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) + where r.value is null ) -SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries -WHERE id in (SELECT queryId FROM unsyncedQueries) +SELECT DISTINCT queryId FROM unsyncedQueries as baseuq + WHERE NOT EXISTS ( + SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds) + ) ` -// ListUnsyncedQueriesByDocId +// ListUnsyncedNoDepsQueriesByDocId // -// WITH RECURSIVE unsyncedQueries AS ( -// SELECT dt.queryId, dt.requiredIds -// from documents as d -// JOIN fullActiveCollectors as c on d.jobId = c.jobId -// JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId +// WITH docs as ( +// SELECT id, jobId from documents where id = $1 +// ), +// unsyncedQueries AS ( +// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id +// from docs as d +// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId +// JOIN fullActiveCollectors as c on c.id = dt.collectorId // LEFT JOIN results as r on r.queryId = dt.queryId +// and r.documentId = d.id // and r.queryVersion = dt.queryVersion // and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion -// where d.id = $1 and r.value is null -// -// UNION -// -// SELECT DISTINCT dt.queryId, dt.requiredIds -// FROM unsyncedQueries as u -// JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) +// where r.value is null // ) -// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries -// WHERE id in (SELECT queryId FROM unsyncedQueries) -func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID) ([]*Fullactivequery, error) { - rows, err := q.db.Query(ctx, listUnsyncedQueriesByDocId, id) +// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq +// WHERE NOT EXISTS ( +// SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds) +// ) +func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1 pgtype.UUID) ([]pgtype.UUID, error) { + rows, err := q.db.Query(ctx, listUnsyncedNoDepsQueriesByDocId, dollar_1) if err != nil { return nil, err } defer rows.Close() - items := []*Fullactivequery{} + items := []pgtype.UUID{} for rows.Next() { - var i Fullactivequery - if err := rows.Scan( - &i.ID, - &i.Type, - &i.Activeversion, - &i.Latestversion, - &i.Config, - &i.Requiredids, - ); err != nil { + var queryid pgtype.UUID + if err := rows.Scan(&queryid); err != nil { return nil, err } - items = append(items, &i) + items = append(items, queryid) } if err := rows.Err(); err != nil { return nil, err @@ -201,8 +240,8 @@ func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID return items, nil } -const setResult = `-- name: SetResult :one -INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id +const setResult = `-- name: SetResult :exec +INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) ` type SetResultParams struct { @@ -216,9 +255,9 @@ type SetResultParams struct { // SetResult // -// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id -func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.UUID, error) { - row := q.db.QueryRow(ctx, setResult, +// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) +func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error { + _, err := q.db.Exec(ctx, setResult, arg.Queryid, arg.Documentid, arg.Value, @@ -226,7 +265,5 @@ func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.U arg.Textversion, arg.Queryversion, ) - var id pgtype.UUID - err := row.Scan(&id) - return id, err + return err } diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index 786ad003..7707b941 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -2,6 +2,7 @@ package repository_test import ( "context" + "log" "os" "path" "queryorchestration/internal/database/repository" @@ -50,7 +51,7 @@ func TestResults(t *testing.T) { textVersion := int32(1) jsonResultValue := "example_value" - jsonResultID, err := queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: jsonQueryID, Documentid: documentID, Value: jsonResultValue, @@ -59,7 +60,6 @@ func TestResults(t *testing.T) { Queryversion: jsonQuery.Activeversion, }) assert.NoError(t, err) - assert.True(t, jsonResultID.Valid) res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ Queryid: jsonQueryID, @@ -69,10 +69,7 @@ func TestResults(t *testing.T) { Textversion: textVersion, }) assert.NoError(t, err) - assert.EqualExportedValues(t, &repository.GetResultValueWithVersionRow{ - ID: jsonResultID, - Value: jsonResultValue, - }, res) + assert.EqualExportedValues(t, jsonResultValue, res.Value) } func TestResultValues(t *testing.T) { @@ -104,79 +101,155 @@ func TestResultValues(t *testing.T) { Hash: "example_hash", }) assert.NoError(t, err) + documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash_two", + }) + assert.NoError(t, err) contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) + + jsonVersion := int32(1) err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ Queryid: jsonQueryID, Requiredqueryid: contextQueryID, - Addedversion: 1, + Addedversion: jsonVersion, }) assert.NoError(t, err) contextQuery, err := queries.GetQuery(ctx, contextQueryID) assert.NoError(t, err) + qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Version: jsonVersion, + }) + assert.NoError(t, err) + assert.Len(t, qResults, 0) + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentTwoID, + Version: jsonVersion, + }) + assert.NoError(t, err) + assert.Len(t, qResults, 0) + result := repository.SetResultParams{ Queryid: contextQueryID, Documentid: documentID, - Value: "context_value", + Value: "context_value_1", Cleanversion: 1, Textversion: 2, Queryversion: contextQuery.Activeversion, } - _, err = queries.SetResult(ctx, &result) + err = queries.SetResult(ctx, &result) assert.NoError(t, err) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Version: jsonVersion, + }) + assert.NoError(t, err) + assert.Len(t, qResults, 1) + assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ + { + Queryid: contextQueryID, + Type: repository.QuerytypeContextFull, + Value: "context_value_1", + }, + }, qResults) + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentTwoID, + Version: jsonVersion, + }) + assert.NoError(t, err) + assert.Len(t, qResults, 0) + + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: contextQueryID, Documentid: documentID, - Value: "context_value", + Value: "context_value_2", Cleanversion: 1, Textversion: 2, Queryversion: contextQuery.Activeversion - 1, }) assert.NoError(t, err) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Version: jsonVersion, + }) + assert.NoError(t, err) + assert.Len(t, qResults, 1) + assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ + { + Queryid: contextQueryID, + Type: repository.QuerytypeContextFull, + Value: "context_value_1", + }, + }, qResults) + + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: contextQueryID, Documentid: documentID, - Value: "context_value", + Value: "context_value_3", Cleanversion: 2, Textversion: 2, Queryversion: contextQuery.Activeversion, }) assert.NoError(t, err) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ - Queryid: jsonQueryID, - Documentid: documentID, - Value: "context_value", - Cleanversion: 1, - Textversion: 2, - Queryversion: contextQuery.Activeversion, - }) - assert.NoError(t, err) - - qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ - Queryid: jsonQueryID, - Documentid: documentID, - Addedversion: 1, - Cleanversion: result.Cleanversion, - Textversion: result.Textversion, + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Version: jsonVersion, }) assert.NoError(t, err) + assert.Len(t, qResults, 1) assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ { Queryid: contextQueryID, Type: repository.QuerytypeContextFull, - Value: result.Value, + Value: "context_value_3", + }, + }, qResults) + + err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Value: "json_value_1", + Cleanversion: 1, + Textversion: 2, + Queryversion: jsonVersion, + }) + assert.NoError(t, err) + + qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Version: jsonVersion, + }) + for _, r := range qResults { + log.Print(r) + } + assert.NoError(t, err) + assert.Len(t, qResults, 1) + assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ + { + Queryid: contextQueryID, + Type: repository.QuerytypeContextFull, + Value: "context_value_3", }, }, qResults) } -func TestUnsyncedQueries(t *testing.T) { +func TestUnsyncedNoDepsQueries(t *testing.T) { if testing.Short() { t.Skip("Skipping long test in short mode") } @@ -204,6 +277,11 @@ func TestUnsyncedQueries(t *testing.T) { Hash: "example_hash", }) assert.NoError(t, err) + documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash_two", + }) + assert.NoError(t, err) contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) assert.NoError(t, err) @@ -224,29 +302,16 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err := queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) - assert.Len(t, qs, 2) - assert.ElementsMatch(t, []*repository.Fullactivequery{ - { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, - Activeversion: 1, - Latestversion: 1, - Config: nil, - Requiredids: []pgtype.UUID{contextQueryID}, - }, - { - ID: contextQueryID, - Type: repository.QuerytypeContextFull, - Activeversion: 1, - Latestversion: 1, - Config: nil, - Requiredids: []pgtype.UUID{}, - }, - }, qs) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) + assert.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: contextQueryID, Documentid: documentID, Value: "context_value", @@ -256,19 +321,14 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*repository.Fullactivequery{ - { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, - Activeversion: 1, - Latestversion: 1, - Config: nil, - Requiredids: []pgtype.UUID{contextQueryID}, - }, - }, qs) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID) + assert.NoError(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ Latestversion: 2, @@ -277,29 +337,12 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) - assert.Len(t, qs, 2) - assert.ElementsMatch(t, []*repository.Fullactivequery{ - { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, - Activeversion: 1, - Latestversion: 1, - Config: nil, - Requiredids: []pgtype.UUID{contextQueryID}, - }, - { - ID: contextQueryID, - Type: repository.QuerytypeContextFull, - Activeversion: 2, - Latestversion: 2, - Config: nil, - Requiredids: []pgtype.UUID{}, - }, - }, qs) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: contextQueryID, Documentid: documentID, Value: "context_value", @@ -308,7 +351,7 @@ func TestUnsyncedQueries(t *testing.T) { Queryversion: 2, }) assert.NoError(t, err) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: jsonQueryID, Documentid: documentID, Value: "context_value", @@ -318,7 +361,7 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) assert.Len(t, qs, 0) @@ -329,21 +372,12 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) assert.Len(t, qs, 1) - assert.ElementsMatch(t, []*repository.Fullactivequery{ - { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, - Activeversion: 2, - Latestversion: 2, - Config: nil, - Requiredids: []pgtype.UUID{contextQueryID}, - }, - }, qs) + assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs) - _, err = queries.SetResult(ctx, &repository.SetResultParams{ + err = queries.SetResult(ctx, &repository.SetResultParams{ Queryid: jsonQueryID, Documentid: documentID, Value: "context_value", @@ -353,7 +387,7 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) assert.Len(t, qs, 0) @@ -364,25 +398,9 @@ func TestUnsyncedQueries(t *testing.T) { }) assert.NoError(t, err) - qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID) assert.NoError(t, err) - assert.Len(t, qs, 2) - assert.ElementsMatch(t, []*repository.Fullactivequery{ - { - ID: jsonQueryID, - Type: repository.QuerytypeJsonExtractor, - Activeversion: 2, - Latestversion: 2, - Config: nil, - Requiredids: []pgtype.UUID{contextQueryID}, - }, - { - ID: contextQueryID, - Type: repository.QuerytypeContextFull, - Activeversion: 3, - Latestversion: 3, - Config: nil, - Requiredids: []pgtype.UUID{}, - }, - }, qs) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs) + } diff --git a/internal/database/repository/text.sql.go b/internal/database/repository/text.sql.go index 6efdb616..2fd47281 100644 --- a/internal/database/repository/text.sql.go +++ b/internal/database/repository/text.sql.go @@ -36,12 +36,12 @@ func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentText } const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one -SELECT dc.documentId, dc.bucket, dc.key +SELECT dc.documentId, dc.bucket, dc.key, dc.version FROM documentTextExtractions AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) ORDER BY d.id DESC LIMIT 1 ` @@ -49,21 +49,27 @@ type GetDocumentTextEntryRow struct { Documentid pgtype.UUID `db:"documentid"` Bucket string `db:"bucket"` Key string `db:"key"` + Version int32 `db:"version"` } // GetDocumentTextEntry // -// SELECT dc.documentId, dc.bucket, dc.key +// SELECT dc.documentId, dc.bucket, dc.key, dc.version // FROM documentTextExtractions AS dc // JOIN documents AS d ON d.id = dc.documentId -// JOIN collectors as c ON d.jobId = c.jobId -// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId -// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion +// LEFT JOIN collectors as c ON d.jobId = c.jobId +// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId +// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) // ORDER BY d.id DESC LIMIT 1 func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) { row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid) var i GetDocumentTextEntryRow - err := row.Scan(&i.Documentid, &i.Bucket, &i.Key) + err := row.Scan( + &i.Documentid, + &i.Bucket, + &i.Key, + &i.Version, + ) return &i, err } @@ -72,9 +78,9 @@ SELECT EXISTS( SELECT 1 FROM documentTextExtractions AS dc JOIN documents AS d ON d.id = dc.documentId - JOIN collectors as c ON d.jobId = c.jobId - JOIN collectorCodeVersions as cv ON c.id = cv.collectorId - WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion + LEFT JOIN collectors as c ON d.jobId = c.jobId + LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId + WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) ) ` @@ -84,9 +90,9 @@ SELECT EXISTS( // SELECT 1 // FROM documentTextExtractions AS dc // JOIN documents AS d ON d.id = dc.documentId -// JOIN collectors as c ON d.jobId = c.jobId -// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId -// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion +// LEFT JOIN collectors as c ON d.jobId = c.jobId +// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId +// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1) // ) func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) { row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid) diff --git a/internal/database/repository/text_test.go b/internal/database/repository/text_test.go index 003e45ab..53c0d086 100644 --- a/internal/database/repository/text_test.go +++ b/internal/database/repository/text_test.go @@ -42,20 +42,13 @@ func TestTextExtraction(t *testing.T) { assert.NoError(t, err) assert.NotEmpty(t, id) - collId, err := queries.CreateCollector(ctx, jobId) - assert.NoError(t, err) - err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{ - Collectorid: collId, - Addedversion: 1, - Mincleanversion: 1, - Mintextversion: 2, - }) - assert.NoError(t, err) - isextract, err := queries.IsDocumentTextExtracted(ctx, id) assert.NoError(t, err) assert.False(t, isextract) + _, err = queries.GetDocumentTextEntry(ctx, id) + assert.Error(t, err) + bucket := "example_bucket" key := "example_key" err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{ @@ -66,18 +59,6 @@ func TestTextExtraction(t *testing.T) { }) assert.NoError(t, err) - isextract, err = queries.IsDocumentTextExtracted(ctx, id) - assert.NoError(t, err) - assert.False(t, isextract) - - err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{ - Collectorid: collId, - Addedversion: 1, - Mincleanversion: 1, - Mintextversion: 1, - }) - assert.NoError(t, err) - isextract, err = queries.IsDocumentTextExtracted(ctx, id) assert.NoError(t, err) assert.True(t, isextract) @@ -88,5 +69,6 @@ func TestTextExtraction(t *testing.T) { Documentid: id, Bucket: bucket, Key: key, + Version: 1, }, clean) } diff --git a/internal/document/clean/clean.go b/internal/document/clean/clean.go index 9c65842b..eb185d15 100644 --- a/internal/document/clean/clean.go +++ b/internal/document/clean/clean.go @@ -2,6 +2,7 @@ package documentclean import ( "context" + "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" @@ -20,6 +21,8 @@ func (s *Service) executeCleanTasks(params *CleanParams) (*document.Location, er } func (s *Service) clean(ctx context.Context, id uuid.UUID) error { + slog.Debug("cleaning document", "id", id.String()) + docId := database.MustToDBUUID(id) entry, err := s.cfg.GetDBQueries().GetDocumentEntry(ctx, docId) diff --git a/internal/document/clean/create.go b/internal/document/clean/create.go index 953bb437..1fb6ab1b 100644 --- a/internal/document/clean/create.go +++ b/internal/document/clean/create.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" ) -func (s *Service) Create(ctx context.Context, id uuid.UUID) error { +func (s *Service) Clean(ctx context.Context, id uuid.UUID) error { isclean, err := s.cfg.GetDBQueries().IsDocumentClean(ctx, database.MustToDBUUID(id)) if err != nil { return err @@ -33,7 +33,7 @@ func (s *Service) Create(ctx context.Context, id uuid.UUID) error { func (s *Service) informClean(ctx context.Context, id uuid.UUID) error { err := s.cfg.SendToQueue(ctx, &queue.SendParams{ QueueURL: s.cfg.GetDocumentTextURL(), - Body: doctextrunner.Create{ + Body: doctextrunner.Body{ ID: id, }, }) diff --git a/internal/document/clean/create_test.go b/internal/document/clean/create_test.go index 63b939ae..2dedde31 100644 --- a/internal/document/clean/create_test.go +++ b/internal/document/clean/create_test.go @@ -74,7 +74,7 @@ func TestCreate(t *testing.T) { ). Return(&sqs.SendMessageOutput{}, nil) - err = svc.Create(ctx, id) + err = svc.Clean(ctx, id) assert.NoError(t, err) } diff --git a/internal/document/init/create.go b/internal/document/init/create.go index fba9bb3b..a709ce9d 100644 --- a/internal/document/init/create.go +++ b/internal/document/init/create.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "log/slog" doccleanrunner "queryorchestration/api/docCleanRunner" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" @@ -90,9 +91,12 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams if err != nil { return err } + slog.Debug("document created", "id", createid.String(), "job", params.JobID.String()) dbid = createid } else { + slog.Debug("document exists", "id", params.ID.String(), "job", params.JobID.String()) + dbid = *params.ID } @@ -123,7 +127,7 @@ func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) er err := s.cfg.SendToQueue(ctx, &queue.SendParams{ QueueURL: s.cfg.GetDocumentCleanURL(), - Body: doccleanrunner.Create{ + Body: doccleanrunner.Body{ ID: id, }, }) diff --git a/internal/document/text/create.go b/internal/document/text/create.go index 93e5bef4..61ad0fb4 100644 --- a/internal/document/text/create.go +++ b/internal/document/text/create.go @@ -2,6 +2,7 @@ package documenttext import ( "context" + "log/slog" querysyncrunner "queryorchestration/api/querySyncRunner" "queryorchestration/internal/database" "queryorchestration/internal/serviceconfig/queue" @@ -9,7 +10,9 @@ import ( "github.com/google/uuid" ) -func (s *Service) Create(ctx context.Context, id uuid.UUID) error { +func (s *Service) Extract(ctx context.Context, id uuid.UUID) error { + slog.Debug("extracting document text", "id", id.String()) + isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, database.MustToDBUUID(id)) if err != nil { return err @@ -33,7 +36,7 @@ func (s *Service) Create(ctx context.Context, id uuid.UUID) error { func (s *Service) informExtraction(ctx context.Context, id uuid.UUID) error { err := s.cfg.SendToQueue(ctx, &queue.SendParams{ QueueURL: s.cfg.GetQuerySyncURL(), - Body: querysyncrunner.Create{ + Body: querysyncrunner.Body{ ID: id, }, }) diff --git a/internal/document/text/create_test.go b/internal/document/text/create_test.go index 05619d17..da6c0abb 100644 --- a/internal/document/text/create_test.go +++ b/internal/document/text/create_test.go @@ -58,8 +58,8 @@ func TestCreate(t *testing.T) { AddRow(false), ) pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key), + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), ) pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key). WillReturnResult(pgxmock.NewResult("", 1)) @@ -74,7 +74,7 @@ func TestCreate(t *testing.T) { ). Return(&sqs.SendMessageOutput{}, nil) - err = svc.Create(ctx, id) + err = svc.Extract(ctx, id) assert.NoError(t, err) } diff --git a/internal/document/text/extract_test.go b/internal/document/text/extract_test.go index ff555f2b..0ddb78f2 100644 --- a/internal/document/text/extract_test.go +++ b/internal/document/text/extract_test.go @@ -37,8 +37,8 @@ func TestExtract(t *testing.T) { } pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows( - pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key), + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key, int32(1)), ) pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key). WillReturnResult(pgxmock.NewResult("", 1)) diff --git a/internal/document/version.go b/internal/document/version.go index 8e3279f1..222ffb1e 100644 --- a/internal/document/version.go +++ b/internal/document/version.go @@ -2,21 +2,8 @@ package document import ( "fmt" - - "github.com/google/uuid" ) -type IsTextExtractedParams struct { - DocumentID uuid.UUID - MinCleanVersion int32 - MinTextVersion int32 -} - -func (s *Service) IsTextExtracted(params *IsTextExtractedParams) error { - // TODO - return nil -} - func (s *Service) GetCleanVersion() int32 { // TODO - actual version return 1 diff --git a/internal/document/version_test.go b/internal/document/version_test.go index e2399ab9..96fad2ee 100644 --- a/internal/document/version_test.go +++ b/internal/document/version_test.go @@ -5,7 +5,6 @@ import ( "queryorchestration/internal/serviceconfig" "testing" - "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -42,14 +41,3 @@ func TestGetTextVersion(t *testing.T) { assert.Equal(t, int32(1), svc.GetTextVersion()) } - -func TestIsTextExtracted(t *testing.T) { - cfg := &serviceconfig.BaseConfig{} - svc := document.New(cfg) - - assert.Nil(t, svc.IsTextExtracted(&document.IsTextExtractedParams{ - DocumentID: uuid.New(), - MinCleanVersion: int32(1), - MinTextVersion: int32(1), - })) -} diff --git a/internal/job/collector/create.go b/internal/job/collector/create.go index f09df4a3..9de50675 100644 --- a/internal/job/collector/create.go +++ b/internal/job/collector/create.go @@ -2,6 +2,7 @@ package collector import ( "context" + "errors" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/validation" @@ -68,6 +69,39 @@ func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*d }, nil } +func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]uuid.UUID) (*map[string]pgtype.UUID, error) { + if ofields == nil || *ofields == nil { + return nil, nil + } else if len(*ofields) == 0 { + return nil, nil + } + + dbm := map[string]pgtype.UUID{} + for name, id := range *ofields { + dbm[name] = database.MustToDBUUID(id) + } + + dbids := []pgtype.UUID{} + for _, id := range dbm { + dbids = append(dbids, id) + } + + dedup := validation.DeduplicateArray(dbids) + + if len(dedup) != len(dbids) { + return nil, errors.New("duplicate output fields") + } + + exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids) + if err != nil { + return nil, err + } else if !exist { + return nil, errors.New("not all required ids are present") + } + + return &dbm, nil +} + func (s *Service) submitCreate(ctx context.Context, params *dbCreateParams) (uuid.UUID, error) { var dbID pgtype.UUID err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error { diff --git a/internal/job/collector/createprivate_test.go b/internal/job/collector/createprivate_test.go index 57ebb10f..e3e8d3a4 100644 --- a/internal/job/collector/createprivate_test.go +++ b/internal/job/collector/createprivate_test.go @@ -106,3 +106,45 @@ func TestSubmitCreate(t *testing.T) { assert.NoError(t, err) assert.Equal(t, id, aid) } + +func TestNormalizeFieldsToDB(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + fields := map[string]uuid.UUID{ + "example_key": uuid.New(), + } + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + dbparams, err := svc.normalizeFieldsToDB(ctx, &fields) + assert.NoError(t, err) + assert.EqualExportedValues(t, &map[string]pgtype.UUID{ + "example_key": database.MustToDBUUID(fields["example_key"]), + }, dbparams) + + fields["second_key"] = fields["example_key"] + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + _, err = svc.normalizeFieldsToDB(ctx, &fields) + assert.Error(t, err) +} diff --git a/internal/job/collector/update.go b/internal/job/collector/update.go index 8379eec9..7edff377 100644 --- a/internal/job/collector/update.go +++ b/internal/job/collector/update.go @@ -3,6 +3,7 @@ package collector import ( "context" "errors" + "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/validation" @@ -52,7 +53,7 @@ func (s *Service) getUpdateParams(ctx context.Context, current *Collector, param return nil, err } - fs, err := s.normalizeFieldsToDB(ctx, params.Fields) + fields, err := s.normalizeUpdateFieldsToDB(ctx, current.Fields, params.Fields) if err != nil { return nil, err } @@ -62,19 +63,43 @@ func (s *Service) getUpdateParams(ctx context.Context, current *Collector, param return nil, err } - if params.ActiveVersion == nil && - params.MinCleanVersion == nil && - params.MinTextVersion == nil && - fs == nil { + minClean := params.MinCleanVersion + if minClean != nil { + err := s.svc.Document.IsValidCleanVersion(*minClean) + if err != nil { + return nil, err + } + } + + minText := params.MinTextVersion + if minText != nil { + err := s.svc.Document.IsValidTextVersion(*minText) + if err != nil { + return nil, err + } + } + + activeVersionName, err := validation.GetFieldName(params, params.ActiveVersion) + if err != nil { + return nil, err + } + fieldsName, err := validation.GetFieldName(params, params.Fields) + if err != nil { + return nil, err + } + + if (params.ActiveVersion == nil || *params.ActiveVersion == current.LatestVersion+1) && + validation.AreAllPointersNilExcept(params, activeVersionName, fieldsName) && + fields == nil { return nil, errors.New("no changes") } return &dbUpdateParams{ JobID: database.MustToDBUUID(params.JobID), ActiveVersion: params.ActiveVersion, - MinCleanVersion: params.MinCleanVersion, - MinTextVersion: params.MinTextVersion, - Fields: fs, + MinCleanVersion: minClean, + MinTextVersion: minText, + Fields: fields, }, nil } @@ -132,7 +157,7 @@ func (s *Service) normalizeActiveVersion(current *Collector, params *UpdateParam return nil } -func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]uuid.UUID) (*map[string]pgtype.UUID, error) { +func (s *Service) normalizeUpdateFieldsToDB(ctx context.Context, current map[string]uuid.UUID, ofields *map[string]uuid.UUID) (*map[string]pgtype.UUID, error) { if ofields == nil || *ofields == nil { return nil, nil } else if len(*ofields) == 0 { @@ -144,25 +169,14 @@ func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]u dbm[name] = database.MustToDBUUID(id) } - dbids := []pgtype.UUID{} - for _, id := range dbm { - dbids = append(dbids, id) + removeIDs := getRemoveFields(current, &dbm) + addIDs := getAddFields(current, &dbm) + + if len(addIDs) == 0 && len(removeIDs) == 0 { + return nil, nil } - dedup := validation.DeduplicateArray(dbids) - - if len(dedup) != len(dbids) { - return nil, errors.New("duplicate output fields") - } - - exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids) - if err != nil { - return nil, err - } else if !exist { - return nil, errors.New("not all required ids are present") - } - - return &dbm, nil + return s.normalizeFieldsToDB(ctx, ofields) } func (s *Service) submitUpdate(ctx context.Context, current *Collector, params *dbUpdateParams) error { @@ -229,6 +243,8 @@ func (s *Service) submitUpdate(ctx context.Context, current *Collector, params * return err } + slog.Debug("job collector updated", "update", *params) + return nil }) if err != nil { @@ -266,12 +282,9 @@ func getAddFields(current map[string]uuid.UUID, update *map[string]pgtype.UUID) return diff } - for ckey, cid := range current { - for ukey, uid := range *update { - if ckey == ukey && cid != database.MustToUUID(uid) || - current[ukey] == uuid.Nil { - diff[ukey] = uid - } + for ukey, uid := range *update { + if current[ukey] != database.MustToUUID(uid) { + diff[ukey] = uid } } diff --git a/internal/job/collector/updateprivate_test.go b/internal/job/collector/updateprivate_test.go index 0394adbd..f994bfaa 100644 --- a/internal/job/collector/updateprivate_test.go +++ b/internal/job/collector/updateprivate_test.go @@ -16,65 +16,145 @@ import ( ) func TestGetUpdateParams(t *testing.T) { - ctx := context.Background() + t.Run("all params", func(t *testing.T) { + ctx := context.Background() - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) - svc := Service{ - cfg: cfg, - svc: &Services{}, - } + svc := Service{ + cfg: cfg, + svc: &Services{}, + } - minCleanV := int32(1) - minTextV := int32(1) - aV := int32(3) - current := Collector{ - ActiveVersion: 1, - LatestVersion: 10, - } - params := UpdateParams{ - JobID: uuid.New(), - ActiveVersion: &aV, - MinCleanVersion: &minCleanV, - MinTextVersion: &minTextV, - Fields: &map[string]uuid.UUID{ - "example_key": uuid.New(), - }, - } + minCleanV := int32(1) + minTextV := int32(1) + aV := int32(3) + current := Collector{ + ActiveVersion: 1, + LatestVersion: 10, + } + params := UpdateParams{ + JobID: uuid.New(), + ActiveVersion: &aV, + MinCleanVersion: &minCleanV, + MinTextVersion: &minTextV, + Fields: &map[string]uuid.UUID{ + "example_key": uuid.New(), + }, + } - pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{(*params.Fields)["example_key"]})).WillReturnRows( - pgxmock.NewRows([]string{"all_exist"}). - AddRow(true), - ) + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{(*params.Fields)["example_key"]})).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) - dbparams, err := svc.getUpdateParams(ctx, ¤t, ¶ms) - assert.NoError(t, err) - assert.EqualExportedValues(t, &dbUpdateParams{ - JobID: database.MustToDBUUID(params.JobID), - ActiveVersion: &aV, - MinCleanVersion: &minCleanV, - MinTextVersion: &minTextV, - Fields: &map[string]pgtype.UUID{ - "example_key": database.MustToDBUUID((*params.Fields)["example_key"]), - }, - }, dbparams) + dbparams, err := svc.getUpdateParams(ctx, ¤t, ¶ms) + assert.NoError(t, err) + assert.EqualExportedValues(t, &dbUpdateParams{ + JobID: database.MustToDBUUID(params.JobID), + ActiveVersion: &aV, + MinCleanVersion: &minCleanV, + MinTextVersion: &minTextV, + Fields: &map[string]pgtype.UUID{ + "example_key": database.MustToDBUUID((*params.Fields)["example_key"]), + }, + }, dbparams) - (*params.Fields)["second_key"] = (*params.Fields)["example_key"] - assert.Len(t, *params.Fields, 2) - _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) - assert.Error(t, err) + (*params.Fields)["second_key"] = (*params.Fields)["example_key"] + assert.Len(t, *params.Fields, 2) + _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) + assert.Error(t, err) + }) - params = UpdateParams{ - JobID: current.JobID, - } - _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) - assert.Error(t, err) + t.Run("no params", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := Collector{} + params := UpdateParams{ + JobID: current.JobID, + } + _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) + assert.EqualError(t, err, "no changes") + }) + + t.Run("ONLY active version to next latest", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := Collector{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(5) + params := UpdateParams{ + JobID: current.JobID, + ActiveVersion: &version, + } + _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) + assert.EqualError(t, err, "no changes") + }) + + t.Run("no fields change", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := Collector{ + Fields: map[string]uuid.UUID{ + "example": uuid.New(), + }, + } + params := UpdateParams{ + JobID: current.JobID, + Fields: &map[string]uuid.UUID{ + "example": current.Fields["example"], + }, + } + _, err = svc.getUpdateParams(ctx, ¤t, ¶ms) + assert.EqualError(t, err, "no changes") + }) } func TestSubmitUpdate(t *testing.T) { @@ -144,83 +224,75 @@ func TestSubmitUpdate(t *testing.T) { assert.NoError(t, err) } -func TestNormalizeFieldsToDB(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := Service{ - cfg: cfg, - svc: &Services{}, - } - - fields := map[string]uuid.UUID{ - "example_key": uuid.New(), - } - - pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( - pgxmock.NewRows([]string{"all_exist"}). - AddRow(true), - ) - - dbparams, err := svc.normalizeFieldsToDB(ctx, &fields) - assert.NoError(t, err) - assert.EqualExportedValues(t, &map[string]pgtype.UUID{ - "example_key": database.MustToDBUUID(fields["example_key"]), - }, dbparams) - - fields["second_key"] = fields["example_key"] - - pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( - pgxmock.NewRows([]string{"all_exist"}). - AddRow(true), - ) - - _, err = svc.normalizeFieldsToDB(ctx, &fields) - assert.Error(t, err) -} - func TestNormalizeActiveVersion(t *testing.T) { svc := Service{} - err := svc.normalizeActiveVersion(nil, nil) - assert.Error(t, err) + t.Run("all nil", func(t *testing.T) { + err := svc.normalizeActiveVersion(nil, nil) + assert.Error(t, err) + }) - current := Collector{ - ActiveVersion: 1, - LatestVersion: 4, - } + t.Run("nil update", func(t *testing.T) { + current := Collector{ + ActiveVersion: 1, + LatestVersion: 4, + } - err = svc.normalizeActiveVersion(¤t, nil) - assert.NoError(t, err) + err := svc.normalizeActiveVersion(¤t, nil) + assert.NoError(t, err) + }) - update := UpdateParams{} + t.Run("no update", func(t *testing.T) { + current := Collector{ + ActiveVersion: 1, + LatestVersion: 4, + } + update := UpdateParams{} + err := svc.normalizeActiveVersion(¤t, &update) + assert.NoError(t, err) + assert.Nil(t, update.ActiveVersion) + }) - err = svc.normalizeActiveVersion(¤t, &update) - assert.NoError(t, err) - assert.Nil(t, update.ActiveVersion) + t.Run("same version", func(t *testing.T) { + version := int32(2) + current := Collector{ + ActiveVersion: version, + LatestVersion: 4, + } + update := UpdateParams{ + ActiveVersion: &version, + } + err := svc.normalizeActiveVersion(¤t, &update) + assert.NoError(t, err) + assert.Nil(t, update.ActiveVersion) + }) - update.ActiveVersion = ¤t.ActiveVersion - err = svc.normalizeActiveVersion(¤t, &update) - assert.NoError(t, err) - assert.Nil(t, update.ActiveVersion) + t.Run("latest version plus 1", func(t *testing.T) { + current := Collector{ + ActiveVersion: 1, + LatestVersion: 4, + } + version := int32(5) + update := UpdateParams{ + ActiveVersion: &version, + } + err := svc.normalizeActiveVersion(¤t, &update) + assert.NoError(t, err) + assert.Equal(t, version, *update.ActiveVersion) + }) - av := current.LatestVersion + 2 - update.ActiveVersion = &av - err = svc.normalizeActiveVersion(¤t, &update) - assert.Error(t, err) - - av = current.ActiveVersion + 1 - update.ActiveVersion = &av - err = svc.normalizeActiveVersion(¤t, &update) - assert.NoError(t, err) - assert.Equal(t, av, *update.ActiveVersion) + t.Run("latest version plus 2", func(t *testing.T) { + current := Collector{ + ActiveVersion: 1, + LatestVersion: 4, + } + version := int32(6) + update := UpdateParams{ + ActiveVersion: &version, + } + err := svc.normalizeActiveVersion(¤t, &update) + assert.Error(t, err) + }) } func TestNormalizeCodeVersions(t *testing.T) { @@ -356,4 +428,99 @@ func TestGetAddFields(t *testing.T) { assert.Equal(t, map[string]pgtype.UUID{ "b": (*update)["b"], }, add) + + current = map[string]uuid.UUID{} + add = getAddFields(current, update) + assert.Len(t, add, 2) + assert.Equal(t, map[string]pgtype.UUID{ + "b": (*update)["b"], + "a": (*update)["a"], + }, add) +} + +func TestNormalizeUpdateFieldsToDB(t *testing.T) { + t.Run("valid key", func(t *testing.T) { + + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := map[string]uuid.UUID{} + fields := map[string]uuid.UUID{ + "example_key": uuid.New(), + } + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + dbparams, err := svc.normalizeUpdateFieldsToDB(ctx, current, &fields) + assert.NoError(t, err) + assert.EqualExportedValues(t, &map[string]pgtype.UUID{ + "example_key": database.MustToDBUUID(fields["example_key"]), + }, dbparams) + }) + + t.Run("duplicate values", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := map[string]uuid.UUID{} + fields := map[string]uuid.UUID{ + "example_key": uuid.New(), + } + fields["second_key"] = fields["example_key"] + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray([]uuid.UUID{fields["example_key"]})).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + _, err = svc.normalizeUpdateFieldsToDB(ctx, current, &fields) + assert.Error(t, err) + }) + t.Run("no changes", func(t *testing.T) { + ctx := context.Background() + cfg := &serviceconfig.BaseConfig{} + + svc := Service{ + cfg: cfg, + svc: &Services{}, + } + + current := map[string]uuid.UUID{ + "example_key": uuid.New(), + } + fields := map[string]uuid.UUID{ + "example_key": current["example_key"], + } + + val, err := svc.normalizeUpdateFieldsToDB(ctx, current, &fields) + assert.NoError(t, err) + assert.Nil(t, val) + }) } diff --git a/internal/job/update.go b/internal/job/update.go index 3a89ee61..2192b514 100644 --- a/internal/job/update.go +++ b/internal/job/update.go @@ -3,8 +3,10 @@ package job import ( "context" "errors" + "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/validation" "github.com/google/uuid" ) @@ -43,7 +45,7 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Job, update *Upd return err } - if update.CanSync == nil { + if validation.AreAllPointersNilExcept(update) { return errors.New("no changes") } @@ -62,6 +64,8 @@ func (s *Service) submitUpdate(ctx context.Context, update *Update) error { } } + slog.Debug("job updated", "update", *update) + return nil }) diff --git a/internal/query/create_test.go b/internal/query/create_test.go index a4c1e2d8..56b1a813 100644 --- a/internal/query/create_test.go +++ b/internal/query/create_test.go @@ -26,7 +26,7 @@ func TestCreate(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" q := query.Query{ @@ -78,11 +78,11 @@ func TestCreateMinimal(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) q := query.Query{ ID: uuid.New(), - Type: resultprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeContextFull, } create := &resultprocessor.Create{ Type: q.Type, @@ -113,10 +113,12 @@ func TestCreateRollback(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) + ccfg := "{}" create := &resultprocessor.Create{ - Type: resultprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, + Config: &ccfg, } dbType, err := resultprocessor.ToDBQueryType(create.Type) diff --git a/internal/query/createprivate_test.go b/internal/query/createprivate_test.go index 488ae522..eb6cca1e 100644 --- a/internal/query/createprivate_test.go +++ b/internal/query/createprivate_test.go @@ -22,7 +22,7 @@ func TestGetCreator(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) queryType := resultprocessor.Type(resultprocessor.TypeContextFull) creator, err := svc.getCreator(queryType) @@ -84,7 +84,7 @@ func TestSubmitCreate(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) config := "{\"path\":\"example_path\"}" q := Query{ @@ -132,7 +132,7 @@ func TestSubmitCreateNoReqsOrConfig(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) q := Query{ ID: uuid.New(), @@ -167,10 +167,11 @@ func TestNormalizeCreate(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) create := &resultprocessor.Create{ - Type: resultprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeContextFull, + Config: nil, RequiredQueryIDs: &[]uuid.UUID{}, } @@ -184,7 +185,7 @@ func TestNormalizeCreate(t *testing.T) { err = svc.normalizeCreate(ctx, create) assert.NoError(t, err) assert.EqualExportedValues(t, resultprocessor.Create{ - Type: resultprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeContextFull, Config: nil, RequiredQueryIDs: nil, }, *create) diff --git a/internal/query/get_test.go b/internal/query/get_test.go index 49e3e53d..07015f71 100644 --- a/internal/query/get_test.go +++ b/internal/query/get_test.go @@ -24,7 +24,7 @@ func TestGet(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" query := query.Query{ @@ -61,7 +61,7 @@ func TestGetWithVersion(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" query := query.Query{ diff --git a/internal/query/list_test.go b/internal/query/list_test.go index b2cb434e..43858340 100644 --- a/internal/query/list_test.go +++ b/internal/query/list_test.go @@ -25,7 +25,7 @@ func TestList(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" q := &query.Query{ @@ -62,7 +62,7 @@ func TestListById(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" q := &query.Query{ diff --git a/internal/query/normalize.go b/internal/query/normalize.go index 690db057..12771d9f 100644 --- a/internal/query/normalize.go +++ b/internal/query/normalize.go @@ -86,9 +86,7 @@ func (s *Service) NormalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) e func (s *Service) normalizeActiveVersion(current *Query, entity *resultprocessor.Update) error { if current == nil { return errors.New("current query required") - } - - if entity == nil { + } else if entity == nil { return nil } diff --git a/internal/query/normalize_test.go b/internal/query/normalize_test.go index 2a6f7081..cbd88df5 100644 --- a/internal/query/normalize_test.go +++ b/internal/query/normalize_test.go @@ -142,42 +142,103 @@ func TestNormalizeQueryIDs(t *testing.T) { func TestNormalizeActiveVersion(t *testing.T) { s := Service{} - err := s.normalizeActiveVersion(nil, nil) - assert.Error(t, err) + t.Run("all nil", func(t *testing.T) { + err := s.normalizeActiveVersion(nil, nil) + assert.Error(t, err) + }) - current := Query{ - ActiveVersion: 2, - LatestVersion: 4, - } - entity := resultprocessor.Update{} + t.Run("nil current", func(t *testing.T) { + entity := resultprocessor.Update{} - err = s.normalizeActiveVersion(nil, &entity) - assert.Error(t, err) + err := s.normalizeActiveVersion(nil, &entity) + assert.Error(t, err) + }) - err = s.normalizeActiveVersion(¤t, nil) - assert.NoError(t, err) + t.Run("nil update", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + err := s.normalizeActiveVersion(¤t, nil) + assert.NoError(t, err) + }) - err = s.normalizeActiveVersion(¤t, &entity) - assert.NoError(t, err) + t.Run("no update", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + entity := resultprocessor.Update{} + err := s.normalizeActiveVersion(¤t, &entity) + assert.NoError(t, err) + assert.Nil(t, entity.ActiveVersion) + }) - entity.ActiveVersion = ¤t.ActiveVersion - err = s.normalizeActiveVersion(¤t, &entity) - assert.NoError(t, err) - assert.Nil(t, entity.ActiveVersion) + t.Run("equal update to current", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(2) + entity := resultprocessor.Update{ + ActiveVersion: &version, + } + err := s.normalizeActiveVersion(¤t, &entity) + assert.NoError(t, err) + assert.Nil(t, entity.ActiveVersion) + }) - aV := current.ActiveVersion + 1 - entity.ActiveVersion = &aV - err = s.normalizeActiveVersion(¤t, &entity) - assert.NoError(t, err) - assert.Equal(t, aV, *entity.ActiveVersion) + t.Run("valid update", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(3) + entity := resultprocessor.Update{ + ActiveVersion: &version, + } + err := s.normalizeActiveVersion(¤t, &entity) + assert.NoError(t, err) + assert.Equal(t, version, *entity.ActiveVersion) + }) - aV = current.LatestVersion + 2 - entity.ActiveVersion = &aV - err = s.normalizeActiveVersion(¤t, &entity) - assert.Error(t, err) + t.Run("update greater than latest by 1", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(5) + entity := resultprocessor.Update{ + ActiveVersion: &version, + } + err := s.normalizeActiveVersion(¤t, &entity) + assert.NoError(t, err) + assert.Equal(t, version, *entity.ActiveVersion) + }) - aV = 0 - entity.ActiveVersion = &aV - err = s.normalizeActiveVersion(¤t, &entity) - assert.Error(t, err) + t.Run("update greater than latest by 2", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(6) + entity := resultprocessor.Update{ + ActiveVersion: &version, + } + err := s.normalizeActiveVersion(¤t, &entity) + assert.Error(t, err) + }) + + t.Run("update to 0", func(t *testing.T) { + current := Query{ + ActiveVersion: 2, + LatestVersion: 4, + } + version := int32(0) + entity := resultprocessor.Update{ + ActiveVersion: &version, + } + err := s.normalizeActiveVersion(¤t, &entity) + assert.Error(t, err) + }) } diff --git a/internal/query/result/get_test.go b/internal/query/result/get_test.go index a32b34f7..101080a6 100644 --- a/internal/query/result/get_test.go +++ b/internal/query/result/get_test.go @@ -39,7 +39,7 @@ func TestGetValueWithVersion(t *testing.T) { cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg) + svc := New(cfg, &Services{}) params := &GetValueWithVersionParams{ Type: resultprocessor.TypeJsonExtractor, diff --git a/internal/query/result/list.go b/internal/query/result/list.go deleted file mode 100644 index 064acec9..00000000 --- a/internal/query/result/list.go +++ /dev/null @@ -1,52 +0,0 @@ -package result - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - resultprocessor "queryorchestration/internal/query/result/processor" - - "github.com/google/uuid" -) - -type ListQueryRequirementValuesParams struct { - QueryID uuid.UUID - DocumentID uuid.UUID - QueryVersion int32 - MinCleanVersion int32 - MinTextVersion int32 -} - -func (s *Service) ListQueryRequirementValues(ctx context.Context, params *ListQueryRequirementValuesParams) (*[]resultprocessor.Value, error) { - qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ - Queryid: database.MustToDBUUID(params.QueryID), - Documentid: database.MustToDBUUID(params.DocumentID), - Addedversion: params.QueryVersion, - Cleanversion: params.MinCleanVersion, - Textversion: params.MinTextVersion, - }) - if err != nil { - return nil, err - } - - return parseQueryRequirementValueArray(qResults) -} - -func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesRow) (*[]resultprocessor.Value, error) { - values := make([]resultprocessor.Value, len(v)) - for index, r := range v { - qType, err := resultprocessor.ParseDBType(r.Type) - if err != nil { - return nil, err - } - - cleanValue, err := getValueByType(qType, r.Value) - if err != nil { - return nil, err - } - - values[index] = cleanValue - } - - return &values, nil -} diff --git a/internal/query/result/list_test.go b/internal/query/result/list_test.go deleted file mode 100644 index 10408b72..00000000 --- a/internal/query/result/list_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package result - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - resultprocessor "queryorchestration/internal/query/result/processor" - jsonextractor "queryorchestration/internal/query/types/jsonExtractor" - "queryorchestration/internal/serviceconfig" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestListQueryRequirementValues(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := Service{ - cfg: cfg, - } - - params := &ListQueryRequirementValuesParams{ - QueryID: uuid.New(), - QueryVersion: 1, - DocumentID: uuid.New(), - MinCleanVersion: 2, - MinTextVersion: 3, - } - - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(pgtype.UUID{}, "example_value", repository.QuerytypeJsonExtractor), - ) - - val, err := svc.ListQueryRequirementValues(ctx, params) - assert.NoError(t, err) - assert.ElementsMatch(t, []resultprocessor.Value{ - jsonextractor.NewResult("example_value"), - }, *val) -} - -func TestParseQueryRequirementValueArray(t *testing.T) { - in := []*repository.ListQueryRequirementValuesRow{ - { - Queryid: database.MustToDBUUID(uuid.New()), - Value: "example_value", - Type: repository.QuerytypeJsonExtractor, - }, - } - - out, err := parseQueryRequirementValueArray(in) - assert.NoError(t, err) - assert.ElementsMatch(t, []resultprocessor.Value{ - jsonextractor.NewResult("example_value"), - }, *out) -} diff --git a/internal/query/result/process.go b/internal/query/result/process.go index ba931a62..296155b5 100644 --- a/internal/query/result/process.go +++ b/internal/query/result/process.go @@ -2,8 +2,12 @@ package result import ( "context" + "database/sql" "errors" "fmt" + "log/slog" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" @@ -12,50 +16,81 @@ import ( ) type Process struct { - DocumentID uuid.UUID - MinCleanVersion int32 - MinTextVersion int32 - Query *resultprocessor.Query + DocumentID uuid.UUID + QueryID uuid.UUID + QueryVersion int32 } func (s *Service) Process(ctx context.Context, p *Process) (resultprocessor.Value, error) { - values, err := s.listRequiredValues(ctx, p) + slog.Debug("processing query", "query_id", p.QueryID.String(), "version", p.QueryVersion, "document_id", p.DocumentID.String()) + + query, err := s.svc.Query.GetWithVersion(ctx, p.QueryID, p.QueryVersion) if err != nil { return nil, err } - processor, err := s.getProcessor(p.Query.Type) + processQuery := &resultprocessor.Query{ + ID: query.ID, + Version: p.QueryVersion, + Type: query.Type, + RequiredQueryIDs: query.RequiredQueryIDs, + Config: query.Config, + } + + values, err := s.listRequiredValues(ctx, p, processQuery) if err != nil { return nil, err } - val, err := processor.Process(ctx, p.Query, values) + processor, err := s.getProcessor(query.Type) if err != nil { return nil, err } - return getValueByType(p.Query.Type, val) + val, err := processor.Process(ctx, processQuery, values) + if err != nil { + return nil, err + } + + return getValueByType(query.Type, val) } -func (s *Service) listRequiredValues(ctx context.Context, p *Process) (*[]resultprocessor.Value, error) { - if p == nil || p.Query == nil || p.Query.RequiredQueryIDs == nil || len(*p.Query.RequiredQueryIDs) == 0 { +func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *resultprocessor.Query) (*[]resultprocessor.Value, error) { + if p == nil || query == nil || query.RequiredQueryIDs == nil || len(*query.RequiredQueryIDs) == 0 { return nil, nil } - values, err := s.ListQueryRequirementValues(ctx, &ListQueryRequirementValuesParams{ - QueryID: p.Query.ID, - QueryVersion: p.Query.Version, - DocumentID: p.DocumentID, - MinCleanVersion: p.MinCleanVersion, - MinTextVersion: p.MinTextVersion, + qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: database.MustToDBUUID(p.QueryID), + Documentid: database.MustToDBUUID(p.DocumentID), + Version: p.QueryVersion, }) - if err != nil { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err - } else if values == nil || len(*values) != len(*p.Query.RequiredQueryIDs) { - return nil, errors.New("document not synced") + } else if len(qResults) != len(*query.RequiredQueryIDs) { + return nil, fmt.Errorf("required results not found for the query (%s) in document (%s)", p.QueryID, p.DocumentID) } - return values, nil + return parseQueryRequirementValueArray(qResults) +} + +func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesRow) (*[]resultprocessor.Value, error) { + values := make([]resultprocessor.Value, len(v)) + for index, r := range v { + qType, err := resultprocessor.ParseDBType(r.Type) + if err != nil { + return nil, err + } + + cleanValue, err := getValueByType(qType, r.Value) + if err != nil { + return nil, err + } + + values[index] = cleanValue + } + + return &values, nil } func (s *Service) getProcessor(queryType resultprocessor.Type) (resultprocessor.Processor, error) { diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go index ae78a676..c7468b81 100644 --- a/internal/query/result/process_test.go +++ b/internal/query/result/process_test.go @@ -4,6 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/query" resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "queryorchestration/internal/serviceconfig" @@ -28,29 +29,34 @@ func TestProcess(t *testing.T) { svc := Service{ cfg: cfg, - } - - qcfg := "{\"path\":\"examplekey\"}" - params := Process{ - DocumentID: uuid.New(), - MinCleanVersion: 1, - MinTextVersion: 3, - Query: &resultprocessor.Query{ - ID: uuid.New(), - Version: 2, - Config: &qcfg, - RequiredQueryIDs: &[]uuid.UUID{ - uuid.New(), - }, + svc: &Services{ + Query: query.New(cfg), }, } - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + qcfg := "{\"path\":\"examplekey\"}" + query := &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + Config: &qcfg, + } + params := Process{ + DocumentID: uuid.New(), + QueryID: query.ID, + QueryVersion: query.Version, + } + + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "{\"examplekey\":\"example_value\"}", repository.QuerytypeContextFull), + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( pgxmock.NewRows([]string{"id", "config"}). AddRow(pgtype.UUID{}, []byte(qcfg)), ) @@ -76,48 +82,40 @@ func TestListRequiredValue(t *testing.T) { cfg: cfg, } - pr, err := svc.listRequiredValues(ctx, nil) + pr, err := svc.listRequiredValues(ctx, nil, nil) assert.NoError(t, err) assert.Nil(t, pr) - pr, err = svc.listRequiredValues(ctx, &Process{}) + pr, err = svc.listRequiredValues(ctx, &Process{}, &resultprocessor.Query{}) assert.NoError(t, err) assert.Nil(t, pr) - pr, err = svc.listRequiredValues(ctx, &Process{ - Query: &resultprocessor.Query{}, - }) - assert.NoError(t, err) - assert.Nil(t, pr) - - pr, err = svc.listRequiredValues(ctx, &Process{ - Query: &resultprocessor.Query{ + pr, err = svc.listRequiredValues(ctx, &Process{}, + &resultprocessor.Query{ RequiredQueryIDs: &[]uuid.UUID{}, }, - }) + ) assert.NoError(t, err) assert.Nil(t, pr) + query := &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + } params := Process{ - DocumentID: uuid.New(), - MinCleanVersion: 1, - MinTextVersion: 3, - Query: &resultprocessor.Query{ - ID: uuid.New(), - Version: 2, - RequiredQueryIDs: &[]uuid.UUID{ - uuid.New(), - }, - }, + DocumentID: uuid.New(), + QueryID: query.ID, + QueryVersion: query.Version, } - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "example_value", repository.QuerytypeJsonExtractor), + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, "example_value"), ) - pr, err = svc.listRequiredValues(ctx, ¶ms) + pr, err = svc.listRequiredValues(ctx, ¶ms, query) assert.NoError(t, err) assert.ElementsMatch(t, []resultprocessor.Value{ jsonextractor.NewResult("example_value"), @@ -138,3 +136,19 @@ func TestGetProcessor(t *testing.T) { _, err = svc.getProcessor(resultprocessor.Type(-1)) assert.Error(t, err) } + +func TestParseQueryRequirementValueArray(t *testing.T) { + in := []*repository.ListQueryRequirementValuesRow{ + { + Queryid: database.MustToDBUUID(uuid.New()), + Value: "example_value", + Type: repository.QuerytypeJsonExtractor, + }, + } + + out, err := parseQueryRequirementValueArray(in) + assert.NoError(t, err) + assert.ElementsMatch(t, []resultprocessor.Value{ + jsonextractor.NewResult("example_value"), + }, *out) +} diff --git a/internal/query/result/service.go b/internal/query/result/service.go index 575da710..2d9b32b7 100644 --- a/internal/query/result/service.go +++ b/internal/query/result/service.go @@ -1,6 +1,7 @@ package result import ( + "queryorchestration/internal/query" "queryorchestration/internal/serviceconfig" "github.com/google/uuid" @@ -12,12 +13,18 @@ type Result struct { QueryVersion int32 } -type Service struct { - cfg serviceconfig.ConfigProvider +type Services struct { + Query *query.Service } -func New(cfg serviceconfig.ConfigProvider) *Service { +type Service struct { + cfg serviceconfig.ConfigProvider + svc *Services +} + +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { return &Service{ cfg, + svc, } } diff --git a/internal/query/result/service_test.go b/internal/query/result/service_test.go index 1189f1d4..82ca66ed 100644 --- a/internal/query/result/service_test.go +++ b/internal/query/result/service_test.go @@ -18,6 +18,6 @@ func TestService(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := result.New(cfg) + svc := result.New(cfg, &result.Services{}) assert.NotNil(t, svc) } diff --git a/internal/query/result/set.go b/internal/query/result/set.go deleted file mode 100644 index 34c91927..00000000 --- a/internal/query/result/set.go +++ /dev/null @@ -1,45 +0,0 @@ -package result - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - resultprocessor "queryorchestration/internal/query/result/processor" - - "github.com/google/uuid" -) - -type Set struct { - DocumentID uuid.UUID - CleanVersion int32 - TextVersion int32 - Query *resultprocessor.Query -} - -func (s *Service) Set(ctx context.Context, params *Set) (uuid.UUID, error) { - value, err := s.Process(ctx, &Process{ - DocumentID: params.DocumentID, - MinCleanVersion: params.CleanVersion, - MinTextVersion: params.TextVersion, - Query: params.Query, - }) - if err != nil { - return uuid.Nil, err - } - - dbId, err := s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{ - Queryid: database.MustToDBUUID(params.Query.ID), - Documentid: database.MustToDBUUID(params.DocumentID), - Value: value.GetStoreValue(), - Cleanversion: params.CleanVersion, - Textversion: params.TextVersion, - Queryversion: params.Query.Version, - }) - if err != nil { - return uuid.Nil, err - } - - id := database.MustToUUID(dbId) - - return id, nil -} diff --git a/internal/query/result/set/service.go b/internal/query/result/set/service.go new file mode 100644 index 00000000..a1c8a595 --- /dev/null +++ b/internal/query/result/set/service.go @@ -0,0 +1,30 @@ +package resultset + +import ( + "queryorchestration/internal/query" + "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" + queryc "queryorchestration/internal/serviceconfig/queue/query" +) + +type ConfigProvider interface { + serviceconfig.ConfigProvider + queryc.ConfigProvider +} + +type Services struct { + Result *result.Service + Query *query.Service +} + +type Service struct { + cfg ConfigProvider + svc *Services +} + +func New(cfg ConfigProvider, svc *Services) *Service { + return &Service{ + cfg, + svc, + } +} diff --git a/internal/query/result/set/service_test.go b/internal/query/result/set/service_test.go new file mode 100644 index 00000000..7b03bda5 --- /dev/null +++ b/internal/query/result/set/service_test.go @@ -0,0 +1,21 @@ +package resultset_test + +import ( + resultset "queryorchestration/internal/query/result/set" + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/queue/query" + "testing" + + "github.com/stretchr/testify/assert" +) + +type QuerySyncConfig struct { + serviceconfig.BaseConfig + query.QueryConfig +} + +func TestService(t *testing.T) { + cfg := &QuerySyncConfig{} + svc := resultset.New(cfg, &resultset.Services{}) + assert.NotNil(t, svc) +} diff --git a/internal/query/result/set/set.go b/internal/query/result/set/set.go new file mode 100644 index 00000000..747f8746 --- /dev/null +++ b/internal/query/result/set/set.go @@ -0,0 +1,94 @@ +package resultset + +import ( + "context" + "log/slog" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig/queue" + + "github.com/google/uuid" +) + +type Set struct { + DocumentID uuid.UUID `json:"document_id" validate:"required,uuid"` + QueryID uuid.UUID `json:"query_id" validate:"required,uuid"` +} + +func (s *Service) Set(ctx context.Context, params *Set) error { + dbid := database.MustToDBUUID(params.DocumentID) + cleanVersion, err := s.cfg.GetDBQueries().GetDocumentCleanEntry(ctx, dbid) + if err != nil { + return err + } + + textVersion, err := s.cfg.GetDBQueries().GetDocumentTextEntry(ctx, dbid) + if err != nil { + return err + } + + query, err := s.svc.Query.Get(ctx, params.QueryID) + if err != nil { + return err + } + + value, err := s.svc.Result.Process(ctx, &result.Process{ + DocumentID: params.DocumentID, + QueryID: query.ID, + QueryVersion: query.ActiveVersion, + }) + if err != nil { + return err + } + + err = s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{ + Queryid: database.MustToDBUUID(params.QueryID), + Documentid: database.MustToDBUUID(params.DocumentID), + Value: value.GetStoreValue(), + Cleanversion: cleanVersion.Version, + Textversion: textVersion.Version, + Queryversion: query.ActiveVersion, + }) + if err != nil { + return err + } + + slog.Debug("set query result", "query_id", params.QueryID.String(), "document_id", params.DocumentID.String()) + + return s.informQueryDependents(ctx, params) +} + +func (s *Service) informQueryDependents(ctx context.Context, params *Set) error { + ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{ + ID: database.MustToDBUUID(params.DocumentID), + Requiredids: database.MustToDBUUID(params.QueryID), + }) + if err != nil { + return err + } + + return s.TriggerQueriesSync(ctx, params.DocumentID, database.MustToUUIDArray(ids)) +} + +func (s *Service) TriggerQueriesSync(ctx context.Context, documentID uuid.UUID, queryIDs []uuid.UUID) error { + for _, id := range queryIDs { + err := s.TriggerQuerySync(ctx, &Set{ + DocumentID: documentID, + QueryID: id, + }) + + if err != nil { + return err + } + } + + return nil +} + +func (s *Service) TriggerQuerySync(ctx context.Context, params *Set) error { + return s.cfg.SendToQueue(ctx, &queue.SendParams{ + QueueURL: s.cfg.GetQueryURL(), + Body: params, + }) +} diff --git a/internal/query/result/set/set_test.go b/internal/query/result/set/set_test.go new file mode 100644 index 00000000..83552723 --- /dev/null +++ b/internal/query/result/set/set_test.go @@ -0,0 +1,277 @@ +package resultset + +import ( + "context" + "fmt" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query" + "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" + "queryorchestration/internal/serviceconfig" + queryc "queryorchestration/internal/serviceconfig/queue/query" + queuemock "queryorchestration/mocks/queue" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type ResultSetConfig struct { + serviceconfig.BaseConfig + queryc.QueryConfig +} + +func TestSet(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &ResultSetConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + + que := query.New(cfg) + svc := Service{ + cfg: cfg, + svc: &Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }, + } + + qcfg := "{\"path\":\"examplekey\"}" + query := &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + Config: &qcfg, + } + params := Set{ + DocumentID: uuid.New(), + QueryID: query.ID, + } + + pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), + ) + pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}). + AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)), + ) + pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), + ) + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte(qcfg)), + ) + pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version). + WillReturnResult(pgxmock.NewResult("", 1)) + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(params.QueryID), database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"queryId"}). + AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0])), + ) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), (*query.RequiredQueryIDs)[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + err = svc.Set(ctx, ¶ms) + assert.NoError(t, err) +} + +func TestTriggerQuerySync(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &ResultSetConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + + que := query.New(cfg) + svc := Service{ + cfg: cfg, + svc: &Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }, + } + + params := &Set{ + DocumentID: uuid.New(), + QueryID: uuid.New(), + } + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), params.QueryID.String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + err = svc.TriggerQuerySync(ctx, params) + assert.NoError(t, err) +} + +func TestTriggerQueriesSync(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &ResultSetConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + + que := query.New(cfg) + svc := Service{ + cfg: cfg, + svc: &Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }, + } + + docId := uuid.New() + qIds := []uuid.UUID{ + uuid.New(), + uuid.New(), + } + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId.String(), qIds[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", docId.String(), qIds[1].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + err = svc.TriggerQueriesSync(ctx, docId, qIds) + assert.NoError(t, err) +} + +func TestInformQueryDependents(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &ResultSetConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + + que := query.New(cfg) + svc := Service{ + cfg: cfg, + svc: &Services{ + Query: que, + Result: result.New(cfg, &result.Services{ + Query: que, + }), + }, + } + + qIds := []uuid.UUID{ + uuid.New(), + uuid.New(), + } + params := &Set{ + DocumentID: uuid.New(), + QueryID: uuid.New(), + } + + pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(params.QueryID), database.MustToDBUUID(params.DocumentID)). + WillReturnRows( + pgxmock.NewRows([]string{"queryId"}). + AddRow(database.MustToDBUUID(qIds[0])). + AddRow(database.MustToDBUUID(qIds[1])), + ) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), qIds[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), qIds[1].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + err = svc.informQueryDependents(ctx, params) + assert.NoError(t, err) +} diff --git a/internal/query/result/set_test.go b/internal/query/result/set_test.go deleted file mode 100644 index 9f8ce5ba..00000000 --- a/internal/query/result/set_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package result - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - resultprocessor "queryorchestration/internal/query/result/processor" - "queryorchestration/internal/serviceconfig" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestSet(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - - svc := Service{ - cfg: cfg, - } - - rid := uuid.New() - qcfg := "{\"path\":\"examplekey\"}" - params := Set{ - DocumentID: uuid.New(), - CleanVersion: 1, - TextVersion: 3, - Query: &resultprocessor.Query{ - ID: uuid.New(), - Version: 2, - Config: &qcfg, - RequiredQueryIDs: &[]uuid.UUID{ - uuid.New(), - }, - }, - } - - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.CleanVersion, params.TextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "{\"examplekey\":\"example_value\"}", repository.QuerytypeContextFull), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(qcfg)), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(params.Query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), params.CleanVersion, params.TextVersion, params.Query.Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(database.MustToDBUUID(rid)), - ) - - id, err := svc.Set(ctx, ¶ms) - assert.NoError(t, err) - assert.Equal(t, rid, id) -} diff --git a/internal/query/result/sync.go b/internal/query/result/sync.go deleted file mode 100644 index 1c1567b0..00000000 --- a/internal/query/result/sync.go +++ /dev/null @@ -1,18 +0,0 @@ -package result - -import ( - "context" - "queryorchestration/internal/database" - resultprocessor "queryorchestration/internal/query/result/processor" - - "github.com/google/uuid" -) - -func (s *Service) ListUnsyncedQueriesByDocId(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) { - qs, err := s.cfg.GetDBQueries().ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id)) - if err != nil { - return nil, err - } - - return resultprocessor.ParseFullQueryArray(qs) -} diff --git a/internal/query/result/sync_test.go b/internal/query/result/sync_test.go deleted file mode 100644 index f668ae0d..00000000 --- a/internal/query/result/sync_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package result_test - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query/result" - resultprocessor "queryorchestration/internal/query/result/processor" - "queryorchestration/internal/serviceconfig" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestListUnsyncedQueriesByDocId(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := result.New(cfg) - - documentId := uuid.New() - actualQs := []*resultprocessor.Query{ - { - ID: uuid.New(), - Type: resultprocessor.TypeJsonExtractor, - Version: 2, - }, - } - - pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(documentId)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). - AddRow(database.MustToDBUUID(actualQs[0].ID), repository.QuerytypeJsonExtractor, actualQs[0].Version, actualQs[0].Version, nil, []pgtype.UUID{}), - ) - - val, err := svc.ListUnsyncedQueriesByDocId(ctx, documentId) - assert.NoError(t, err) - assert.ElementsMatch(t, actualQs, val) -} diff --git a/internal/query/service.go b/internal/query/service.go index de621b7b..8d52f772 100644 --- a/internal/query/service.go +++ b/internal/query/service.go @@ -1,35 +1,15 @@ package query import ( - "queryorchestration/internal/document" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/query/result" "queryorchestration/internal/serviceconfig" - - "github.com/google/uuid" ) -type Services struct { - Result *result.Service - Collector *collector.Service - Document *document.Service -} - type Service struct { cfg serviceconfig.ConfigProvider - svc *Services } -func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { +func New(cfg serviceconfig.ConfigProvider) *Service { return &Service{ cfg, - svc, } } - -type Document struct { - ID uuid.UUID `json:"id" validate:"required,uuid"` - JobID uuid.UUID `json:"jobId" validate:"required,uuid"` - CleanVersion int32 `json:"cleanVersion" validate:"required,gt=0"` - TextVersion int32 `json:"textVersion" validate:"required,gt=0"` -} diff --git a/internal/query/service_test.go b/internal/query/service_test.go index 930b33b2..b75fe895 100644 --- a/internal/query/service_test.go +++ b/internal/query/service_test.go @@ -18,6 +18,6 @@ func TestService(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) assert.NotNil(t, svc) } diff --git a/internal/query/sync.go b/internal/query/sync.go deleted file mode 100644 index 86942685..00000000 --- a/internal/query/sync.go +++ /dev/null @@ -1,163 +0,0 @@ -package query - -import ( - "context" - "errors" - "queryorchestration/internal/document" - "queryorchestration/internal/query/result" - resultprocessor "queryorchestration/internal/query/result/processor" - "sync" - - "github.com/google/uuid" -) - -func (s *Service) Sync(ctx context.Context, doc *Document) error { - err := s.svc.Document.IsTextExtracted(&document.IsTextExtractedParams{ - DocumentID: doc.ID, - MinCleanVersion: doc.CleanVersion, - MinTextVersion: doc.TextVersion, - }) - if err != nil { - return err - } - - unsyncedQueries, err := s.svc.Result.ListUnsyncedQueriesByDocId(ctx, doc.ID) - if err != nil { - return err - } - - batchedQueries := s.batchQueries(unsyncedQueries) - - for _, queries := range batchedQueries { - err := s.processBatch(ctx, doc, queries) - if err != nil { - return err - } - } - - return nil -} - -func (s *Service) batchQueries(queries []*resultprocessor.Query) [][]*resultprocessor.Query { - n := len(queries) - if n == 0 { - return nil - } - - idToIndex := make(map[uuid.UUID]int, n) - for i, q := range queries { - idToIndex[q.ID] = i - } - - result := make([][]*resultprocessor.Query, 0, n) - assigned := make([]bool, n) - deps := make([][]int, n) - remaining := n - - for i, q := range queries { - toAppend := false - if q.RequiredQueryIDs == nil || len(*q.RequiredQueryIDs) == 0 { - toAppend = true - } else { - deps[i] = make([]int, 0, len(*q.RequiredQueryIDs)) - - for _, reqID := range *q.RequiredQueryIDs { - if idx, exists := idToIndex[reqID]; exists { - deps[i] = append(deps[i], idx) - } - } - - toAppend = len(deps[i]) == 0 - } - - if toAppend { - if len(result) == 0 { - result = [][]*resultprocessor.Query{{q}} - } else { - result[0] = append(result[0], q) - } - assigned[idToIndex[q.ID]] = true - remaining-- - } - } - - for remaining > 0 { - currentLayer := make([]*resultprocessor.Query, 0, remaining) - - for i, q := range queries { - if assigned[i] { - continue - } - - allSatisfied := true - for _, depIdx := range deps[i] { - if !assigned[depIdx] { - allSatisfied = false - break - } - } - - if allSatisfied { - currentLayer = append(currentLayer, q) - } - } - - for _, v := range currentLayer { - assigned[idToIndex[v.ID]] = true - remaining-- - } - - result = append(result, currentLayer) - } - - return result -} - -func (s *Service) processBatch(ctx context.Context, doc *Document, queries []*resultprocessor.Query) error { - if doc == nil { - return errors.New("document required") - } - - errChan := make(chan error, len(queries)) - var wg sync.WaitGroup - sem := make(chan struct{}, 10) - - for _, query := range queries { - wg.Add(1) - go func(q *resultprocessor.Query) { - defer wg.Done() - - sem <- struct{}{} - defer func() { - <-sem - }() - - select { - case <-ctx.Done(): - errChan <- ctx.Err() - return - default: - } - - _, err := s.svc.Result.Set(ctx, &result.Set{ - DocumentID: doc.ID, - CleanVersion: doc.CleanVersion, - TextVersion: doc.TextVersion, - Query: query, - }) - if err != nil { - errChan <- err - } - }(query) - } - - wg.Wait() - close(errChan) - for err := range errChan { - if err != nil { - return err - } - } - - return nil -} diff --git a/internal/query/sync/service.go b/internal/query/sync/service.go new file mode 100644 index 00000000..ea3e321a --- /dev/null +++ b/internal/query/sync/service.go @@ -0,0 +1,28 @@ +package querysync + +import ( + resultset "queryorchestration/internal/query/result/set" + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/queue/query" +) + +type ConfigProvider interface { + serviceconfig.ConfigProvider + query.ConfigProvider +} + +type Services struct { + ResultSet *resultset.Service +} + +type Service struct { + cfg ConfigProvider + svc *Services +} + +func New(cfg ConfigProvider, svc *Services) *Service { + return &Service{ + cfg, + svc, + } +} diff --git a/internal/query/sync/service_test.go b/internal/query/sync/service_test.go new file mode 100644 index 00000000..781fbf9c --- /dev/null +++ b/internal/query/sync/service_test.go @@ -0,0 +1,29 @@ +package querysync_test + +import ( + "queryorchestration/internal/database/repository" + querysync "queryorchestration/internal/query/sync" + "queryorchestration/internal/serviceconfig" + "queryorchestration/internal/serviceconfig/queue/query" + "testing" + + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +type QuerySyncConfig struct { + serviceconfig.BaseConfig + query.QueryConfig +} + +func TestService(t *testing.T) { + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &QuerySyncConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := querysync.New(cfg, &querysync.Services{}) + assert.NotNil(t, svc) +} diff --git a/internal/query/sync/sync.go b/internal/query/sync/sync.go new file mode 100644 index 00000000..f3130f6c --- /dev/null +++ b/internal/query/sync/sync.go @@ -0,0 +1,35 @@ +package querysync + +import ( + "context" + "fmt" + "log/slog" + "queryorchestration/internal/database" + + "github.com/google/uuid" +) + +func (s *Service) Sync(ctx context.Context, id uuid.UUID) error { + dbid := database.MustToDBUUID(id) + + isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, dbid) + if err != nil { + return err + } else if !isextracted { + return fmt.Errorf("document text must be extracted") + } + + unsyncedQueries, err := s.cfg.GetDBQueries().ListUnsyncedNoDepsQueriesByDocId(ctx, dbid) + if err != nil { + return err + } + + slog.Debug("unsynced queries", "document_id", id.String(), "queries", unsyncedQueries) + + ids := make([]uuid.UUID, len(unsyncedQueries)) + for i, dbId := range unsyncedQueries { + ids[i] = database.MustToUUID(dbId) + } + + return s.svc.ResultSet.TriggerQueriesSync(ctx, id, ids) +} diff --git a/internal/query/sync/sync_test.go b/internal/query/sync/sync_test.go new file mode 100644 index 00000000..d711a163 --- /dev/null +++ b/internal/query/sync/sync_test.go @@ -0,0 +1,88 @@ +package querysync + +import ( + "context" + "fmt" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query" + resultset "queryorchestration/internal/query/result/set" + "queryorchestration/internal/serviceconfig" + queryc "queryorchestration/internal/serviceconfig/queue/query" + queuemock "queryorchestration/mocks/queue" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/google/uuid" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type QuerySyncConfig struct { + serviceconfig.BaseConfig + queryc.QueryConfig +} + +func TestSync(t *testing.T) { + ctx := context.Background() + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &QuerySyncConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + mockSQS := queuemock.NewMockSQSClient(t) + cfg.QueueClient = mockSQS + cfg.QueryURL = "/i/am/here" + que := query.New(cfg) + svc := Service{ + cfg: cfg, + svc: &Services{ + ResultSet: resultset.New(cfg, &resultset.Services{ + Query: que, + }), + }, + } + + id := uuid.New() + qs := []uuid.UUID{ + uuid.New(), + uuid.New(), + } + + pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(database.MustToDBUUID(id)). + WillReturnRows( + pgxmock.NewRows([]string{"isextracted"}). + AddRow(true), + ) + pool.ExpectQuery("name: ListUnsyncedNoDepsQueriesByDocId :many").WithArgs(database.MustToDBUUID(id)). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(qs[0])). + AddRow(database.MustToDBUUID(qs[1])), + ) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", id.String(), qs[0].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + mockSQS.EXPECT(). + SendMessage( + mock.Anything, + mock.MatchedBy(func(in *sqs.SendMessageInput) bool { + return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", id.String(), qs[1].String()) + }), + mock.Anything, + ). + Return(&sqs.SendMessageOutput{}, nil) + + assert.Nil(t, svc.Sync(ctx, id)) +} diff --git a/internal/query/sync_test.go b/internal/query/sync_test.go deleted file mode 100644 index 83b5fc0a..00000000 --- a/internal/query/sync_test.go +++ /dev/null @@ -1,304 +0,0 @@ -package query - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/query/result" - resultprocessor "queryorchestration/internal/query/result/processor" - "queryorchestration/internal/serviceconfig" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestBatchQueries(t *testing.T) { - svc := Service{} - - in := []*resultprocessor.Query{} - out := svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{}, out) - - in = []*resultprocessor.Query{ - { - ID: uuid.New(), - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - }, - }, out) - - in = []*resultprocessor.Query{ - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - }, - }, out) - - in = []*resultprocessor.Query{ - { - ID: uuid.New(), - }, - { - ID: uuid.New(), - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - in[1], - }, - }, out) - - idone := uuid.New() - in = []*resultprocessor.Query{ - { - ID: idone, - }, - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{ - idone, - }, - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - }, - { - in[1], - }, - }, out) - - in = []*resultprocessor.Query{ - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{ - idone, - }, - }, - { - ID: idone, - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[1], - }, - { - in[0], - }, - }, out) - - in = []*resultprocessor.Query{ - { - ID: idone, - }, - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{ - idone, - }, - }, - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{ - idone, - }, - }, - { - ID: uuid.New(), - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - in[3], - }, - { - in[1], - in[2], - }, - }, out) - - idtwo := uuid.New() - idthree := uuid.New() - in = []*resultprocessor.Query{ - { - ID: idone, - }, - { - ID: idtwo, - RequiredQueryIDs: &[]uuid.UUID{ - idone, - }, - }, - { - ID: idthree, - RequiredQueryIDs: &[]uuid.UUID{ - idtwo, - }, - }, - { - ID: uuid.New(), - RequiredQueryIDs: &[]uuid.UUID{ - idthree, - }, - }, - } - out = svc.batchQueries(in) - assert.ElementsMatch(t, [][]*resultprocessor.Query{ - { - in[0], - }, - { - in[1], - }, - { - in[2], - }, - { - in[3], - }, - }, out) -} - -func TestProcessBatch(t *testing.T) { - ctx := context.Background() - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := Service{ - svc: &Services{ - Result: result.New(cfg), - }, - } - - assert.Error(t, svc.processBatch(ctx, nil, nil)) - - doc := &Document{ - ID: uuid.New(), - CleanVersion: 1, - TextVersion: 1, - } - - assert.Nil(t, svc.processBatch(ctx, doc, nil)) - - qs := []*resultprocessor.Query{} - assert.Nil(t, svc.processBatch(ctx, doc, qs)) - - coll := &collector.Collector{ - ID: uuid.New(), - MinCleanVersion: 1, - MinTextVersion: 1, - } - qs = []*resultprocessor.Query{ - { - ID: uuid.New(), - Type: resultprocessor.TypeContextFull, - Version: 1, - }, - } - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qs[0].ID), database.MustToDBUUID(doc.ID), pgxmock.AnyArg(), coll.MinCleanVersion, coll.MinTextVersion, qs[0].Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(pgtype.UUID{}), - ) - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, []pgtype.UUID{}), - ) - assert.Nil(t, svc.processBatch(ctx, doc, qs)) -} - -func TestSync(t *testing.T) { - ctx := context.Background() - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := Service{ - svc: &Services{ - Result: result.New(cfg), - }, - } - - assert.Error(t, svc.processBatch(ctx, nil, nil)) - - doc := &Document{ - ID: uuid.New(), - CleanVersion: 1, - TextVersion: 1, - } - - assert.Nil(t, svc.processBatch(ctx, doc, nil)) - - qs := []*resultprocessor.Query{} - assert.Nil(t, svc.processBatch(ctx, doc, qs)) - - coll := &collector.Collector{ - ID: uuid.New(), - MinCleanVersion: 1, - MinTextVersion: 1, - } - qs = []*resultprocessor.Query{ - { - ID: uuid.New(), - Type: resultprocessor.TypeContextFull, - Version: 1, - }, - } - - pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). - AddRow(database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, qs[0].Version, nil, []pgtype.UUID{}), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qs[0].ID), database.MustToDBUUID(doc.ID), pgxmock.AnyArg(), coll.MinCleanVersion, coll.MinTextVersion, qs[0].Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}). - AddRow(pgtype.UUID{}), - ) - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, []pgtype.UUID{}), - ) - assert.Nil(t, svc.Sync(ctx, doc)) -} diff --git a/internal/query/test/service.go b/internal/query/test/service.go new file mode 100644 index 00000000..60cd6557 --- /dev/null +++ b/internal/query/test/service.go @@ -0,0 +1,26 @@ +package querytest + +import ( + "queryorchestration/internal/document" + "queryorchestration/internal/job/collector" + "queryorchestration/internal/query/result" + "queryorchestration/internal/serviceconfig" +) + +type Services struct { + Result *result.Service + Collector *collector.Service + Document *document.Service +} + +type Service struct { + cfg serviceconfig.ConfigProvider + svc *Services +} + +func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service { + return &Service{ + cfg, + svc, + } +} diff --git a/internal/query/test/service_test.go b/internal/query/test/service_test.go new file mode 100644 index 00000000..df308182 --- /dev/null +++ b/internal/query/test/service_test.go @@ -0,0 +1,23 @@ +package querytest_test + +import ( + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query" + "queryorchestration/internal/serviceconfig" + "testing" + + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestService(t *testing.T) { + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := query.New(cfg) + assert.NotNil(t, svc) +} diff --git a/internal/query/test.go b/internal/query/test/test.go similarity index 50% rename from internal/query/test.go rename to internal/query/test/test.go index 77d841ac..0ac6990c 100644 --- a/internal/query/test.go +++ b/internal/query/test/test.go @@ -1,20 +1,11 @@ -package query +package querytest import ( "context" "queryorchestration/internal/query/result" - resultprocessor "queryorchestration/internal/query/result/processor" - - "github.com/google/uuid" ) -type Test struct { - QueryID uuid.UUID - DocumentID uuid.UUID - QueryVersion int32 -} - -func (s *Service) Test(ctx context.Context, filters Test) (string, error) { +func (s *Service) Test(ctx context.Context, filters result.Process) (string, error) { doc, err := s.svc.Document.Get(ctx, filters.DocumentID) if err != nil { return "", err @@ -36,22 +27,10 @@ func (s *Service) Test(ctx context.Context, filters Test) (string, error) { return res.GetValue(ctx) } - q, err := s.GetWithVersion(ctx, filters.QueryID, filters.QueryVersion) - if err != nil { - return "", err - } - value, err := s.svc.Result.Process(ctx, &result.Process{ - DocumentID: filters.DocumentID, - MinCleanVersion: col.MinCleanVersion, - MinTextVersion: col.MinTextVersion, - Query: &resultprocessor.Query{ - ID: q.ID, - Type: q.Type, - Version: filters.QueryVersion, - RequiredQueryIDs: q.RequiredQueryIDs, - Config: q.Config, - }, + DocumentID: filters.DocumentID, + QueryID: filters.QueryID, + QueryVersion: filters.QueryVersion, }) if err != nil { return "", err diff --git a/internal/query/test_test.go b/internal/query/test/test_test.go similarity index 84% rename from internal/query/test_test.go rename to internal/query/test/test_test.go index 1059d989..4b9abcb1 100644 --- a/internal/query/test_test.go +++ b/internal/query/test/test_test.go @@ -1,4 +1,4 @@ -package query_test +package querytest_test import ( "context" @@ -8,6 +8,7 @@ import ( "queryorchestration/internal/job/collector" "queryorchestration/internal/query" "queryorchestration/internal/query/result" + querytest "queryorchestration/internal/query/test" "queryorchestration/internal/serviceconfig" "testing" @@ -31,10 +32,12 @@ func TestTest(t *testing.T) { col := collector.New(cfg, &collector.Services{ Document: docsvc, }) - svc := query.New(cfg, &query.Services{ + svc := querytest.New(cfg, &querytest.Services{ Document: docsvc, Collector: col, - Result: result.New(cfg), + Result: result.New(cfg, &result.Services{ + Query: query.New(cfg), + }), }) coll := collector.Collector{ @@ -46,7 +49,7 @@ func TestTest(t *testing.T) { JobID: coll.JobID, Hash: "example_hash", } - params := &query.Test{ + params := &result.Process{ QueryID: uuid.New(), DocumentID: doc.ID, QueryVersion: int32(1), @@ -67,10 +70,10 @@ func TestTest(t *testing.T) { pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), ) - pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion). + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion). WillReturnRows( - pgxmock.NewRows([]string{"queryId", "value", "type"}). - AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull), + pgxmock.NewRows([]string{"queryId", "type", "value"}). + AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"), ) pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( pgxmock.NewRows([]string{"id", "config"}). diff --git a/internal/query/types/contextFull/process_test.go b/internal/query/types/contextFull/process_test.go index 6873b322..10c25fd3 100644 --- a/internal/query/types/contextFull/process_test.go +++ b/internal/query/types/contextFull/process_test.go @@ -2,6 +2,7 @@ package contextfull_test import ( "context" + "fmt" resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" "testing" @@ -25,7 +26,7 @@ func TestContextFull(t *testing.T) { value, err := extractor.Process(ctx, query, &values) assert.NoError(t, err) - assert.Equal(t, "", value) + assert.Equal(t, fmt.Sprintf("{\"id\":\"%s\"}", query.ID), value) values = []resultprocessor.Value{ contextfull.NewResult("example_result"), diff --git a/internal/query/types/contextFull/service.go b/internal/query/types/contextFull/service.go index 3950f444..343017ba 100644 --- a/internal/query/types/contextFull/service.go +++ b/internal/query/types/contextFull/service.go @@ -3,6 +3,7 @@ package contextfull import ( "context" "errors" + "fmt" resultprocessor "queryorchestration/internal/query/result/processor" ) @@ -19,5 +20,5 @@ func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, v } // TODO - return "", nil + return fmt.Sprintf(`{"id":"%s"}`, query.ID.String()), nil } diff --git a/internal/query/types/jsonExtractor/creator.go b/internal/query/types/jsonExtractor/creator.go index a71db13a..2e3c1f5f 100644 --- a/internal/query/types/jsonExtractor/creator.go +++ b/internal/query/types/jsonExtractor/creator.go @@ -2,6 +2,7 @@ package jsonextractor import ( "context" + "errors" resultprocessor "queryorchestration/internal/query/result/processor" ) @@ -12,6 +13,9 @@ func NewCreator() *Creator { } func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { + if entity.Config == nil { + return errors.New("config required") + } // TODO return nil } diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go index 140bb0d0..5787eaea 100644 --- a/internal/query/types/jsonExtractor/creator_test.go +++ b/internal/query/types/jsonExtractor/creator_test.go @@ -26,8 +26,10 @@ func TestCreatorValidate(t *testing.T) { svc := jsonextractor.NewCreator() assert.NotNil(t, svc) + ccfg := "{}" entity := &resultprocessor.Create{ - Type: resultprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, + Config: &ccfg, } err = svc.Validate(ctx, entity) diff --git a/internal/query/update.go b/internal/query/update.go index 17afe35d..8510550d 100644 --- a/internal/query/update.go +++ b/internal/query/update.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "log/slog" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "queryorchestration/internal/validation" "github.com/google/uuid" ) @@ -32,37 +34,59 @@ func (s *Service) Update(ctx context.Context, entity *resultprocessor.Update) er return nil } +func (s *Service) normalizeUpdateRequiredQueryIDs(ctx context.Context, current *Query, entity RequiredQueryIDs) error { + err := s.NormalizeQueryIDs(ctx, entity) + if err != nil { + return err + } + + if entity.GetRequiredQueryIDs() == nil { + return nil + } + + createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ + Requiredqueryid: database.MustToDBUUID(current.ID), + ID: database.MustToDBUUIDArray(*entity.GetRequiredQueryIDs()), + }) + if err != nil { + return err + } else if createsloop { + return errors.New("required ids create a loop") + } + + addIDs := getSetDifference(entity.GetRequiredQueryIDs(), current.RequiredQueryIDs) + removeIDs := getSetDifference(current.RequiredQueryIDs, entity.GetRequiredQueryIDs()) + + if len(addIDs) == 0 && len(removeIDs) == 0 { + entity.SetRequiredQueryIDs(nil) + } + + return nil +} + func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *resultprocessor.Update) error { err := s.normalizeActiveVersion(current, entity) if err != nil { return err } - err = s.NormalizeQueryIDs(ctx, entity) + err = s.normalizeUpdateRequiredQueryIDs(ctx, current, entity) if err != nil { return err } - if entity.RequiredQueryIDs != nil { - createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{ - Requiredqueryid: database.MustToDBUUID(current.ID), - ID: database.MustToDBUUIDArray(*entity.RequiredQueryIDs), - }) - if err != nil { - return err - } else if createsloop { - return errors.New("required ids create a loop") - } - } - err = s.normalizeConfig(entity) if err != nil { return err } - if entity.ActiveVersion == nil && - entity.RequiredQueryIDs == nil && - entity.Config == nil { + activeVersionName, err := validation.GetFieldName(entity, entity.ActiveVersion) + if err != nil { + return err + } + + if (entity.ActiveVersion == nil || *entity.ActiveVersion == current.LatestVersion+1) && + validation.AreAllPointersNilExcept(entity, activeVersionName) { return errors.New("no changes") } @@ -141,6 +165,8 @@ func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *resu return err } + slog.Debug("query updated", "update", *entity) + return nil }) if err != nil { diff --git a/internal/query/update_test.go b/internal/query/update_test.go index 02d8381d..41487be3 100644 --- a/internal/query/update_test.go +++ b/internal/query/update_test.go @@ -25,7 +25,7 @@ func TestUpdate(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := query.New(cfg, &query.Services{}) + svc := query.New(cfg) config := "{\"path\":\"example_path\"}" existing := query.Query{ diff --git a/internal/query/updateprivate_test.go b/internal/query/updateprivate_test.go index a0e9fd61..a80bb3d6 100644 --- a/internal/query/updateprivate_test.go +++ b/internal/query/updateprivate_test.go @@ -23,7 +23,7 @@ func TestGetUpdator(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) queryType := resultprocessor.Type(resultprocessor.TypeContextFull) updator, err := svc.getUpdator(queryType) @@ -50,7 +50,7 @@ func TestSubmitUpdate(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) config := "{\"path\":\"example_path\"}" q := Query{ @@ -99,7 +99,7 @@ func TestSubmitUpdateRollback(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) q := Query{ ID: uuid.New(), @@ -132,7 +132,7 @@ func TestSubmitUpdateRequiredQueries(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) config := "{\"path\":\"example_path\"}" q := Query{ @@ -182,7 +182,7 @@ func TestSubmitUpdateActiveVersion(t *testing.T) { cfg := &serviceconfig.BaseConfig{} cfg.DBPool = pool cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + svc := New(cfg) q := Query{ ID: uuid.New(), @@ -218,63 +218,252 @@ func TestGetSetDifference(t *testing.T) { } func TestNormalizeUpdate(t *testing.T) { - ctx := context.Background() + t.Run("all fields to change", func(t *testing.T) { + ctx := context.Background() - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - cfg := &serviceconfig.BaseConfig{} - cfg.DBPool = pool - cfg.DBQueries = repository.New(pool) - svc := New(cfg, &Services{}) + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) - current := &Query{ - ID: uuid.New(), - ActiveVersion: int32(1), - LatestVersion: int32(2), - Type: resultprocessor.TypeJsonExtractor, - } - qcfg := "{}" - aV := int32(2) - update := &resultprocessor.Update{ - ID: current.ID, - Config: &qcfg, - ActiveVersion: &aV, - RequiredQueryIDs: &[]uuid.UUID{}, - } + current := &Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + Type: resultprocessor.TypeJsonExtractor, + } + qcfg := "{}" + aV := int32(2) + update := &resultprocessor.Update{ + ID: current.ID, + Config: &qcfg, + ActiveVersion: &aV, + RequiredQueryIDs: &[]uuid.UUID{}, + } - dbids := database.MustToDBUUIDArray(*update.RequiredQueryIDs) + err = svc.normalizeUpdate(ctx, current, update) + assert.NoError(t, err) + assert.EqualExportedValues(t, resultprocessor.Update{ + ID: current.ID, + ActiveVersion: &aV, + Config: &qcfg, + RequiredQueryIDs: nil, + }, *update) + }) - pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( - pgxmock.NewRows([]string{"all_exist"}). - AddRow(true), - ) + t.Run("no changes", func(t *testing.T) { + ctx := context.Background() - err = svc.normalizeUpdate(ctx, current, update) - assert.NoError(t, err) - assert.EqualExportedValues(t, resultprocessor.Update{ - ID: current.ID, - ActiveVersion: &aV, - Config: &qcfg, - RequiredQueryIDs: nil, - }, *update) + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) - update.RequiredQueryIDs = &[]uuid.UUID{uuid.New()} + current := &Query{ + ID: uuid.New(), + ActiveVersion: int32(1), + LatestVersion: int32(2), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + } + err = svc.normalizeUpdate(ctx, current, update) + assert.EqualError(t, err, "no changes") + }) - dbids = database.MustToDBUUIDArray(*update.RequiredQueryIDs) + t.Run("same active version", func(t *testing.T) { + ctx := context.Background() - pool.ExpectQuery("name: IsQueryInDependencyTree :one").WithArgs(dbids).WillReturnRows( - pgxmock.NewRows([]string{"all_exist"}). - AddRow(true), - ) + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) - err = svc.normalizeUpdate(ctx, current, update) - assert.Error(t, err) + version := int32(1) + current := &Query{ + ID: uuid.New(), + ActiveVersion: version, + LatestVersion: int32(2), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + ActiveVersion: &version, + } + err = svc.normalizeUpdate(ctx, current, update) + assert.EqualError(t, err, "no changes") + }) - update = &resultprocessor.Update{ - ID: current.ID, - } - err = svc.normalizeUpdate(ctx, current, update) - assert.Error(t, err) + t.Run("ONLY active version to next latest", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) + + current := &Query{ + ID: uuid.New(), + ActiveVersion: 1, + LatestVersion: 2, + Type: resultprocessor.TypeJsonExtractor, + } + version := int32(3) + update := &resultprocessor.Update{ + ID: current.ID, + ActiveVersion: &version, + } + err = svc.normalizeUpdate(ctx, current, update) + assert.EqualError(t, err, "no changes") + }) +} + +func TestNormalizeUpdateRequiredQueryIDs(t *testing.T) { + t.Run("Empty Array", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) + + current := &Query{ + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + RequiredQueryIDs: &[]uuid.UUID{}, + } + + err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update) + assert.NoError(t, err) + assert.EqualExportedValues(t, resultprocessor.Update{ + ID: current.ID, + }, *update) + }) + + t.Run("entry doesn't exist", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) + + current := &Query{ + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + } + dbids := database.MustToDBUUIDArray(*update.RequiredQueryIDs) + + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(false), + ) + + err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update) + assert.Error(t, err) + }) + t.Run("in dependency tree", func(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) + + current := &Query{ + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + } + + dbids := database.MustToDBUUIDArray(*update.RequiredQueryIDs) + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + pool.ExpectQuery("name: IsQueryInDependencyTree :one").WithArgs(dbids, database.MustToDBUUID(current.ID)).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + + err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update) + assert.Error(t, err) + }) + t.Run("same update and current", func(t *testing.T) { + ctx := context.Background() + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + cfg := &serviceconfig.BaseConfig{} + cfg.DBPool = pool + cfg.DBQueries = repository.New(pool) + svc := New(cfg) + + current := &Query{ + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, + } + update := &resultprocessor.Update{ + ID: current.ID, + } + ids := []uuid.UUID{uuid.New()} + update.RequiredQueryIDs = &ids + current.RequiredQueryIDs = &ids + + dbids := database.MustToDBUUIDArray(*update.RequiredQueryIDs) + pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(dbids).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(true), + ) + pool.ExpectQuery("name: IsQueryInDependencyTree :one").WithArgs(dbids, database.MustToDBUUID(current.ID)).WillReturnRows( + pgxmock.NewRows([]string{"all_exist"}). + AddRow(false), + ) + + err = svc.normalizeUpdateRequiredQueryIDs(ctx, current, update) + assert.NoError(t, err) + assert.EqualExportedValues(t, resultprocessor.Update{ + ID: current.ID, + }, *update) + }) } diff --git a/internal/server/runner/poll.go b/internal/server/runner/poll.go index 92b237cd..b7f30916 100644 --- a/internal/server/runner/poll.go +++ b/internal/server/runner/poll.go @@ -43,7 +43,7 @@ func (c *Server) pollMessage(ctx context.Context) error { } for _, message := range result.Messages { - slog.Info("processing message", "id", *message.MessageId) + slog.Debug("processing message", "id", *message.MessageId, "body", *message.Body) err := c.cfg.GetController().Process(ctx, &message) if err != nil { diff --git a/internal/serviceconfig/common.go b/internal/serviceconfig/common.go index a4ae98c8..3e95163e 100644 --- a/internal/serviceconfig/common.go +++ b/internal/serviceconfig/common.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "log/slog" - "os" "queryorchestration/internal/serviceconfig/aws" "queryorchestration/internal/serviceconfig/database" "queryorchestration/internal/serviceconfig/logger" @@ -97,8 +96,7 @@ func InitializeConfig(cfg ConfigProvider) error { return errors.New("no BaseConfig found in the provided config struct") } - baseConfig.Logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) - slog.SetDefault(baseConfig.Logger) + baseConfig.SetDefaultLogger() if err := godotenv.Load(); err != nil { slog.Warn("No .env file found or error loading it", "error", err) @@ -117,6 +115,8 @@ func InitializeConfig(cfg ConfigProvider) error { return err } + baseConfig.SetDefaultLogger() + return nil } diff --git a/internal/serviceconfig/logger/config.go b/internal/serviceconfig/logger/config.go index 663542e4..4c04e642 100644 --- a/internal/serviceconfig/logger/config.go +++ b/internal/serviceconfig/logger/config.go @@ -2,13 +2,17 @@ package logger import ( "log/slog" + "os" ) type LogConfig struct { - Logger *slog.Logger + Logger *slog.Logger + LogLevel string `env:"LOG_LEVEL"` } type ConfigProvider interface { + SetDefaultLogger() + GetLogLevel() slog.Leveler GetLogger() *slog.Logger PrintConfig(prefixSecret string) } @@ -16,3 +20,27 @@ type ConfigProvider interface { func (b *LogConfig) GetLogger() *slog.Logger { return b.Logger } + +func (b *LogConfig) GetLogLevel() slog.Leveler { + level := slog.LevelInfo + switch b.LogLevel { + case "DEBUG": + level = slog.LevelDebug + case "ERROR": + level = slog.LevelError + case "INFO": + level = slog.LevelInfo + case "WARN": + level = slog.LevelWarn + } + + return level +} + +func (b *LogConfig) SetDefaultLogger() { + b.Logger = slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{ + Level: b.GetLogLevel(), + })) + + slog.SetDefault(b.Logger) +} diff --git a/internal/serviceconfig/logger/config_test.go b/internal/serviceconfig/logger/config_test.go index 89727273..9d5565dc 100644 --- a/internal/serviceconfig/logger/config_test.go +++ b/internal/serviceconfig/logger/config_test.go @@ -13,3 +13,23 @@ func TestGetLogger(t *testing.T) { cfg.Logger = slog.Default() assert.NotNil(t, cfg.GetLogger()) } + +func TestSetLogger(t *testing.T) { + cfg := &LogConfig{} + assert.Nil(t, cfg.Logger) + cfg.SetDefaultLogger() + assert.NotNil(t, cfg.Logger) +} + +func TestGetLogLevel(t *testing.T) { + cfg := &LogConfig{} + assert.Equal(t, slog.LevelInfo, cfg.GetLogLevel()) + cfg.LogLevel = "DEBUG" + assert.Equal(t, slog.LevelDebug, cfg.GetLogLevel()) + cfg.LogLevel = "WARN" + assert.Equal(t, slog.LevelWarn, cfg.GetLogLevel()) + cfg.LogLevel = "ERROR" + assert.Equal(t, slog.LevelError, cfg.GetLogLevel()) + cfg.LogLevel = "INFO" + assert.Equal(t, slog.LevelInfo, cfg.GetLogLevel()) +} diff --git a/internal/serviceconfig/queue/delete.go b/internal/serviceconfig/queue/delete.go index 8366e265..423baad0 100644 --- a/internal/serviceconfig/queue/delete.go +++ b/internal/serviceconfig/queue/delete.go @@ -2,6 +2,7 @@ package queue import ( "context" + "log/slog" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -13,6 +14,8 @@ type DeleteParams struct { } func (c *QueueConfig) DeleteFromQueue(ctx context.Context, params *DeleteParams) error { + slog.Debug("deleting message from queue", "url", params.QueueURL, "receipt_handle", params.ReceiptHandle) + _, err := c.QueueClient.DeleteMessage(ctx, &sqs.DeleteMessageInput{ QueueUrl: aws.String(params.QueueURL), ReceiptHandle: params.ReceiptHandle, diff --git a/internal/serviceconfig/queue/query/config.go b/internal/serviceconfig/queue/query/config.go new file mode 100644 index 00000000..0c7e5544 --- /dev/null +++ b/internal/serviceconfig/queue/query/config.go @@ -0,0 +1,13 @@ +package query + +type QueryConfig struct { + QueryURL string `env:"QUERY_URL,required,notEmpty"` +} + +func (c *QueryConfig) GetQueryURL() string { + return c.QueryURL +} + +type ConfigProvider interface { + GetQueryURL() string +} diff --git a/internal/serviceconfig/queue/query/config_test.go b/internal/serviceconfig/queue/query/config_test.go new file mode 100644 index 00000000..3091003e --- /dev/null +++ b/internal/serviceconfig/queue/query/config_test.go @@ -0,0 +1,20 @@ +package query_test + +import ( + "queryorchestration/internal/serviceconfig/queue/query" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetQueryURL(t *testing.T) { + cfg := query.QueryConfig{} + + name := cfg.GetQueryURL() + assert.Equal(t, "", name) + + cfg.QueryURL = "name" + name = cfg.GetQueryURL() + assert.Equal(t, "name", name) + assert.Equal(t, cfg.QueryURL, name) +} diff --git a/internal/serviceconfig/queue/send.go b/internal/serviceconfig/queue/send.go index b27c5712..5ff8f311 100644 --- a/internal/serviceconfig/queue/send.go +++ b/internal/serviceconfig/queue/send.go @@ -3,6 +3,7 @@ package queue import ( "context" "encoding/json" + "log/slog" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/sqs" @@ -23,6 +24,8 @@ func (c *QueueConfig) SendToQueue(ctx context.Context, params *SendParams) error strBody := string(jsonBytes) + slog.Debug("sending to queue", "url", params.QueueURL, "body", strBody) + _, err = c.QueueClient.SendMessage(ctx, &sqs.SendMessageInput{ MessageAttributes: params.Attributes, QueueUrl: aws.String(params.QueueURL), diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index 90cf4475..03d6786b 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -21,6 +21,9 @@ func TestCreateRunnerNetwork(t *testing.T) { conn, cleanup := CreateRunnerNetwork(t, ctx, &RunnerNetworkConfig{ Cfg: cfg, Name: QueryRunner, + Env: map[string]string{ + "QUERY_URL": "/i/am/here", + }, }) assert.NotNil(t, conn) @@ -57,7 +60,12 @@ func TestCreateRunnersAndServicesNetwork(t *testing.T) { conn, cleanup := CreateRunnersAndServicesNetwork(t, ctx, &EcosystemNetworkConfig{ Cfg: cfg, Runners: []*RunnerNetworkConfig{ - {Name: QueryRunner}, + { + Name: QueryRunner, + Env: map[string]string{ + "QUERY_URL": "/i/am/here", + }, + }, }, Services: []*ServiceNetworkConfig{ {Name: QueryService}, diff --git a/internal/test/runner_test.go b/internal/test/runner_test.go index 06936a24..1af92d91 100644 --- a/internal/test/runner_test.go +++ b/internal/test/runner_test.go @@ -40,6 +40,9 @@ func TestCreateRunner(t *testing.T) { Cfg: cfg, Network: ncfg, QueueURL: url, + Env: map[string]string{ + "QUERY_URL": "/i/am/here", + }, } c, cleanup := CreateRunner(t, ctx, qccfg) diff --git a/internal/validation/validation.go b/internal/validation/validation.go index 8e2dffdc..0a3fc132 100644 --- a/internal/validation/validation.go +++ b/internal/validation/validation.go @@ -1,6 +1,10 @@ package validation -import "fmt" +import ( + "errors" + "fmt" + "reflect" +) func GetUpdatedValue[T any](def T, new *T) T { if new != nil { @@ -39,3 +43,56 @@ func NormalizeInClosedInterval(updated **int32, current int32, min int32, max in return nil } + +func AreAllPointersNilExcept[T any](s T, exceptions ...string) bool { + v := reflect.ValueOf(s) + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + if v.Kind() != reflect.Struct { + return false + } + + exceptFields := make(map[string]bool) + for _, name := range exceptions { + exceptFields[name] = true + } + + t := v.Type() + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + fieldName := t.Field(i).Name + + if exceptFields[fieldName] || field.Kind() != reflect.Ptr { + continue + } + + if !field.IsNil() { + return false + } + } + + return true +} + +func GetFieldName[T any](s T, field any) (string, error) { + v := reflect.ValueOf(s) + if v.Kind() == reflect.Ptr { + v = v.Elem() + } + + t := v.Type() + fieldValue := reflect.ValueOf(field) + + for i := 0; i < v.NumField(); i++ { + f := v.Field(i) + + if f.Kind() == reflect.Ptr && f.Pointer() == fieldValue.Pointer() || + f.Kind() != reflect.Ptr && f.CanAddr() && f.Addr().Pointer() == fieldValue.Pointer() { + return t.Field(i).Name, nil + } + } + + return "", errors.New("field not found") +} diff --git a/internal/validation/validation_test.go b/internal/validation/validation_test.go index f9f14ef2..3d87f006 100644 --- a/internal/validation/validation_test.go +++ b/internal/validation/validation_test.go @@ -40,6 +40,7 @@ func TestNormalizeInClosedInterval(t *testing.T) { param = &updated err = validation.NormalizeInClosedInterval(¶m, 1, 1, 1) assert.NoError(t, err) + assert.Nil(t, param) assert.Equal(t, int32(1), updated) updated = -1 @@ -58,3 +59,134 @@ func TestNormalizeInClosedInterval(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int32(2), updated) } + +func TestAreAllPointersNilExcept(t *testing.T) { + t.Run("empty", func(t *testing.T) { + assert.True(t, validation.AreAllPointersNilExcept(struct{}{})) + }) + t.Run("no pointers", func(t *testing.T) { + assert.True(t, validation.AreAllPointersNilExcept(struct{ field string }{field: "example_string"})) + }) + t.Run("nil pointer", func(t *testing.T) { + assert.True(t, validation.AreAllPointersNilExcept(struct{ field *string }{field: nil})) + }) + t.Run("not nil pointer", func(t *testing.T) { + field := "example string" + assert.False(t, validation.AreAllPointersNilExcept(struct{ field *string }{field: &field})) + }) + t.Run("multiple mixed pointers", func(t *testing.T) { + fieldone := "example string" + assert.False(t, validation.AreAllPointersNilExcept(struct { + fieldone *string + fieldtwo *string + }{ + fieldone: &fieldone, + fieldtwo: nil, + })) + }) + t.Run("multiple filled pointers", func(t *testing.T) { + fieldone := "example string" + assert.False(t, validation.AreAllPointersNilExcept(struct { + fieldone *string + fieldtwo *string + }{ + fieldone: &fieldone, + fieldtwo: &fieldone, + })) + }) + t.Run("multiple nil pointers", func(t *testing.T) { + assert.True(t, validation.AreAllPointersNilExcept(struct { + fieldone *string + fieldtwo *string + }{ + fieldone: nil, + fieldtwo: nil, + })) + }) + t.Run("multiple pointers with exception", func(t *testing.T) { + fieldone := "example string" + type TestStruct struct { + fieldone *string + fieldtwo *string + } + val := TestStruct{ + fieldone: &fieldone, + fieldtwo: nil, + } + assert.True(t, validation.AreAllPointersNilExcept(val, "fieldone")) + }) + t.Run("multiple pointers with exception", func(t *testing.T) { + fieldone := "example string" + type TestStruct struct { + fieldone *string + fieldtwo *string + fieldthree *string + } + val := TestStruct{ + fieldone: &fieldone, + fieldtwo: &fieldone, + fieldthree: nil, + } + assert.False(t, validation.AreAllPointersNilExcept(val, "fieldone")) + }) + t.Run("invalid field name", func(t *testing.T) { + type TestStruct struct { + fieldone *string + fieldtwo *string + fieldthree *string + } + val := TestStruct{ + fieldone: nil, + fieldtwo: nil, + fieldthree: nil, + } + assert.True(t, validation.AreAllPointersNilExcept(val, "invalid_field")) + }) +} + +func TestGetFieldName(t *testing.T) { + t.Run("empty", func(t *testing.T) { + name, err := validation.GetFieldName(struct{}{}, "") + assert.EqualError(t, err, "field not found") + assert.Empty(t, name) + }) + t.Run("invalid field", func(t *testing.T) { + name, err := validation.GetFieldName(struct{ field string }{field: ""}, "invalid_field") + assert.EqualError(t, err, "field not found") + assert.Empty(t, name) + }) + t.Run("invalid field name", func(t *testing.T) { + type TestStruct struct { + fieldone *string + } + val := TestStruct{ + fieldone: nil, + } + name, err := validation.GetFieldName(val, "invalid_field") + assert.EqualError(t, err, "field not found") + assert.Empty(t, name) + }) + t.Run("valid field is nil", func(t *testing.T) { + type TestStruct struct { + fieldone *string + } + val := TestStruct{ + fieldone: nil, + } + name, err := validation.GetFieldName(val, val.fieldone) + assert.NoError(t, err) + assert.Equal(t, "fieldone", name) + }) + t.Run("valid field is populated", func(t *testing.T) { + type TestStruct struct { + fieldone *string + } + strval := "example_value" + val := TestStruct{ + fieldone: &strval, + } + name, err := validation.GetFieldName(val, val.fieldone) + assert.NoError(t, err) + assert.Equal(t, "fieldone", name) + }) +} diff --git a/scripts/Taskfile.yml b/scripts/Taskfile.yml index c57376bc..b8020c36 100644 --- a/scripts/Taskfile.yml +++ b/scripts/Taskfile.yml @@ -22,7 +22,7 @@ includes: taskfile: openapi.yml compose: dir: "{{.CONTEXT}}" - taskfile: compose.yml + taskfile: local-deployments.yml db: dir: "{{.CONTEXT}}" taskfile: database.yml diff --git a/scripts/compose.yml b/scripts/local-deployments.yml similarity index 92% rename from scripts/compose.yml rename to scripts/local-deployments.yml index 72ec5fe7..c0661f93 100644 --- a/scripts/compose.yml +++ b/scripts/local-deployments.yml @@ -23,11 +23,15 @@ tasks: - docker compose -f {{.TEST_COMPOSE_FILE}} up --no-recreate -d refresh: cmds: + - task docker:build + - task: build - task: down - - task: up + - docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d + - task: init up: cmds: - task docker:build + - task: build - docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d - task: init init: diff --git a/test/process_test.go b/test/process_test.go index 036fea50..d483f3ae 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -10,12 +10,12 @@ import ( documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean" "queryorchestration/internal/test" queryservice "queryorchestration/pkg/queryService" - "regexp" "strings" "testing" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" + awstypes "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/oapi-codegen/runtime/types" "github.com/stretchr/testify/assert" ) @@ -51,12 +51,12 @@ func TestProcess(t *testing.T) { arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner) _, err = cfg.StoreClient.PutBucketNotificationConfiguration(ctx, &s3.PutBucketNotificationConfigurationInput{ Bucket: &bucketName, - NotificationConfiguration: &types.NotificationConfiguration{ - QueueConfigurations: []types.QueueConfiguration{ + NotificationConfiguration: &awstypes.NotificationConfiguration{ + QueueConfigurations: []awstypes.QueueConfiguration{ { QueueArn: &arn, - Events: []types.Event{ - types.EventS3ObjectCreated, + Events: []awstypes.Event{ + awstypes.EventS3ObjectCreated, }, }, }, @@ -67,6 +67,7 @@ func TestProcess(t *testing.T) { doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner) doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner) querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner) + queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner) net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{ Cfg: cfg, @@ -90,6 +91,18 @@ func TestProcess(t *testing.T) { "QUERY_SYNC_URL": querysyncurl, }, }, + { + Name: test.QuerySyncRunner, + Env: map[string]string{ + "QUERY_URL": queryurl, + }, + }, + { + Name: test.QueryRunner, + Env: map[string]string{ + "QUERY_URL": queryurl, + }, + }, }, Services: []*test.ServiceNetworkConfig{ { @@ -121,6 +134,27 @@ func TestProcess(t *testing.T) { }) assert.NoError(t, err) + contextQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ + Type: queryservice.CONTEXTFULL, + }) + assert.NoError(t, err) + jsonQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ + Type: queryservice.JSONEXTRACTOR, + RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id}, + }) + assert.NoError(t, err) + newActiveVersion := int32(2) + _, err = qService.UpdateJobCollectorByJobIdWithResponse(ctx, jobRes.JSON201.Id, queryservice.JobCollectorUpdate{ + ActiveVersion: &newActiveVersion, + Fields: &[]queryservice.JobCollectorField{ + { + Name: "JSON_QUERY", + QueryId: jsonQueryRes.JSON201.Id, + }, + }, + }) + assert.NoError(t, err) + location := fmt.Sprintf("%s/%s/%s", clientRes.JSON201.Id, jobRes.JSON201.Id, "object_name") body := strings.NewReader("hello world") _, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{ @@ -129,7 +163,4 @@ func TestProcess(t *testing.T) { Body: body, }) assert.NoError(t, err) - - resRegex := regexp.MustCompile(`{"id": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"`) - test.AssertMessageBody(t, ctx, cfg, querysyncurl, resRegex) } diff --git a/test/queryService/queryservice_test.go b/test/queryService/queryservice_test.go index 90f2bab8..888d348f 100644 --- a/test/queryService/queryservice_test.go +++ b/test/queryService/queryservice_test.go @@ -30,8 +30,10 @@ func TestQueryService(t *testing.T) { contextID := idRes.JSON201.Id assert.NotEmpty(t, contextID) + jcfg := "{}" idRes, err = client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ - Type: queryservice.JSONEXTRACTOR, + Type: queryservice.JSONEXTRACTOR, + Config: &jcfg, }) assert.NoError(t, err) jsonID := idRes.JSON201.Id @@ -42,7 +44,7 @@ func TestQueryService(t *testing.T) { assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type) assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion) assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion) - assert.Nil(t, queryRes.JSON200.Config) + assert.Equal(t, jcfg, *queryRes.JSON200.Config) assert.Nil(t, queryRes.JSON200.RequiredQueries) queriesRes, err := client.ListQueriesWithResponse(ctx) @@ -65,6 +67,6 @@ func TestQueryService(t *testing.T) { assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type) assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion) assert.Equal(t, int32(2), queryRes.JSON200.LatestVersion) - assert.Nil(t, queryRes.JSON200.Config) + assert.Equal(t, jcfg, *queryRes.JSON200.Config) assert.ElementsMatch(t, []types.UUID{contextID}, *queryRes.JSON200.RequiredQueries) }