From 0ac5ff9e155a4e0264df968914dbdfb61e8eca51 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Wed, 29 Jan 2025 11:52:37 +0000 Subject: [PATCH] Merged in feature/testquery (pull request #39) Test Query * depstextandclean * startedcleaningresult * resulttidyup * roundone * cleaning * unsyncedquery * startedtestsandsimplification * api * querytests * resultprocessortests * unittests * cleanup --- api/queryRunner/queryrunner.go | 8 +- api/queryRunner/queryrunner_test.go | 37 +- api/queryService/controllers.go | 10 +- api/queryService/job_test.go | 9 +- api/queryService/jobcollector.go | 4 +- api/queryService/jobcollector_test.go | 21 +- api/queryService/parse.go | 16 +- api/queryService/parse_test.go | 18 +- api/queryService/query.go | 10 +- api/queryService/query_test.go | 74 +++- cmd/queryRunner/main.go | 18 +- cmd/queryService/main.go | 27 +- .../00000000000100_query_views.up.sql | 9 +- .../00000000000101_collector_views.up.sql | 4 +- database/queries/query.sql | 15 + database/queries/result.sql | 54 ++- .../database/repository/collector_test.go | 19 +- internal/database/repository/models.go | 4 +- internal/database/repository/query.sql.go | 87 +++++ internal/database/repository/query_test.go | 36 +- internal/database/repository/result.sql.go | 188 ++++++++-- internal/database/repository/result_test.go | 327 ++++++++++++++++-- internal/document/clean/service_test.go | 7 + internal/document/sync/service.go | 31 -- internal/document/sync/service_test.go | 13 - internal/document/sync/sync.go | 63 ---- internal/document/sync/sync_test.go | 215 ------------ internal/document/sync/syncprivate_test.go | 60 ---- internal/document/text/service.go | 32 ++ internal/document/text/service_test.go | 31 ++ internal/job/collector/create.go | 4 +- internal/job/collector/create_test.go | 5 +- internal/job/collector/createprivate_test.go | 13 +- internal/job/collector/get.go | 20 ++ internal/job/collector/get_test.go | 4 +- internal/job/collector/parse.go | 14 +- internal/job/collector/parse_test.go | 4 +- internal/job/collector/service.go | 8 +- internal/job/collector/update.go | 35 +- internal/job/collector/update_test.go | 10 +- internal/job/collector/updateprivate_test.go | 25 +- internal/query/create.go | 18 +- internal/query/create_test.go | 30 +- internal/query/createprivate_test.go | 44 +-- internal/query/get.go | 61 +--- internal/query/get_test.go | 87 ++--- internal/query/list.go | 22 +- internal/query/list_test.go | 23 +- internal/query/normalize.go | 4 +- internal/query/normalize_test.go | 8 +- internal/query/parse.go | 65 ++++ internal/query/parse_test.go | 66 +++- internal/query/processor/parse_test.go | 109 ------ internal/query/queue/create.go | 98 ------ internal/query/queue/create_test.go | 143 -------- internal/query/queue/execute.go | 84 ----- internal/query/queue/execute_test.go | 175 ---------- internal/query/queue/executeprivate_test.go | 85 ----- internal/query/queue/result.go | 73 ---- internal/query/queue/resultprivate_test.go | 74 ---- internal/query/queue/service.go | 55 --- internal/query/queue/service_test.go | 135 -------- internal/query/result/get.go | 48 +++ internal/query/result/get_test.go | 64 ++++ internal/query/result/list.go | 52 +++ internal/query/result/list_test.go | 68 ++++ internal/query/result/parse.go | 14 - internal/query/result/parse_test.go | 24 -- internal/query/result/process.go | 70 ++++ internal/query/result/process_test.go | 143 ++++++++ .../query/{ => result}/processor/parse.go | 47 ++- internal/query/result/processor/parse_test.go | 197 +++++++++++ .../query/{ => result}/processor/service.go | 10 +- .../{ => result}/processor/service_test.go | 20 +- internal/query/result/service.go | 33 +- internal/query/result/service_test.go | 25 ++ internal/query/result/set.go | 45 +++ internal/query/result/set_test.go | 67 ++++ internal/query/result/store_test.go | 50 --- internal/query/result/sync.go | 18 + internal/query/result/sync_test.go | 49 +++ internal/query/service.go | 30 +- internal/query/service_test.go | 2 +- internal/query/sync.go | 163 +++++++++ internal/query/sync_test.go | 307 ++++++++++++++++ internal/query/test.go | 48 ++- internal/query/test_test.go | 59 +++- internal/query/types/contextFull/creator.go | 4 +- .../query/types/contextFull/creator_test.go | 6 +- .../query/types/contextFull/process_test.go | 11 +- internal/query/types/contextFull/result.go | 4 + internal/query/types/contextFull/service.go | 5 +- internal/query/types/contextFull/updator.go | 4 +- .../query/types/contextFull/updator_test.go | 8 +- internal/query/types/jsonExtractor/creator.go | 4 +- .../query/types/jsonExtractor/creator_test.go | 6 +- .../query/types/jsonExtractor/process_test.go | 29 +- internal/query/types/jsonExtractor/result.go | 4 + internal/query/types/jsonExtractor/service.go | 5 +- internal/query/types/jsonExtractor/updator.go | 4 +- .../query/types/jsonExtractor/updator_test.go | 8 +- internal/query/update.go | 18 +- internal/query/update_test.go | 6 +- internal/query/updateprivate_test.go | 36 +- internal/test/container.go | 4 +- internal/text_extraction/service.go | 18 - internal/text_extraction/service_test.go | 13 - test/queryRunner/queryrunner_test.go | 4 +- test/queryService/testquery_test.go | 2 + 109 files changed, 2804 insertions(+), 2100 deletions(-) delete mode 100644 internal/document/sync/service.go delete mode 100644 internal/document/sync/service_test.go delete mode 100644 internal/document/sync/sync.go delete mode 100644 internal/document/sync/sync_test.go delete mode 100644 internal/document/sync/syncprivate_test.go create mode 100644 internal/document/text/service.go create mode 100644 internal/document/text/service_test.go create mode 100644 internal/query/parse.go delete mode 100644 internal/query/processor/parse_test.go delete mode 100644 internal/query/queue/create.go delete mode 100644 internal/query/queue/create_test.go delete mode 100644 internal/query/queue/execute.go delete mode 100644 internal/query/queue/execute_test.go delete mode 100644 internal/query/queue/executeprivate_test.go delete mode 100644 internal/query/queue/result.go delete mode 100644 internal/query/queue/resultprivate_test.go delete mode 100644 internal/query/queue/service.go delete mode 100644 internal/query/queue/service_test.go create mode 100644 internal/query/result/get.go create mode 100644 internal/query/result/get_test.go create mode 100644 internal/query/result/list.go create mode 100644 internal/query/result/list_test.go delete mode 100644 internal/query/result/parse.go delete mode 100644 internal/query/result/parse_test.go create mode 100644 internal/query/result/process.go create mode 100644 internal/query/result/process_test.go rename internal/query/{ => result}/processor/parse.go (70%) create mode 100644 internal/query/result/processor/parse_test.go rename internal/query/{ => result}/processor/service.go (86%) rename internal/query/{ => result}/processor/service_test.go (82%) create mode 100644 internal/query/result/service_test.go create mode 100644 internal/query/result/set.go create mode 100644 internal/query/result/set_test.go delete mode 100644 internal/query/result/store_test.go create mode 100644 internal/query/result/sync.go create mode 100644 internal/query/result/sync_test.go create mode 100644 internal/query/sync.go create mode 100644 internal/query/sync_test.go delete mode 100644 internal/text_extraction/service.go delete mode 100644 internal/text_extraction/service_test.go diff --git a/api/queryRunner/queryrunner.go b/api/queryRunner/queryrunner.go index 61a64a12..2e3fa28d 100644 --- a/api/queryRunner/queryrunner.go +++ b/api/queryRunner/queryrunner.go @@ -3,7 +3,7 @@ package controllers import ( "context" "encoding/json" - documentsync "queryorchestration/internal/document/sync" + "queryorchestration/internal/query" "github.com/go-playground/validator/v10" @@ -12,7 +12,7 @@ import ( ) type Services struct { - DocumentSync *documentsync.Service + Query *query.Service } type QueryRunner struct { @@ -32,7 +32,7 @@ type DocumentQueryEvent struct { } func (s QueryRunner) Process(ctx context.Context, req *types.Message) error { - var body documentsync.Document + var body query.Document err := json.Unmarshal([]byte(*req.Body), &body) if err != nil { return err @@ -43,7 +43,7 @@ func (s QueryRunner) Process(ctx context.Context, req *types.Message) error { return err } - err = s.svc.DocumentSync.Sync(ctx, &body) + err = s.svc.Query.Sync(ctx, &body) if err != nil { return err } diff --git a/api/queryRunner/queryrunner_test.go b/api/queryRunner/queryrunner_test.go index 9ca147f4..81118507 100644 --- a/api/queryRunner/queryrunner_test.go +++ b/api/queryRunner/queryrunner_test.go @@ -6,15 +6,15 @@ import ( controllers "queryorchestration/api/queryRunner" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - documentsync "queryorchestration/internal/document/sync" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + "queryorchestration/internal/query/result" "testing" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/go-playground/validator/v10" "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -32,18 +32,18 @@ func TestQueryRunner(t *testing.T) { Pool: pool, } - svc := documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{ - Query: query.New(db), - }), + svc := query.New(db, &query.Services{ + Result: result.New(db), + Text: documenttext.New(), + Collector: collector.New(db, &collector.Services{}), }) runner := controllers.NewQueryRunner(validator.New(), &controllers.Services{ - DocumentSync: svc, + Query: svc, }) assert.NotNil(t, runner) - doc := documentsync.Document{ + doc := query.Document{ ID: uuid.New(), JobID: uuid.New(), CleanVersion: 1, @@ -55,27 +55,10 @@ func TestQueryRunner(t *testing.T) { msg := &types.Message{ Body: &body, } - collectorID := database.MustToDBUUID(uuid.New()) - queryID := database.MustToDBUUID(uuid.New()) - minCleanVersion := int32(1) - minTextVersion := int32(1) - qV := int32(1) - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). + pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)). WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(collectorID, database.MustToDBUUID(doc.JobID), &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}). - AddRow(collectorID, queryID, qV), - ) - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(collectorID). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(collectorID, queryID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}), + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}), ) err = runner.Process(ctx, msg) diff --git a/api/queryService/controllers.go b/api/queryService/controllers.go index a5d9432a..36a89759 100644 --- a/api/queryService/controllers.go +++ b/api/queryService/controllers.go @@ -11,11 +11,11 @@ import ( ) type Services struct { - Export *export.Service - JobCollector *collector.Service - Query *query.Service - Client *client.Service - Job *job.Service + Export *export.Service + Collector *collector.Service + Query *query.Service + Client *client.Service + Job *job.Service } type Controllers struct { diff --git a/api/queryService/job_test.go b/api/queryService/job_test.go index a9cb3de8..88e86d3a 100644 --- a/api/queryService/job_test.go +++ b/api/queryService/job_test.go @@ -10,10 +10,9 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job" "queryorchestration/internal/job/collector" - "queryorchestration/internal/query" - textextraction "queryorchestration/internal/text_extraction" "strings" "testing" @@ -36,12 +35,12 @@ func TestCreateJob(t *testing.T) { Pool: pool, } + extract := documenttext.New() cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ Job: job.New(db, &job.Services{ Collector: collector.New(db, &collector.Services{ - Query: query.New(db), - DocumentClean: documentclean.New(), - TextExtraction: textextraction.New(), + Clean: documentclean.New(), + Text: extract, }), }), }) diff --git a/api/queryService/jobcollector.go b/api/queryService/jobcollector.go index 9c405225..ab697200 100644 --- a/api/queryService/jobcollector.go +++ b/api/queryService/jobcollector.go @@ -11,7 +11,7 @@ import ( ) func (s *Controllers) GetJobCollectorByJobId(ctx echo.Context, jobId types.UUID) error { - coll, err := s.svc.JobCollector.GetByJobID(ctx.Request().Context(), jobId) + coll, err := s.svc.Collector.GetByJobID(ctx.Request().Context(), jobId) if err != nil { return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to get collector: %s", err)) } @@ -50,7 +50,7 @@ func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UU } } - err := s.svc.JobCollector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{ + err := s.svc.Collector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{ JobID: jobId, ActiveVersion: req.ActiveVersion, MinCleanVersion: req.MinimumCleanerVersion, diff --git a/api/queryService/jobcollector_test.go b/api/queryService/jobcollector_test.go index b226808b..8e25b474 100644 --- a/api/queryService/jobcollector_test.go +++ b/api/queryService/jobcollector_test.go @@ -8,9 +8,8 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" - "queryorchestration/internal/query" - textextraction "queryorchestration/internal/text_extraction" "strings" "testing" @@ -46,11 +45,11 @@ func TestUpdateJobCollector(t *testing.T) { rec := httptest.NewRecorder() ctx := e.NewContext(req, rec) + extract := documenttext.New() cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - JobCollector: collector.New(db, &collector.Services{ - Query: query.New(db), - DocumentClean: documentclean.New(), - TextExtraction: textextraction.New(), + Collector: collector.New(db, &collector.Services{ + Clean: documentclean.New(), + Text: extract, }), }) @@ -66,7 +65,7 @@ func TestUpdateJobCollector(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)). WillReturnRows( pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), ¤t.MinCleanVersion, ¤t.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectBeginTx(pgx.TxOptions{}) pool.ExpectExec("name: UpdateCollector :exec").WithArgs(int32(5), int32(2), database.MustToDBUUID(current.ID)). @@ -95,11 +94,13 @@ func TestGetJobCollectorByJobId(t *testing.T) { rec := httptest.NewRecorder() ctx := e.NewContext(req, rec) + extract := documenttext.New() svc := collector.New(db, &collector.Services{ - Query: query.New(db), + Clean: documentclean.New(), + Text: extract, }) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - JobCollector: svc, + Collector: svc, }) coll := collector.Collector{ @@ -112,7 +113,7 @@ func TestGetJobCollectorByJobId(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)). WillReturnRows( pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), &coll.MinCleanVersion, &coll.MinTextVersion, int32(1), int32(2), []byte("")), + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), ) err = cons.GetJobCollectorByJobId(ctx, coll.JobID) diff --git a/api/queryService/parse.go b/api/queryService/parse.go index 1f9cd462..6f01759c 100644 --- a/api/queryService/parse.go +++ b/api/queryService/parse.go @@ -3,7 +3,7 @@ package queryservice import ( "errors" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/google/uuid" ) @@ -44,25 +44,25 @@ func parseQuery(query *query.Query) (*Query, error) { return q, nil } -func parseQueryType(qType queryprocessor.Type) (QueryType, error) { +func parseQueryType(qType resultprocessor.Type) (QueryType, error) { switch qType { - case queryprocessor.TypeJsonExtractor: + case resultprocessor.TypeJsonExtractor: return JSONEXTRACTOR, nil - case queryprocessor.TypeContextFull: + case resultprocessor.TypeContextFull: return CONTEXTFULL, nil default: return "", errors.New("invalid query type") } } -func parseSpecQueryType(qType QueryType) (queryprocessor.Type, error) { +func parseSpecQueryType(qType QueryType) (resultprocessor.Type, error) { switch qType { case JSONEXTRACTOR: - return queryprocessor.TypeJsonExtractor, nil + return resultprocessor.TypeJsonExtractor, nil case CONTEXTFULL: - return queryprocessor.TypeContextFull, nil + return resultprocessor.TypeContextFull, nil default: - return queryprocessor.Type(-1), errors.New("invalid query type") + return resultprocessor.Type(-1), errors.New("invalid query type") } } diff --git a/api/queryService/parse_test.go b/api/queryService/parse_test.go index e1ca7bcd..b2d7428a 100644 --- a/api/queryService/parse_test.go +++ b/api/queryService/parse_test.go @@ -2,7 +2,7 @@ package queryservice import ( "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -15,7 +15,7 @@ func TestParseQueries(t *testing.T) { in := []*query.Query{ { ID: uuid.New(), - Type: queryprocessor.TypeContextFull, + Type: resultprocessor.TypeContextFull, ActiveVersion: 1, LatestVersion: 2, RequiredQueryIDs: &[]uuid.UUID{ @@ -45,7 +45,7 @@ func TestParseQuery(t *testing.T) { cfg := "hey" in := &query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeContextFull, + Type: resultprocessor.TypeContextFull, ActiveVersion: 1, LatestVersion: 2, RequiredQueryIDs: &[]uuid.UUID{ @@ -71,7 +71,7 @@ func TestParseQuery(t *testing.T) { func TestParseQueryMinimal(t *testing.T) { in := &query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeContextFull, + Type: resultprocessor.TypeContextFull, ActiveVersion: 1, LatestVersion: 2, } @@ -87,26 +87,26 @@ func TestParseQueryMinimal(t *testing.T) { } func TestParseQueryType(t *testing.T) { - qt, err := parseQueryType(queryprocessor.TypeContextFull) + qt, err := parseQueryType(resultprocessor.TypeContextFull) assert.Nil(t, err) assert.Equal(t, CONTEXTFULL, qt) - qt, err = parseQueryType(queryprocessor.TypeJsonExtractor) + qt, err = parseQueryType(resultprocessor.TypeJsonExtractor) assert.Nil(t, err) assert.Equal(t, JSONEXTRACTOR, qt) - _, err = parseQueryType(queryprocessor.Type(-1)) + _, err = parseQueryType(resultprocessor.Type(-1)) assert.NotNil(t, err) } func TestParseSpecQueryType(t *testing.T) { qt, err := parseSpecQueryType(CONTEXTFULL) assert.Nil(t, err) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeContextFull), qt) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeContextFull), qt) qt, err = parseSpecQueryType(JSONEXTRACTOR) assert.Nil(t, err) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), qt) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), qt) _, err = parseSpecQueryType("invalid") assert.NotNil(t, err) diff --git a/api/queryService/query.go b/api/queryService/query.go index 2e116504..2c1a67cf 100644 --- a/api/queryService/query.go +++ b/api/queryService/query.go @@ -4,16 +4,14 @@ import ( "fmt" "net/http" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/labstack/echo/v4" "github.com/oapi-codegen/runtime/types" ) func (s *Controllers) ListQueries(ctx echo.Context) error { - filters := query.ListFilters{} - - queries, err := s.svc.Query.List(ctx.Request().Context(), filters) + queries, err := s.svc.Query.List(ctx.Request().Context()) if err != nil { return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to list query: %s", err)) } @@ -53,7 +51,7 @@ func (s *Controllers) CreateQuery(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, "Invalid Query Type") } - id, err := s.svc.Query.Create(ctx.Request().Context(), &queryprocessor.Create{ + id, err := s.svc.Query.Create(ctx.Request().Context(), &resultprocessor.Create{ Type: qt, RequiredQueryIDs: req.RequiredQueries, Config: req.Config, @@ -73,7 +71,7 @@ func (s *Controllers) UpdateQuery(ctx echo.Context, id types.UUID) error { return echo.NewHTTPError(http.StatusBadRequest, err) } - err := s.svc.Query.Update(ctx.Request().Context(), &queryprocessor.Update{ + err := s.svc.Query.Update(ctx.Request().Context(), &resultprocessor.Update{ ActiveVersion: req.ActiveVersion, Config: req.Config, RequiredQueryIDs: req.RequiredQueries, diff --git a/api/queryService/query_test.go b/api/queryService/query_test.go index d830e1b3..31b080c0 100644 --- a/api/queryService/query_test.go +++ b/api/queryService/query_test.go @@ -8,7 +8,12 @@ import ( queryservice "queryorchestration/api/queryService" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/document" + documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + "queryorchestration/internal/query/result" "strings" "testing" @@ -33,7 +38,7 @@ func TestCreateQuery(t *testing.T) { } cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db), + Query: query.New(db, &query.Services{}), }) body := queryservice.QueryCreate{ @@ -75,7 +80,7 @@ func TestListQueries(t *testing.T) { } cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db), + Query: query.New(db, &query.Services{}), }) e := echo.New() @@ -85,7 +90,7 @@ func TestListQueries(t *testing.T) { id := uuid.New() - pool.ExpectQuery("-- name: ListQueries :many").WithArgs().WillReturnRows( + pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(id), repository.QuerytypeContextFull, int32(1), int32(2), []byte(""), []pgtype.UUID{}), ) @@ -120,7 +125,7 @@ func TestGetQuery(t *testing.T) { } cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db), + Query: query.New(db, &query.Services{}), }) e := echo.New() @@ -164,7 +169,7 @@ func TestUpdateQuery(t *testing.T) { } cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db), + Query: query.New(db, &query.Services{}), }) av := int32(2) @@ -211,12 +216,38 @@ func TestTestQuery(t *testing.T) { Pool: pool, } + docsvc := document.New(db) + col := collector.New(db, &collector.Services{ + Clean: documentclean.New(), + Text: documenttext.New(), + }) cons := queryservice.NewControllers(validator.New(), &queryservice.Services{ - Query: query.New(db), + Collector: col, + Query: query.New(db, &query.Services{ + Document: docsvc, + Collector: col, + Result: result.New(db), + }), }) + coll := collector.Collector{ + ID: uuid.New(), + JobID: uuid.New(), + } + doc := document.Document{ + ID: uuid.New(), + JobID: coll.JobID, + Hash: "example_hash", + Location: "example_location", + } + params := &query.Test{ + QueryID: uuid.New(), + DocumentID: doc.ID, + QueryVersion: int32(3), + } body := queryservice.QueryTestRequest{ - DocumentId: uuid.New(), + DocumentId: params.DocumentID, + QueryVersion: params.QueryVersion, } bodyBytes, err := json.Marshal(body) assert.Nil(t, err) @@ -227,9 +258,32 @@ func TestTestQuery(t *testing.T) { rec := httptest.NewRecorder() ctx := e.NewContext(req, rec) - id := uuid.New() + reqID := database.MustToDBUUID(uuid.New()) + pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}). + AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location), + ) + pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + ) + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")), + ) - err = cons.TestQuery(ctx, id) + err = cons.TestQuery(ctx, params.QueryID) assert.Nil(t, err) assert.Equal(t, http.StatusOK, rec.Code) @@ -237,6 +291,6 @@ func TestTestQuery(t *testing.T) { err = json.Unmarshal(rec.Body.Bytes(), &res) assert.Nil(t, err) assert.EqualExportedValues(t, queryservice.QueryTestResponse{ - Value: "", + Value: "old_value", }, res) } diff --git a/cmd/queryRunner/main.go b/cmd/queryRunner/main.go index 03546e91..dac3d2ec 100644 --- a/cmd/queryRunner/main.go +++ b/cmd/queryRunner/main.go @@ -3,9 +3,11 @@ package main import ( "context" controllers "queryorchestration/api/queryRunner" - documentsync "queryorchestration/internal/document/sync" + documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + "queryorchestration/internal/query/result" "queryorchestration/internal/server" "queryorchestration/internal/server/queue" @@ -16,16 +18,22 @@ func main() { ctx := context.Background() queryrunner := func(cfg *server.Config) queue.Controller { - que := query.New(cfg.Database) + extract := documenttext.New() + text := documenttext.New() + clean := documentclean.New() coll := collector.New(cfg.Database, &collector.Services{ - Query: que, + Text: text, + Clean: clean, }) - svc := documentsync.New(cfg.Database, &documentsync.Services{ + res := result.New(cfg.Database) + svc := query.New(cfg.Database, &query.Services{ + Result: res, + Text: extract, Collector: coll, }) return controllers.NewQueryRunner(cfg.Validator, &controllers.Services{ - DocumentSync: svc, + Query: svc, }) } diff --git a/cmd/queryService/main.go b/cmd/queryService/main.go index a92f19b6..e15cab45 100644 --- a/cmd/queryService/main.go +++ b/cmd/queryService/main.go @@ -6,13 +6,14 @@ import ( queryservice "queryorchestration/api/queryService" "queryorchestration/internal/client" documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/export" "queryorchestration/internal/job" "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + "queryorchestration/internal/query/result" "queryorchestration/internal/server" "queryorchestration/internal/server/api" - textextraction "queryorchestration/internal/text_extraction" "github.com/labstack/echo/v4" _ "github.com/lib/pq" @@ -23,13 +24,17 @@ func main() { registerHandlers := func(cfg *server.Config, e *echo.Echo) *api.APIConfig { exp := export.New(cfg.Database) - que := query.New(cfg.Database) - extract := textextraction.New() + extract := documenttext.New() clean := documentclean.New() col := collector.New(cfg.Database, &collector.Services{ - Query: que, - TextExtraction: extract, - DocumentClean: clean, + Text: extract, + Clean: clean, + }) + res := result.New(cfg.Database) + que := query.New(cfg.Database, &query.Services{ + Text: extract, + Result: res, + Collector: col, }) cli := client.New(cfg.Database) jbb := job.New(cfg.Database, &job.Services{ @@ -38,11 +43,11 @@ func main() { }) services := &queryservice.Services{ - Export: exp, - JobCollector: col, - Query: que, - Client: cli, - Job: jbb, + Export: exp, + Collector: col, + Query: que, + Client: cli, + Job: jbb, } cons := queryservice.NewControllers(cfg.Validator, services) diff --git a/database/migrations/00000000000100_query_views.up.sql b/database/migrations/00000000000100_query_views.up.sql index a544ae3e..157fd280 100644 --- a/database/migrations/00000000000100_query_views.up.sql +++ b/database/migrations/00000000000100_query_views.up.sql @@ -1,5 +1,12 @@ CREATE VIEW fullActiveQueries AS - SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds + SELECT DISTINCT + q.id, q.type, q.activeVersion, q.latestVersion, + coalesce(c.config, null) as config, + coalesce( + ARRAY_AGG(r.requiredQueryId) + FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], + array[]::uuid[] + )::uuid[] as requiredIds FROM queries AS q LEFT JOIN queryConfigs AS c ON q.id = c.queryId and q.activeVersion >= c.addedVersion diff --git a/database/migrations/00000000000101_collector_views.up.sql b/database/migrations/00000000000101_collector_views.up.sql index 410b1c0f..b9726495 100644 --- a/database/migrations/00000000000101_collector_views.up.sql +++ b/database/migrations/00000000000101_collector_views.up.sql @@ -1,5 +1,5 @@ CREATE VIEW fullActiveCollectors AS - SELECT DISTINCT c.id, c.jobId, cv.minCleanVersion, cv.minTextVersion, c.activeVersion, c.latestVersion, + SELECT DISTINCT c.id, c.jobId, coalesce(cv.minCleanVersion, 0) as minCleanVersion, coalesce(cv.minTextVersion, 0) as minTextVersion, c.activeVersion, c.latestVersion, jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields FROM collectors AS c LEFT JOIN collectorCodeVersions AS cv ON c.id = cv.collectorId @@ -29,5 +29,5 @@ CREATE VIEW collectorQueryDependencyTree AS FROM fullActiveQueries as q JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds) ) - SELECT collectorId, queryId, type, activeVersion as queryVersion, requiredIds::uuid[] + SELECT collectorId, queryId, type, activeVersion as queryVersion, requiredIds FROM collectorQueryDependencyTree; \ No newline at end of file diff --git a/database/queries/query.sql b/database/queries/query.sql index 4d3dd3bb..82110895 100644 --- a/database/queries/query.sql +++ b/database/queries/query.sql @@ -4,9 +4,24 @@ SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 an -- name: GetQuery :one SELECT * FROM fullActiveQueries WHERE id = $1; +-- name: GetQueryWithVersion :one +SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds + FROM queries AS q + LEFT JOIN queryConfigs AS c ON q.id = c.queryId + and $2 >= c.addedVersion + and $2 < COALESCE(c.removedVersion, $2 + 1) + LEFT JOIN requiredQueries AS r ON q.id = r.queryId + and $2 >= r.addedVersion + and $2 < COALESCE(r.removedVersion, $2 + 1) + WHERE q.id = $1 + GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config; + -- name: ListQueries :many SELECT * FROM fullActiveQueries; +-- name: ListQueriesById :many +SELECT * FROM fullActiveQueries WHERE id = any($1); + -- name: CreateQuery :one INSERT INTO queries (type) VALUES ($1) RETURNING id; diff --git a/database/queries/result.sql b/database/queries/result.sql index 75445662..dc17bc57 100644 --- a/database/queries/result.sql +++ b/database/queries/result.sql @@ -1,9 +1,49 @@ --- name: ListResultsByDocumentID :many -SELECT id, queryId, queryVersion FROM results - where documentId = $1 and cleanVersion >= $2 and textVersion >= $3; - --- name: ListResultValuesByID :many -SELECT id, queryId, value FROM results where id = ANY($1); +-- name: ListQueryRequirementValues :many +WITH latest_versions AS ( + SELECT + r.queryId, + MAX(r.cleanVersion) as max_clean_version, + MAX(r.textVersion) as max_text_version + FROM results r + WHERE r.documentId = $3 + AND r.queryVersion = $2 + GROUP BY r.queryId +) +SELECT rq.requiredQueryId as queryId, r.value, q.type + FROM requiredQueries as rq + JOIN results as r on r.queryId = rq.requiredQueryId + JOIN queries as q on q.id = rq.requiredQueryId + JOIN latest_versions lv ON lv.queryId = r.queryId + WHERE rq.queryId = $1 and r.documentId = $3 + and $2 >= rq.addedVersion + and $2 < COALESCE(rq.removedVersion, $2 + 1) + and r.queryVersion = $2 + and r.cleanVersion >= $4 and r.textVersion >= $5 + AND r.cleanVersion = lv.max_clean_version + AND r.textVersion = lv.max_text_version; -- name: SetResult :one -INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id; \ No newline at end of file +INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id; + +-- name: GetResultValueWithVersion :one +SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5; + +-- name: ListUnsyncedQueriesByDocId :many +WITH RECURSIVE unsyncedQueries AS ( + SELECT dt.queryId, dt.requiredIds + from documents as d + JOIN fullActiveCollectors as c on d.jobId = c.jobId + JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId + LEFT JOIN results as r on r.queryId = dt.queryId + and r.queryVersion = dt.queryVersion + and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion + where d.id = $1 and r.value is null + + UNION + + SELECT DISTINCT dt.queryId, dt.requiredIds + FROM unsyncedQueries as u + JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) +) +SELECT * FROM fullActiveQueries +WHERE id in (SELECT queryId FROM unsyncedQueries); \ No newline at end of file diff --git a/internal/database/repository/collector_test.go b/internal/database/repository/collector_test.go index 041084e2..3606393b 100644 --- a/internal/database/repository/collector_test.go +++ b/internal/database/repository/collector_test.go @@ -10,7 +10,6 @@ import ( "queryorchestration/internal/test" "testing" - "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" ) @@ -51,8 +50,8 @@ func TestCollector(t *testing.T) { assert.EqualExportedValues(t, &repository.Fullactivecollector{ ID: collId, Jobid: jobId, - Mincleanversion: nil, - Mintextversion: nil, + Mincleanversion: 0, + Mintextversion: 0, Activeversion: 1, Latestversion: 1, }, coll) @@ -62,8 +61,8 @@ func TestCollector(t *testing.T) { assert.EqualExportedValues(t, &repository.Fullactivecollector{ ID: collId, Jobid: jobId, - Mincleanversion: nil, - Mintextversion: nil, + Mincleanversion: 0, + Mintextversion: 0, Activeversion: 1, Latestversion: 1, }, coll) @@ -89,8 +88,8 @@ func TestCollector(t *testing.T) { assert.EqualExportedValues(t, &repository.Fullactivecollector{ ID: collId, Jobid: jobId, - Mincleanversion: &minCleanVersion, - Mintextversion: &minTextVersion, + Mincleanversion: minCleanVersion, + Mintextversion: minTextVersion, Activeversion: 1, Latestversion: 1, Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())), @@ -112,7 +111,7 @@ func TestCollector(t *testing.T) { Queryid: contextId, Queryversion: 1, Type: repository.QuerytypeContextFull, - Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, + Requiredids: []pgtype.UUID{}, }, }, qs) @@ -142,8 +141,8 @@ func TestCollector(t *testing.T) { assert.EqualExportedValues(t, &repository.Fullactivecollector{ ID: collId, Jobid: jobId, - Mincleanversion: nil, - Mintextversion: nil, + Mincleanversion: 0, + Mintextversion: 0, Activeversion: 2, Latestversion: 2, Fields: []byte(nil), diff --git a/internal/database/repository/models.go b/internal/database/repository/models.go index fe0ffd19..f6e8e58d 100644 --- a/internal/database/repository/models.go +++ b/internal/database/repository/models.go @@ -111,8 +111,8 @@ type Document struct { type Fullactivecollector struct { ID pgtype.UUID `db:"id"` Jobid pgtype.UUID `db:"jobid"` - Mincleanversion *int32 `db:"mincleanversion"` - Mintextversion *int32 `db:"mintextversion"` + Mincleanversion int32 `db:"mincleanversion"` + Mintextversion int32 `db:"mintextversion"` Activeversion int32 `db:"activeversion"` Latestversion int32 `db:"latestversion"` Fields []byte `db:"fields"` diff --git a/internal/database/repository/query.sql.go b/internal/database/repository/query.sql.go index 470feb29..18ee3b39 100644 --- a/internal/database/repository/query.sql.go +++ b/internal/database/repository/query.sql.go @@ -124,6 +124,59 @@ func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams) return &i, err } +const getQueryWithVersion = `-- name: GetQueryWithVersion :one +SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds + FROM queries AS q + LEFT JOIN queryConfigs AS c ON q.id = c.queryId + and $2 >= c.addedVersion + and $2 < COALESCE(c.removedVersion, $2 + 1) + LEFT JOIN requiredQueries AS r ON q.id = r.queryId + and $2 >= r.addedVersion + and $2 < COALESCE(r.removedVersion, $2 + 1) + WHERE q.id = $1 + GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config +` + +type GetQueryWithVersionParams struct { + ID pgtype.UUID `db:"id"` + Addedversion int32 `db:"addedversion"` +} + +type GetQueryWithVersionRow struct { + ID pgtype.UUID `db:"id"` + Type Querytype `db:"type"` + Activeversion int32 `db:"activeversion"` + Latestversion int32 `db:"latestversion"` + Config []byte `db:"config"` + Requiredids []pgtype.UUID `db:"requiredids"` +} + +// GetQueryWithVersion +// +// SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds +// FROM queries AS q +// LEFT JOIN queryConfigs AS c ON q.id = c.queryId +// and $2 >= c.addedVersion +// and $2 < COALESCE(c.removedVersion, $2 + 1) +// LEFT JOIN requiredQueries AS r ON q.id = r.queryId +// and $2 >= r.addedVersion +// and $2 < COALESCE(r.removedVersion, $2 + 1) +// WHERE q.id = $1 +// GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config +func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) { + row := q.db.QueryRow(ctx, getQueryWithVersion, arg.ID, arg.Addedversion) + var i GetQueryWithVersionRow + err := row.Scan( + &i.ID, + &i.Type, + &i.Activeversion, + &i.Latestversion, + &i.Config, + &i.Requiredids, + ) + return &i, err +} + const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one SELECT EXISTS ( SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1) @@ -181,6 +234,40 @@ func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) { return items, nil } +const listQueriesById = `-- name: ListQueriesById :many +SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1) +` + +// ListQueriesById +// +// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = any($1) +func (q *Queries) ListQueriesById(ctx context.Context, id []pgtype.UUID) ([]*Fullactivequery, error) { + rows, err := q.db.Query(ctx, listQueriesById, id) + if err != nil { + return nil, err + } + defer rows.Close() + items := []*Fullactivequery{} + for rows.Next() { + var i Fullactivequery + if err := rows.Scan( + &i.ID, + &i.Type, + &i.Activeversion, + &i.Latestversion, + &i.Config, + &i.Requiredids, + ); err != nil { + return nil, err + } + items = append(items, &i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const removeQueryConfig = `-- name: RemoveQueryConfig :exec UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null ` diff --git a/internal/database/repository/query_test.go b/internal/database/repository/query_test.go index 263b8c39..820de23e 100644 --- a/internal/database/repository/query_test.go +++ b/internal/database/repository/query_test.go @@ -41,7 +41,7 @@ func TestQueries(t *testing.T) { Activeversion: 1, Latestversion: 1, Config: nil, - Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, + Requiredids: []pgtype.UUID{}, }, jsonQuery) err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ @@ -121,9 +121,23 @@ func TestQueries(t *testing.T) { Activeversion: 2, Latestversion: 2, Config: nil, - Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)}, + Requiredids: []pgtype.UUID{}, }, jsonQuery) + versionedQuery, err := queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{ + ID: jsonQueryID, + Addedversion: 1, + }) + assert.Nil(t, err) + assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{ + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 2, + Latestversion: 2, + Config: jsonConfig, + Requiredids: []pgtype.UUID{contextQueryID}, + }, versionedQuery) + all_exist, err := queries.AllQueriesExist(ctx, []pgtype.UUID{}) assert.Nil(t, err) assert.True(t, all_exist) @@ -240,7 +254,7 @@ func TestQueriesList(t *testing.T) { Activeversion: 1, Latestversion: 1, Config: nil, - Requiredids: []pgtype.UUID{{}}, + Requiredids: []pgtype.UUID{}, }, { ID: contextQueryID, @@ -248,7 +262,21 @@ func TestQueriesList(t *testing.T) { Activeversion: 1, Latestversion: 1, Config: nil, - Requiredids: []pgtype.UUID{{}}, + Requiredids: []pgtype.UUID{}, + }, + }, qs) + + qs, err = queries.ListQueriesById(ctx, []pgtype.UUID{jsonQueryID}) + assert.Nil(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 1, + Latestversion: 1, + Config: nil, + Requiredids: []pgtype.UUID{}, }, }, qs) } diff --git a/internal/database/repository/result.sql.go b/internal/database/repository/result.sql.go index b38b5fc6..b71594df 100644 --- a/internal/database/repository/result.sql.go +++ b/internal/database/repository/result.sql.go @@ -11,29 +11,118 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -const listResultValuesByID = `-- name: ListResultValuesByID :many -SELECT id, queryId, value FROM results where id = ANY($1) +const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one +SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5 ` -type ListResultValuesByIDRow struct { - ID pgtype.UUID `db:"id"` - Queryid pgtype.UUID `db:"queryid"` - Value string `db:"value"` +type GetResultValueWithVersionParams struct { + Queryid pgtype.UUID `db:"queryid"` + Queryversion int32 `db:"queryversion"` + Documentid pgtype.UUID `db:"documentid"` + Cleanversion int32 `db:"cleanversion"` + Textversion int32 `db:"textversion"` } -// ListResultValuesByID +type GetResultValueWithVersionRow struct { + ID pgtype.UUID `db:"id"` + Value string `db:"value"` +} + +// GetResultValueWithVersion // -// SELECT id, queryId, value FROM results where id = ANY($1) -func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([]*ListResultValuesByIDRow, error) { - rows, err := q.db.Query(ctx, listResultValuesByID, id) +// SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5 +func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) { + row := q.db.QueryRow(ctx, getResultValueWithVersion, + arg.Queryid, + arg.Queryversion, + arg.Documentid, + arg.Cleanversion, + arg.Textversion, + ) + var i GetResultValueWithVersionRow + err := row.Scan(&i.ID, &i.Value) + return &i, err +} + +const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many +WITH latest_versions AS ( + SELECT + r.queryId, + MAX(r.cleanVersion) as max_clean_version, + MAX(r.textVersion) as max_text_version + FROM results r + WHERE r.documentId = $3 + AND r.queryVersion = $2 + GROUP BY r.queryId +) +SELECT rq.requiredQueryId as queryId, r.value, q.type + FROM requiredQueries as rq + JOIN results as r on r.queryId = rq.requiredQueryId + JOIN queries as q on q.id = rq.requiredQueryId + JOIN latest_versions lv ON lv.queryId = r.queryId + WHERE rq.queryId = $1 and r.documentId = $3 + and $2 >= rq.addedVersion + and $2 < COALESCE(rq.removedVersion, $2 + 1) + and r.queryVersion = $2 + and r.cleanVersion >= $4 and r.textVersion >= $5 + AND r.cleanVersion = lv.max_clean_version + AND r.textVersion = lv.max_text_version +` + +type ListQueryRequirementValuesParams struct { + Queryid pgtype.UUID `db:"queryid"` + Addedversion int32 `db:"addedversion"` + Documentid pgtype.UUID `db:"documentid"` + Cleanversion int32 `db:"cleanversion"` + Textversion int32 `db:"textversion"` +} + +type ListQueryRequirementValuesRow struct { + Queryid pgtype.UUID `db:"queryid"` + Value string `db:"value"` + Type Querytype `db:"type"` +} + +// ListQueryRequirementValues +// +// WITH latest_versions AS ( +// SELECT +// r.queryId, +// MAX(r.cleanVersion) as max_clean_version, +// MAX(r.textVersion) as max_text_version +// FROM results r +// WHERE r.documentId = $3 +// AND r.queryVersion = $2 +// GROUP BY r.queryId +// ) +// SELECT rq.requiredQueryId as queryId, r.value, q.type +// FROM requiredQueries as rq +// JOIN results as r on r.queryId = rq.requiredQueryId +// JOIN queries as q on q.id = rq.requiredQueryId +// JOIN latest_versions lv ON lv.queryId = r.queryId +// WHERE rq.queryId = $1 and r.documentId = $3 +// and $2 >= rq.addedVersion +// and $2 < COALESCE(rq.removedVersion, $2 + 1) +// and r.queryVersion = $2 +// and r.cleanVersion >= $4 and r.textVersion >= $5 +// AND r.cleanVersion = lv.max_clean_version +// AND r.textVersion = lv.max_text_version +func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) { + rows, err := q.db.Query(ctx, listQueryRequirementValues, + arg.Queryid, + arg.Addedversion, + arg.Documentid, + arg.Cleanversion, + arg.Textversion, + ) if err != nil { return nil, err } defer rows.Close() - items := []*ListResultValuesByIDRow{} + items := []*ListQueryRequirementValuesRow{} for rows.Next() { - var i ListResultValuesByIDRow - if err := rows.Scan(&i.ID, &i.Queryid, &i.Value); err != nil { + var i ListQueryRequirementValuesRow + if err := rows.Scan(&i.Queryid, &i.Value, &i.Type); err != nil { return nil, err } items = append(items, &i) @@ -44,37 +133,64 @@ func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([ return items, nil } -const listResultsByDocumentID = `-- name: ListResultsByDocumentID :many -SELECT id, queryId, queryVersion FROM results - where documentId = $1 and cleanVersion >= $2 and textVersion >= $3 +const listUnsyncedQueriesByDocId = `-- name: ListUnsyncedQueriesByDocId :many +WITH RECURSIVE unsyncedQueries AS ( + SELECT dt.queryId, dt.requiredIds + from documents as d + JOIN fullActiveCollectors as c on d.jobId = c.jobId + JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId + LEFT JOIN results as r on r.queryId = dt.queryId + and r.queryVersion = dt.queryVersion + and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion + where d.id = $1 and r.value is null + + UNION + + SELECT DISTINCT dt.queryId, dt.requiredIds + FROM unsyncedQueries as u + JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) +) +SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries +WHERE id in (SELECT queryId FROM unsyncedQueries) ` -type ListResultsByDocumentIDParams struct { - Documentid pgtype.UUID `db:"documentid"` - Cleanversion int32 `db:"cleanversion"` - Textversion int32 `db:"textversion"` -} - -type ListResultsByDocumentIDRow struct { - ID pgtype.UUID `db:"id"` - Queryid pgtype.UUID `db:"queryid"` - Queryversion int32 `db:"queryversion"` -} - -// ListResultsByDocumentID +// ListUnsyncedQueriesByDocId // -// SELECT id, queryId, queryVersion FROM results -// where documentId = $1 and cleanVersion >= $2 and textVersion >= $3 -func (q *Queries) ListResultsByDocumentID(ctx context.Context, arg *ListResultsByDocumentIDParams) ([]*ListResultsByDocumentIDRow, error) { - rows, err := q.db.Query(ctx, listResultsByDocumentID, arg.Documentid, arg.Cleanversion, arg.Textversion) +// WITH RECURSIVE unsyncedQueries AS ( +// SELECT dt.queryId, dt.requiredIds +// from documents as d +// JOIN fullActiveCollectors as c on d.jobId = c.jobId +// JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId +// LEFT JOIN results as r on r.queryId = dt.queryId +// and r.queryVersion = dt.queryVersion +// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion +// where d.id = $1 and r.value is null +// +// UNION +// +// SELECT DISTINCT dt.queryId, dt.requiredIds +// FROM unsyncedQueries as u +// JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds) +// ) +// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries +// WHERE id in (SELECT queryId FROM unsyncedQueries) +func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID) ([]*Fullactivequery, error) { + rows, err := q.db.Query(ctx, listUnsyncedQueriesByDocId, id) if err != nil { return nil, err } defer rows.Close() - items := []*ListResultsByDocumentIDRow{} + items := []*Fullactivequery{} for rows.Next() { - var i ListResultsByDocumentIDRow - if err := rows.Scan(&i.ID, &i.Queryid, &i.Queryversion); err != nil { + var i Fullactivequery + if err := rows.Scan( + &i.ID, + &i.Type, + &i.Activeversion, + &i.Latestversion, + &i.Config, + &i.Requiredids, + ); err != nil { return nil, err } items = append(items, &i) diff --git a/internal/database/repository/result_test.go b/internal/database/repository/result_test.go index 9ed91dd6..60efd68a 100644 --- a/internal/database/repository/result_test.go +++ b/internal/database/repository/result_test.go @@ -55,29 +55,316 @@ func TestResults(t *testing.T) { assert.Nil(t, err) assert.True(t, jsonResultID.Valid) - resultsByDoc, err := queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{ + res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ + Queryid: jsonQueryID, + Queryversion: jsonQuery.Activeversion, Documentid: documentID, Cleanversion: cleanVersion, Textversion: textVersion, }) assert.Nil(t, err) - assert.Len(t, resultsByDoc, 1) - assert.ElementsMatch(t, []*repository.ListResultsByDocumentIDRow{ - { - ID: jsonResultID, - Queryid: jsonQueryID, - Queryversion: jsonQuery.Activeversion, - }, - }, resultsByDoc) - - results, err := queries.ListResultValuesByID(ctx, []pgtype.UUID{jsonResultID}) - assert.Nil(t, err) - assert.Len(t, results, 1) - assert.ElementsMatch(t, []*repository.ListResultValuesByIDRow{ - { - ID: jsonResultID, - Queryid: jsonQueryID, - Value: jsonResultValue, - }, - }, results) + assert.EqualExportedValues(t, &repository.GetResultValueWithVersionRow{ + ID: jsonResultID, + Value: jsonResultValue, + }, res) +} + +func TestResultValues(t *testing.T) { + ctx := context.Background() + + db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Migrations: &database.MigrationConfig{ + BasePath: path.Join(os.Getenv("PWD"), "../../.."), + }}) + defer cleanup() + + queries := repository.New(db.Pool) + + jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) + assert.Nil(t, err) + + clientId, err := queries.CreateClient(ctx, "example_client") + assert.Nil(t, err) + jobId, err := queries.CreateJob(ctx, clientId) + assert.Nil(t, err) + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash", + }) + assert.Nil(t, err) + + contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) + assert.Nil(t, err) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: jsonQueryID, + Requiredqueryid: contextQueryID, + Addedversion: 1, + }) + assert.Nil(t, err) + + contextQuery, err := queries.GetQuery(ctx, contextQueryID) + assert.Nil(t, err) + + result := repository.SetResultParams{ + Queryid: contextQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: contextQuery.Activeversion, + } + + _, err = queries.SetResult(ctx, &result) + assert.Nil(t, err) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: contextQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: contextQuery.Activeversion - 1, + }) + assert.Nil(t, err) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: contextQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 2, + Textversion: 2, + Queryversion: contextQuery.Activeversion, + }) + assert.Nil(t, err) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: contextQuery.Activeversion, + }) + assert.Nil(t, err) + + qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Addedversion: 1, + Cleanversion: result.Cleanversion, + Textversion: result.Textversion, + }) + assert.Nil(t, err) + assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{ + { + Queryid: contextQueryID, + Type: repository.QuerytypeContextFull, + Value: result.Value, + }, + }, qResults) +} + +func TestUnsyncedQueries(t *testing.T) { + ctx := context.Background() + + db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{ + Migrations: &database.MigrationConfig{ + BasePath: path.Join(os.Getenv("PWD"), "../../.."), + }}) + defer cleanup() + + queries := repository.New(db.Pool) + + clientId, err := queries.CreateClient(ctx, "example_client") + assert.Nil(t, err) + jobId, err := queries.CreateJob(ctx, clientId) + assert.Nil(t, err) + collectorId, err := queries.CreateCollector(ctx, jobId) + assert.Nil(t, err) + documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{ + Jobid: jobId, + Hash: "example_hash", + }) + assert.Nil(t, err) + + contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull)) + assert.Nil(t, err) + jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor)) + assert.Nil(t, err) + err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{ + Queryid: jsonQueryID, + Requiredqueryid: contextQueryID, + Addedversion: 1, + }) + assert.Nil(t, err) + + err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{ + Collectorid: collectorId, + Name: "example_name", + Queryid: jsonQueryID, + Addedversion: 1, + }) + assert.Nil(t, err) + + qs, err := queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 2) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 1, + Latestversion: 1, + Config: nil, + Requiredids: []pgtype.UUID{contextQueryID}, + }, + { + ID: contextQueryID, + Type: repository.QuerytypeContextFull, + Activeversion: 1, + Latestversion: 1, + Config: nil, + Requiredids: []pgtype.UUID{}, + }, + }, qs) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: contextQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: 1, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 1, + Latestversion: 1, + Config: nil, + Requiredids: []pgtype.UUID{contextQueryID}, + }, + }, qs) + + err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: 2, + Activeversion: 2, + ID: contextQueryID, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 2) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 1, + Latestversion: 1, + Config: nil, + Requiredids: []pgtype.UUID{contextQueryID}, + }, + { + ID: contextQueryID, + Type: repository.QuerytypeContextFull, + Activeversion: 2, + Latestversion: 2, + Config: nil, + Requiredids: []pgtype.UUID{}, + }, + }, qs) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: contextQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: 2, + }) + assert.Nil(t, err) + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: 1, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 0) + + err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: 2, + Activeversion: 2, + ID: jsonQueryID, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 1) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 2, + Latestversion: 2, + Config: nil, + Requiredids: []pgtype.UUID{contextQueryID}, + }, + }, qs) + + _, err = queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: jsonQueryID, + Documentid: documentID, + Value: "context_value", + Cleanversion: 1, + Textversion: 2, + Queryversion: 2, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 0) + + err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{ + Latestversion: 3, + Activeversion: 3, + ID: contextQueryID, + }) + assert.Nil(t, err) + + qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID) + assert.Nil(t, err) + assert.Len(t, qs, 2) + assert.ElementsMatch(t, []*repository.Fullactivequery{ + { + ID: jsonQueryID, + Type: repository.QuerytypeJsonExtractor, + Activeversion: 2, + Latestversion: 2, + Config: nil, + Requiredids: []pgtype.UUID{contextQueryID}, + }, + { + ID: contextQueryID, + Type: repository.QuerytypeContextFull, + Activeversion: 3, + Latestversion: 3, + Config: nil, + Requiredids: []pgtype.UUID{}, + }, + }, qs) } diff --git a/internal/document/clean/service_test.go b/internal/document/clean/service_test.go index 36e7cacc..cfe9982b 100644 --- a/internal/document/clean/service_test.go +++ b/internal/document/clean/service_test.go @@ -11,3 +11,10 @@ func TestService(t *testing.T) { svc := documentclean.New() assert.NotNil(t, svc) } + +func TestIsValidVersion(t *testing.T) { + svc := documentclean.New() + + assert.Nil(t, svc.IsValidVersion(2)) + assert.Error(t, svc.IsValidVersion(-1)) +} diff --git a/internal/document/sync/service.go b/internal/document/sync/service.go deleted file mode 100644 index 32891c98..00000000 --- a/internal/document/sync/service.go +++ /dev/null @@ -1,31 +0,0 @@ -package documentsync - -import ( - "queryorchestration/internal/database" - "queryorchestration/internal/job/collector" - - "github.com/google/uuid" -) - -type Document struct { - ID uuid.UUID `json:"id" validate:"required,uuid"` - JobID uuid.UUID `json:"jobId" validate:"required,uuid"` - CleanVersion int32 `json:"cleanVersion" validate:"required,gt=0"` - TextVersion int32 `json:"textVersion" validate:"required,gt=0"` -} - -type Services struct { - Collector *collector.Service -} - -type Service struct { - db *database.Connection - svc *Services -} - -func New(db *database.Connection, svc *Services) *Service { - return &Service{ - db, - svc, - } -} diff --git a/internal/document/sync/service_test.go b/internal/document/sync/service_test.go deleted file mode 100644 index 5b18873e..00000000 --- a/internal/document/sync/service_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package documentsync_test - -import ( - documentsync "queryorchestration/internal/document/sync" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNew(t *testing.T) { - svc := documentsync.New(nil, nil) - assert.NotNil(t, svc) -} diff --git a/internal/document/sync/sync.go b/internal/document/sync/sync.go deleted file mode 100644 index ef6b075a..00000000 --- a/internal/document/sync/sync.go +++ /dev/null @@ -1,63 +0,0 @@ -package documentsync - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - queryqueue "queryorchestration/internal/query/queue" - "queryorchestration/internal/query/result" - - "github.com/google/uuid" -) - -func (s *Service) Sync(ctx context.Context, doc *Document) error { - coll, err := s.svc.Collector.GetByJobID(ctx, doc.JobID) - if err != nil { - return err - } - - results, err := s.getResults(ctx, doc.ID, coll) - if err != nil { - return err - } - - queue, err := queryqueue.New(ctx, &queryqueue.NewConfig{ - DB: s.db, - Collector: coll, - Results: results, - DocumentID: doc.ID, - CleanVersion: doc.CleanVersion, - TextVersion: doc.TextVersion, - }) - if err != nil { - return err - } - - err = queue.Execute(ctx) - if err != nil { - return err - } - - return nil -} - -func (s *Service) getResults(ctx context.Context, id uuid.UUID, coll *collector.Collector) ([]*result.Result, error) { - docID := database.MustToDBUUID(id) - - results, err := s.db.Queries.ListResultsByDocumentID(ctx, &repository.ListResultsByDocumentIDParams{ - Documentid: docID, - Textversion: coll.MinTextVersion, - Cleanversion: coll.MinCleanVersion, - }) - if err != nil { - return nil, err - } - - cleanResults := make([]*result.Result, len(results)) - for index, dbResult := range results { - cleanResults[index] = result.Parse(dbResult) - } - - return cleanResults, nil -} diff --git a/internal/document/sync/sync_test.go b/internal/document/sync/sync_test.go deleted file mode 100644 index 23147552..00000000 --- a/internal/document/sync/sync_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package documentsync_test - -import ( - "context" - "errors" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - documentsync "queryorchestration/internal/document/sync" - "queryorchestration/internal/job/collector" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestSyncIsSynced(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - doc := documentsync.Document{ - ID: uuid.New(), - JobID: uuid.New(), - } - - queryVersion := int32(1) - coll := collector.Collector{ - ID: uuid.New(), - JobID: doc.JobID, - MinCleanVersion: int32(1), - MinTextVersion: int32(2), - } - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), &coll.MinCleanVersion, &coll.MinTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}). - AddRow(pgtype.UUID{}, pgtype.UUID{}, queryVersion), - ) - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(database.MustToDBUUID(coll.ID), pgtype.UUID{}, repository.QuerytypeJsonExtractor, queryVersion, []pgtype.UUID{}), - ) - - docSvc := documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.Nil(t, err) -} - -func TestSyncDBFail(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - doc := documentsync.Document{ - ID: uuid.New(), - JobID: uuid.New(), - } - - dbCollectorId := database.MustToDBUUID(uuid.New()) - dbJobID := database.MustToDBUUID(doc.JobID) - dbQueryID := database.MustToDBUUID(uuid.New()) - minCleanVersion := int32(1) - minTextVersion := int32(1) - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}), - ) - - docSvc := documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.EqualError(t, err, "no rows in result set") - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")), - ) - dbErr := "database failure" - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). - WillReturnError(errors.New(dbErr)) - - docSvc = documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.EqualError(t, err, dbErr) - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}), - ) - dbErr = "database failure" - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId). - WillReturnError(errors.New(dbErr)) - - docSvc = documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.EqualError(t, err, dbErr) - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")), - ) - qV := int32(1) - reqID := database.MustToDBUUID(uuid.New()) - resID := database.MustToDBUUID(uuid.New()) - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}). - AddRow(resID, reqID, qV), - ) - dbErr = "database failure" - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{reqID}). - AddRow(dbCollectorId, reqID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}), - ) - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}). - WillReturnError(errors.New(dbErr)) - - docSvc = documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.EqualError(t, err, dbErr) -} - -func TestSync(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - doc := documentsync.Document{ - ID: uuid.New(), - JobID: uuid.New(), - } - - dbCollectorId := database.MustToDBUUID(uuid.New()) - dbJobID := database.MustToDBUUID(doc.JobID) - dbQueryID := database.MustToDBUUID(uuid.New()) - minCleanVersion := int32(1) - minTextVersion := int32(1) - - pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID). - WillReturnRows( - pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(dbCollectorId, dbJobID, &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")), - ) - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}), - ) - qV := int32(1) - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId). - WillReturnRows( - pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). - AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{}), - ) - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}), - ) - - docSvc := documentsync.New(db, &documentsync.Services{ - Collector: collector.New(db, &collector.Services{}), - }) - err = docSvc.Sync(ctx, &doc) - assert.EqualError(t, err, "JSON Extraction requires 1 result") -} diff --git a/internal/document/sync/syncprivate_test.go b/internal/document/sync/syncprivate_test.go deleted file mode 100644 index 21fd4507..00000000 --- a/internal/document/sync/syncprivate_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package documentsync - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestGetResults(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - doc := Document{ - ID: uuid.New(), - JobID: uuid.New(), - } - - coll := collector.Collector{ - ID: uuid.New(), - JobID: doc.JobID, - MinCleanVersion: int32(1), - MinTextVersion: int32(2), - } - results := []*result.Result{ - { - ID: uuid.New(), - QueryID: uuid.New(), - QueryVersion: 2, - }, - } - - pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}). - AddRow(database.MustToDBUUID(results[0].ID), database.MustToDBUUID(results[0].QueryID), results[0].QueryVersion), - ) - - docSvc := Service{ - db: db, - } - res, err := docSvc.getResults(ctx, doc.ID, &coll) - assert.Nil(t, err) - assert.ElementsMatch(t, results, res) -} diff --git a/internal/document/text/service.go b/internal/document/text/service.go new file mode 100644 index 00000000..6b4f5776 --- /dev/null +++ b/internal/document/text/service.go @@ -0,0 +1,32 @@ +package documenttext + +import ( + "errors" + + "github.com/google/uuid" +) + +type Service struct { +} + +func New() *Service { + return &Service{} +} + +func (s *Service) IsValidVersion(v int32) error { + if v <= 0 { + return errors.New("document clean code version must be > 0") + } + + return nil +} + +type IsExtractedParams struct { + DocumentID uuid.UUID + MinCleanVersion int32 + MinTextVersion int32 +} + +func (s *Service) IsExtracted(params *IsExtractedParams) error { + return nil +} diff --git a/internal/document/text/service_test.go b/internal/document/text/service_test.go new file mode 100644 index 00000000..d631b928 --- /dev/null +++ b/internal/document/text/service_test.go @@ -0,0 +1,31 @@ +package documenttext_test + +import ( + documenttext "queryorchestration/internal/document/text" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestService(t *testing.T) { + svc := documenttext.New() + assert.NotNil(t, svc) +} + +func TestIsValidVersion(t *testing.T) { + svc := documenttext.New() + + assert.Nil(t, svc.IsValidVersion(2)) + assert.Error(t, svc.IsValidVersion(-1)) +} + +func TestIsExtracted(t *testing.T) { + svc := documenttext.New() + + assert.Nil(t, svc.IsExtracted(&documenttext.IsExtractedParams{ + DocumentID: uuid.New(), + MinCleanVersion: int32(1), + MinTextVersion: int32(1), + })) +} diff --git a/internal/job/collector/create.go b/internal/job/collector/create.go index 44c40163..bc3bba75 100644 --- a/internal/job/collector/create.go +++ b/internal/job/collector/create.go @@ -53,7 +53,7 @@ func (f *fields) SetRequiredQueryIDs(v *[]uuid.UUID) { func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*dbCreateParams, error) { minClean := params.MinCleanVersion if minClean != nil { - err := s.svc.DocumentClean.IsValidVersion(*minClean) + err := s.svc.Clean.IsValidVersion(*minClean) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (s *Service) getCreateParams(ctx context.Context, params *CreateParams) (*d minText := params.MinTextVersion if minText != nil { - err := s.svc.TextExtraction.IsValidVersion(*minText) + err := s.svc.Text.IsValidVersion(*minText) if err != nil { return nil, err } diff --git a/internal/job/collector/create_test.go b/internal/job/collector/create_test.go index ffe28f22..2a06abfa 100644 --- a/internal/job/collector/create_test.go +++ b/internal/job/collector/create_test.go @@ -5,7 +5,6 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/job/collector" - "queryorchestration/internal/query" "testing" "github.com/google/uuid" @@ -27,9 +26,7 @@ func TestCreate(t *testing.T) { Pool: pool, } - svc := collector.New(db, &collector.Services{ - Query: query.New(db), - }) + svc := collector.New(db, &collector.Services{}) id := uuid.New() minCleanV := int32(2) diff --git a/internal/job/collector/createprivate_test.go b/internal/job/collector/createprivate_test.go index 649f86c7..2174ea75 100644 --- a/internal/job/collector/createprivate_test.go +++ b/internal/job/collector/createprivate_test.go @@ -4,7 +4,6 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - "queryorchestration/internal/query" "testing" "github.com/google/uuid" @@ -28,10 +27,8 @@ func TestGetCreateParams(t *testing.T) { } svc := Service{ - db: db, - svc: &Services{ - Query: query.New(db), - }, + db: db, + svc: &Services{}, } minCleanV := int32(2) @@ -81,10 +78,8 @@ func TestSubmitCreate(t *testing.T) { } svc := Service{ - db: db, - svc: &Services{ - Query: query.New(db), - }, + db: db, + svc: &Services{}, } minCleanV := int32(2) diff --git a/internal/job/collector/get.go b/internal/job/collector/get.go index 3ebf3120..90948b4f 100644 --- a/internal/job/collector/get.go +++ b/internal/job/collector/get.go @@ -3,6 +3,7 @@ package collector import ( "context" "queryorchestration/internal/database" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/google/uuid" ) @@ -24,3 +25,22 @@ func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, return parseDBCollector(dbColl) } + +func (s *Service) ListQueries(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) { + queries, err := s.db.Queries.ListCollectorQueries(ctx, database.MustToDBUUID(id)) + if err != nil { + return nil, err + } + + cleanQueries := make([]*resultprocessor.Query, len(queries)) + for index, dbQuery := range queries { + cleanQuery, err := resultprocessor.ParseDBCollectorQuery(dbQuery) + if err != nil { + return nil, err + } + + cleanQueries[index] = cleanQuery + } + + return cleanQueries, nil +} diff --git a/internal/job/collector/get_test.go b/internal/job/collector/get_test.go index c5ed884d..ef303844 100644 --- a/internal/job/collector/get_test.go +++ b/internal/job/collector/get_test.go @@ -43,7 +43,7 @@ func TestGet(t *testing.T) { pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.ID)). WillReturnRows( pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), &ogc.MinCleanVersion, &ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), + AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), ) coll, err := svc.Get(ctx, ogc.ID) @@ -81,7 +81,7 @@ func TestGetByJobID(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)). WillReturnRows( pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), &ogc.MinCleanVersion, &ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), + AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))), ) coll, err := svc.GetByJobID(ctx, ogc.JobID) diff --git a/internal/job/collector/parse.go b/internal/job/collector/parse.go index f1a45416..dc5445dd 100644 --- a/internal/job/collector/parse.go +++ b/internal/job/collector/parse.go @@ -25,21 +25,11 @@ func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) { fields = map[string]uuid.UUID{} } - minCleanVersion := int32(0) - if c.Mincleanversion != nil { - minCleanVersion = *c.Mincleanversion - } - - minTextVersion := int32(0) - if c.Mintextversion != nil { - minTextVersion = *c.Mintextversion - } - return &Collector{ ID: database.MustToUUID(c.ID), JobID: database.MustToUUID(c.Jobid), - MinCleanVersion: minCleanVersion, - MinTextVersion: minTextVersion, + MinCleanVersion: c.Mincleanversion, + MinTextVersion: c.Mintextversion, ActiveVersion: c.Activeversion, LatestVersion: c.Latestversion, Fields: fields, diff --git a/internal/job/collector/parse_test.go b/internal/job/collector/parse_test.go index 2d54d9d3..0fbd18e3 100644 --- a/internal/job/collector/parse_test.go +++ b/internal/job/collector/parse_test.go @@ -29,8 +29,8 @@ func TestParseDBCollector(t *testing.T) { c, err = parseDBCollector(&repository.Fullactivecollector{ ID: database.MustToDBUUID(ogc.ID), Jobid: database.MustToDBUUID(ogc.JobID), - Mincleanversion: &ogc.MinCleanVersion, - Mintextversion: &ogc.MinTextVersion, + Mincleanversion: ogc.MinCleanVersion, + Mintextversion: ogc.MinTextVersion, Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])), }) assert.Nil(t, err) diff --git a/internal/job/collector/service.go b/internal/job/collector/service.go index 1f65f06a..a50d6a66 100644 --- a/internal/job/collector/service.go +++ b/internal/job/collector/service.go @@ -3,8 +3,7 @@ package collector import ( "queryorchestration/internal/database" documentclean "queryorchestration/internal/document/clean" - "queryorchestration/internal/query" - textextraction "queryorchestration/internal/text_extraction" + documenttext "queryorchestration/internal/document/text" "github.com/google/uuid" ) @@ -20,9 +19,8 @@ type Collector struct { } type Services struct { - Query *query.Service - DocumentClean *documentclean.Service - TextExtraction *textextraction.Service + Clean *documentclean.Service + Text *documenttext.Service } type Service struct { diff --git a/internal/job/collector/update.go b/internal/job/collector/update.go index 707edb01..085c5600 100644 --- a/internal/job/collector/update.go +++ b/internal/job/collector/update.go @@ -97,7 +97,7 @@ func (s *Service) normalizeCodeVersions(current *Collector, params *UpdateParams if params.MinCleanVersion == nil { params.MinCleanVersion = ¤t.MinCleanVersion } else { - err := s.svc.DocumentClean.IsValidVersion(*params.MinCleanVersion) + err := s.svc.Clean.IsValidVersion(*params.MinCleanVersion) if err != nil { return err } @@ -106,7 +106,7 @@ func (s *Service) normalizeCodeVersions(current *Collector, params *UpdateParams if params.MinTextVersion == nil { params.MinTextVersion = ¤t.MinTextVersion } else { - err := s.svc.TextExtraction.IsValidVersion(*params.MinTextVersion) + err := s.svc.Text.IsValidVersion(*params.MinTextVersion) if err != nil { return err } @@ -139,27 +139,30 @@ func (s *Service) normalizeFieldsToDB(ctx context.Context, ofields *map[string]u return nil, nil } - fids := []uuid.UUID{} - for _, id := range *ofields { - fids = append(fids, id) - } - vfields := fields{ - values: &fids, + dbm := map[string]pgtype.UUID{} + for name, id := range *ofields { + dbm[name] = database.MustToDBUUID(id) } - err := s.svc.Query.NormalizeQueryIDs(ctx, &vfields) - if err != nil { - return nil, err - } else if len(*ofields) != len(*vfields.values) { + dbids := []pgtype.UUID{} + for _, id := range dbm { + dbids = append(dbids, id) + } + + dedup := validation.DeduplicateArray(dbids) + + if len(dedup) != len(dbids) { return nil, errors.New("duplicate output fields") } - fs := map[string]pgtype.UUID{} - for name, id := range *ofields { - fs[name] = database.MustToDBUUID(id) + exist, err := s.db.Queries.AllQueriesExist(ctx, dbids) + if err != nil { + return nil, err + } else if !exist { + return nil, errors.New("not all required ids are present") } - return &fs, nil + return &dbm, nil } func (s *Service) submitUpdate(ctx context.Context, current *Collector, params *dbUpdateParams) error { diff --git a/internal/job/collector/update_test.go b/internal/job/collector/update_test.go index 550f8ad3..3caa0c21 100644 --- a/internal/job/collector/update_test.go +++ b/internal/job/collector/update_test.go @@ -5,9 +5,8 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/job/collector" - "queryorchestration/internal/query" - textextraction "queryorchestration/internal/text_extraction" "testing" "github.com/google/uuid" @@ -30,9 +29,8 @@ func TestUpdate(t *testing.T) { } svc := collector.New(db, &collector.Services{ - Query: query.New(db), - DocumentClean: documentclean.New(), - TextExtraction: textextraction.New(), + Clean: documentclean.New(), + Text: documenttext.New(), }) current := collector.Collector{ @@ -50,7 +48,7 @@ func TestUpdate(t *testing.T) { pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(update.JobID)). WillReturnRows( pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). - AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), ¤t.MinCleanVersion, ¤t.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), + AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")), ) pool.ExpectBeginTx(pgx.TxOptions{}) pool.ExpectExec("name: UpdateCollector :exec").WithArgs(int32(5), int32(2), database.MustToDBUUID(current.ID)). diff --git a/internal/job/collector/updateprivate_test.go b/internal/job/collector/updateprivate_test.go index c67ee0d9..d0fa87be 100644 --- a/internal/job/collector/updateprivate_test.go +++ b/internal/job/collector/updateprivate_test.go @@ -5,8 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" documentclean "queryorchestration/internal/document/clean" - "queryorchestration/internal/query" - textextraction "queryorchestration/internal/text_extraction" + documenttext "queryorchestration/internal/document/text" "testing" "github.com/google/uuid" @@ -30,10 +29,8 @@ func TestGetUpdateParams(t *testing.T) { } svc := Service{ - db: db, - svc: &Services{ - Query: query.New(db), - }, + db: db, + svc: &Services{}, } minCleanV := int32(2) @@ -96,10 +93,8 @@ func TestSubmitUpdate(t *testing.T) { } svc := Service{ - db: db, - svc: &Services{ - Query: query.New(db), - }, + db: db, + svc: &Services{}, } minCleanV := int32(2) @@ -167,10 +162,8 @@ func TestNormalizeFieldsToDB(t *testing.T) { } svc := Service{ - db: db, - svc: &Services{ - Query: query.New(db), - }, + db: db, + svc: &Services{}, } fields := map[string]uuid.UUID{ @@ -239,8 +232,8 @@ func TestNormalizeActiveVersion(t *testing.T) { func TestNormalizeCodeVersions(t *testing.T) { svc := Service{ svc: &Services{ - DocumentClean: documentclean.New(), - TextExtraction: textextraction.New(), + Clean: documentclean.New(), + Text: documenttext.New(), }, } diff --git a/internal/query/create.go b/internal/query/create.go index 3fe61551..91a06cfb 100644 --- a/internal/query/create.go +++ b/internal/query/create.go @@ -5,7 +5,7 @@ import ( "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" @@ -13,7 +13,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -func (s *Service) Create(ctx context.Context, entity *queryprocessor.Create) (uuid.UUID, error) { +func (s *Service) Create(ctx context.Context, entity *resultprocessor.Create) (uuid.UUID, error) { err := s.normalizeCreate(ctx, entity) if err != nil { return uuid.Nil, err @@ -27,7 +27,7 @@ func (s *Service) Create(ctx context.Context, entity *queryprocessor.Create) (uu return id, err } -func (s *Service) normalizeCreate(ctx context.Context, entity *queryprocessor.Create) error { +func (s *Service) normalizeCreate(ctx context.Context, entity *resultprocessor.Create) error { err := s.NormalizeQueryIDs(ctx, entity) if err != nil { return err @@ -51,7 +51,7 @@ func (s *Service) normalizeCreate(ctx context.Context, entity *queryprocessor.Cr return nil } -func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Create) (uuid.UUID, error) { +func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Create) (uuid.UUID, error) { query, err := parseCreateQuery(entity) if err != nil { return uuid.Nil, err @@ -103,11 +103,11 @@ func (s *Service) submitCreate(ctx context.Context, entity *queryprocessor.Creat } -func (s *Service) getCreator(qType queryprocessor.Type) (queryprocessor.Creator, error) { +func (s *Service) getCreator(qType resultprocessor.Type) (resultprocessor.Creator, error) { switch qType { - case queryprocessor.TypeJsonExtractor: + case resultprocessor.TypeJsonExtractor: return jsonextractor.NewCreator(s.db), nil - case queryprocessor.TypeContextFull: + case resultprocessor.TypeContextFull: return contextfull.NewCreator(s.db), nil default: return nil, fmt.Errorf("attempting to process invalid query type") @@ -120,8 +120,8 @@ type createQuery struct { Config *[]byte } -func parseCreateQuery(q *queryprocessor.Create) (*createQuery, error) { - t, err := queryprocessor.ToDBQueryType(q.Type) +func parseCreateQuery(q *resultprocessor.Create) (*createQuery, error) { + t, err := resultprocessor.ToDBQueryType(q.Type) if err != nil { return nil, err } diff --git a/internal/query/create_test.go b/internal/query/create_test.go index 6f26ab0b..304d3b5e 100644 --- a/internal/query/create_test.go +++ b/internal/query/create_test.go @@ -6,7 +6,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -27,24 +27,24 @@ func TestCreate(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) config := "{\"path\":\"example_path\"}" q := query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, Config: &config, } - create := &queryprocessor.Create{ + create := &resultprocessor.Create{ Type: q.Type, RequiredQueryIDs: q.RequiredQueryIDs, Config: q.Config, } - dbType, err := queryprocessor.ToDBQueryType(create.Type) + dbType, err := resultprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) pool.ExpectQuery("name: AllQueriesExist :one").WithArgs(database.MustToDBUUIDArray(*create.RequiredQueryIDs)).WillReturnRows( @@ -81,17 +81,17 @@ func TestCreateMinimal(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) q := query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, } - create := &queryprocessor.Create{ + create := &resultprocessor.Create{ Type: q.Type, } - dbType, err := queryprocessor.ToDBQueryType(create.Type) + dbType, err := resultprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) pool.ExpectBeginTx(pgx.TxOptions{}) @@ -118,17 +118,13 @@ func TestCreateRollback(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) - q := query.Query{ - ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, - } - create := &queryprocessor.Create{ - Type: q.Type, + create := &resultprocessor.Create{ + Type: resultprocessor.TypeJsonExtractor, } - dbType, err := queryprocessor.ToDBQueryType(create.Type) + dbType, err := resultprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) pool.ExpectBeginTx(pgx.TxOptions{}) diff --git a/internal/query/createprivate_test.go b/internal/query/createprivate_test.go index f7744e14..ce9649bd 100644 --- a/internal/query/createprivate_test.go +++ b/internal/query/createprivate_test.go @@ -4,7 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -23,27 +23,27 @@ func TestGetCreator(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) - queryType := queryprocessor.Type(queryprocessor.TypeContextFull) + queryType := resultprocessor.Type(resultprocessor.TypeContextFull) creator, err := svc.getCreator(queryType) assert.Nil(t, err) assert.NotNil(t, creator) - queryType = queryprocessor.Type(queryprocessor.TypeJsonExtractor) + queryType = resultprocessor.Type(resultprocessor.TypeJsonExtractor) creator, err = svc.getCreator(queryType) assert.Nil(t, err) assert.NotNil(t, creator) - queryType = queryprocessor.Type(-1) + queryType = resultprocessor.Type(-1) _, err = svc.getCreator(queryType) assert.NotNil(t, err) } func TestParseCreateQuery(t *testing.T) { cfg := "{\"key\":\"value\"}" - cQuery := &queryprocessor.Create{ - Type: queryprocessor.TypeContextFull, + cQuery := &resultprocessor.Create{ + Type: resultprocessor.TypeContextFull, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, @@ -63,8 +63,8 @@ func TestParseCreateQuery(t *testing.T) { func TestParseCreateQueryInvalidType(t *testing.T) { cfg := "{\"key\":\"value\"}" - cQuery := &queryprocessor.Create{ - Type: queryprocessor.Type(-1), + cQuery := &resultprocessor.Create{ + Type: resultprocessor.Type(-1), RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, @@ -87,24 +87,24 @@ func TestSubmitCreate(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), }, Config: &config, } - create := &queryprocessor.Create{ + create := &resultprocessor.Create{ Type: q.Type, RequiredQueryIDs: q.RequiredQueryIDs, Config: q.Config, } - dbType, err := queryprocessor.ToDBQueryType(create.Type) + dbType, err := resultprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) pool.ExpectBeginTx(pgx.TxOptions{}) @@ -137,17 +137,17 @@ func TestSubmitCreateNoReqsOrConfig(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) q := Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, } - create := &queryprocessor.Create{ + create := &resultprocessor.Create{ Type: q.Type, } - dbType, err := queryprocessor.ToDBQueryType(create.Type) + dbType, err := resultprocessor.ToDBQueryType(create.Type) assert.Nil(t, err) pool.ExpectBeginTx(pgx.TxOptions{}) @@ -174,10 +174,10 @@ func TestNormalizeCreate(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) - create := &queryprocessor.Create{ - Type: queryprocessor.TypeJsonExtractor, + create := &resultprocessor.Create{ + Type: resultprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{}, } @@ -190,8 +190,8 @@ func TestNormalizeCreate(t *testing.T) { err = svc.normalizeCreate(ctx, create) assert.Nil(t, err) - assert.EqualExportedValues(t, queryprocessor.Create{ - Type: queryprocessor.TypeJsonExtractor, + assert.EqualExportedValues(t, resultprocessor.Create{ + Type: resultprocessor.TypeJsonExtractor, Config: nil, RequiredQueryIDs: nil, }, *create) diff --git a/internal/query/get.go b/internal/query/get.go index 6f87ae3f..4e25e01e 100644 --- a/internal/query/get.go +++ b/internal/query/get.go @@ -4,20 +4,32 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/google/uuid" ) type Query struct { ID uuid.UUID - Type queryprocessor.Type + Type resultprocessor.Type ActiveVersion int32 LatestVersion int32 RequiredQueryIDs *[]uuid.UUID Config *string } +func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int32) (*Query, error) { + query, err := s.db.Queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{ + ID: database.MustToDBUUID(id), + Addedversion: version, + }) + if err != nil { + return nil, err + } + + return ParseQueryWithVersion(query) +} + func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) { query, err := s.db.Queries.GetQuery(ctx, database.MustToDBUUID(id)) if err != nil { @@ -26,48 +38,3 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) { return ParseFullActiveQuery(query) } - -func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { - var reqQueryIDs *[]uuid.UUID - if len(q.Requiredids) > 0 { - rQ, err := database.ToUUIDArray(q.Requiredids) - if err != nil { - return nil, err - } - reqQueryIDs = &rQ - } - - qType, err := queryprocessor.ParseDBType(q.Type) - if err != nil { - return nil, err - } - - var scfg *string - if q.Config != nil && string(q.Config) != "" { - s := string(q.Config) - scfg = &s - } - - return &Query{ - ID: database.MustToUUID(q.ID), - ActiveVersion: q.Activeversion, - LatestVersion: q.Latestversion, - Type: qType, - RequiredQueryIDs: reqQueryIDs, - Config: scfg, - }, nil -} - -func ParseFullActiveQueryArray(q []*repository.Fullactivequery) ([]*Query, error) { - queries := make([]*Query, len(q)) - for index, query := range q { - q, err := ParseFullActiveQuery(query) - if err != nil { - return nil, err - } - - queries[index] = q - } - - return queries, nil -} diff --git a/internal/query/get_test.go b/internal/query/get_test.go index a68c4f24..b1f16ecf 100644 --- a/internal/query/get_test.go +++ b/internal/query/get_test.go @@ -5,7 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -25,12 +25,12 @@ func TestGet(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) config := "{\"path\":\"example_path\"}" query := query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), RequiredQueryIDs: &[]uuid.UUID{ @@ -52,60 +52,43 @@ func TestGet(t *testing.T) { assert.EqualExportedValues(t, query, *returnQuery) } -func TestFullActiveQueryEmpty(t *testing.T) { - dbQuery := &repository.Fullactivequery{ - ID: database.MustToDBUUID(uuid.New()), - Type: repository.QuerytypeContextFull, - Activeversion: int32(1), - Latestversion: int32(2), - } +func TestGetWithVersion(t *testing.T) { + ctx := context.Background() - out, err := query.ParseFullActiveQuery(dbQuery) - assert.Nil(t, err) - assert.EqualExportedValues(t, query.Query{ - ID: database.MustToUUID(dbQuery.ID), - Type: queryprocessor.TypeContextFull, + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := query.New(db, &query.Services{}) + + config := "{\"path\":\"example_path\"}" + query := query.Query{ + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, ActiveVersion: int32(1), - LatestVersion: int32(2), - }, *out) -} - -func TestFullActiveQueryWithNullUUID(t *testing.T) { - dbQuery := &repository.Fullactivequery{ - ID: database.MustToDBUUID(uuid.New()), - Type: repository.QuerytypeContextFull, - Activeversion: int32(1), - Latestversion: int32(2), + LatestVersion: int32(3), + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + }, + Config: &config, } - out, err := query.ParseFullActiveQuery(dbQuery) - assert.Nil(t, err) - assert.EqualExportedValues(t, query.Query{ - ID: database.MustToUUID(dbQuery.ID), - Type: queryprocessor.TypeContextFull, - ActiveVersion: int32(1), - LatestVersion: int32(2), - }, *out) -} + version := int32(2) -func TestFullActiveQueryArray(t *testing.T) { - dbQueries := []*repository.Fullactivequery{ - { - ID: database.MustToDBUUID(uuid.New()), - Type: repository.QuerytypeContextFull, - Activeversion: int32(1), - Latestversion: int32(2), - }, - } + dbReqIDs := database.MustToDBUUIDArray(*query.RequiredQueryIDs) - out, err := query.ParseFullActiveQueryArray(dbQueries) + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(query.ID), version).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.ActiveVersion, query.LatestVersion, []byte(config), dbReqIDs), + ) + + returnQuery, err := svc.GetWithVersion(ctx, query.ID, version) assert.Nil(t, err) - assert.EqualExportedValues(t, []*query.Query{ - { - ID: database.MustToUUID(dbQueries[0].ID), - Type: queryprocessor.TypeContextFull, - ActiveVersion: int32(1), - LatestVersion: int32(2), - }, - }, out) + + assert.EqualExportedValues(t, query, *returnQuery) } diff --git a/internal/query/list.go b/internal/query/list.go index e70557d8..5cf4d5f4 100644 --- a/internal/query/list.go +++ b/internal/query/list.go @@ -2,12 +2,12 @@ package query import ( "context" + "queryorchestration/internal/database" + + "github.com/google/uuid" ) -type ListFilters struct { -} - -func (s *Service) List(ctx context.Context, filters ListFilters) ([]*Query, error) { +func (s *Service) List(ctx context.Context) ([]*Query, error) { dbQueries, err := s.db.Queries.ListQueries(ctx) if err != nil { return nil, err @@ -20,3 +20,17 @@ func (s *Service) List(ctx context.Context, filters ListFilters) ([]*Query, erro return queries, nil } + +func (s *Service) ListById(ctx context.Context, ids []uuid.UUID) ([]*Query, error) { + dbQueries, err := s.db.Queries.ListQueriesById(ctx, database.MustToDBUUIDArray(ids)) + if err != nil { + return nil, err + } + + queries, err := ParseFullActiveQueryArray(dbQueries) + if err != nil { + return nil, err + } + + return queries, nil +} diff --git a/internal/query/list_test.go b/internal/query/list_test.go index 9c0c86b9..7a91eb47 100644 --- a/internal/query/list_test.go +++ b/internal/query/list_test.go @@ -5,10 +5,11 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -25,12 +26,12 @@ func TestList(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) config := "{\"path\":\"example_path\"}" q := &query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), RequiredQueryIDs: &[]uuid.UUID{ @@ -41,20 +42,18 @@ func TestList(t *testing.T) { dbReqIDs := database.MustToDBUUIDArray(*q.RequiredQueryIDs) - filters := query.ListFilters{} - pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(q.ID), repository.QuerytypeJsonExtractor, q.ActiveVersion, q.LatestVersion, []byte(config), dbReqIDs), ) - resList, err := svc.List(ctx, filters) + resList, err := svc.List(ctx) assert.Nil(t, err) assert.EqualExportedValues(t, []*query.Query{q}, resList) } -func TestListFilterType(t *testing.T) { +func TestListById(t *testing.T) { ctx := context.Background() pool, err := pgxmock.NewPool() @@ -66,12 +65,12 @@ func TestListFilterType(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) config := "{\"path\":\"example_path\"}" q := &query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(1), RequiredQueryIDs: &[]uuid.UUID{ @@ -82,14 +81,12 @@ func TestListFilterType(t *testing.T) { dbReqIDs := database.MustToDBUUIDArray(*q.RequiredQueryIDs) - filters := query.ListFilters{} - - pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows( + pool.ExpectQuery("name: ListQueriesById :many").WithArgs([]pgtype.UUID{database.MustToDBUUID(q.ID)}).WillReturnRows( pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). AddRow(database.MustToDBUUID(q.ID), repository.QuerytypeJsonExtractor, q.ActiveVersion, q.LatestVersion, []byte(config), dbReqIDs), ) - resList, err := svc.List(ctx, filters) + resList, err := svc.ListById(ctx, []uuid.UUID{q.ID}) assert.Nil(t, err) assert.EqualExportedValues(t, []*query.Query{q}, resList) diff --git a/internal/query/normalize.go b/internal/query/normalize.go index 837c72b5..10d0cfbf 100644 --- a/internal/query/normalize.go +++ b/internal/query/normalize.go @@ -6,7 +6,7 @@ import ( "errors" "fmt" "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "queryorchestration/internal/server/validation" "strings" @@ -83,7 +83,7 @@ func (s *Service) NormalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) e return nil } -func (s *Service) normalizeActiveVersion(current *Query, entity *queryprocessor.Update) error { +func (s *Service) normalizeActiveVersion(current *Query, entity *resultprocessor.Update) error { if current == nil { return errors.New("current query required") } diff --git a/internal/query/normalize_test.go b/internal/query/normalize_test.go index d4f0d168..8e5d444c 100644 --- a/internal/query/normalize_test.go +++ b/internal/query/normalize_test.go @@ -5,7 +5,7 @@ import ( "errors" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -19,7 +19,7 @@ func TestNormalizeConfig(t *testing.T) { err := s.normalizeConfig(nil) assert.Nil(t, err) - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} entity.Config = nil err = s.normalizeConfig(&entity) @@ -84,7 +84,7 @@ func TestNormalizeQueryIDs(t *testing.T) { err = s.NormalizeQueryIDs(ctx, nil) assert.Nil(t, err) - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} entity.RequiredQueryIDs = nil err = s.NormalizeQueryIDs(ctx, &entity) @@ -150,7 +150,7 @@ func TestNormalizeActiveVersion(t *testing.T) { ActiveVersion: 2, LatestVersion: 4, } - entity := queryprocessor.Update{} + entity := resultprocessor.Update{} err = s.normalizeActiveVersion(nil, &entity) assert.Error(t, err) diff --git a/internal/query/parse.go b/internal/query/parse.go new file mode 100644 index 00000000..a84bce80 --- /dev/null +++ b/internal/query/parse.go @@ -0,0 +1,65 @@ +package query + +import ( + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + + "github.com/google/uuid" +) + +func ParseQueryWithVersion(q *repository.GetQueryWithVersionRow) (*Query, error) { + return ParseFullActiveQuery(&repository.Fullactivequery{ + ID: q.ID, + Type: q.Type, + Activeversion: q.Activeversion, + Latestversion: q.Latestversion, + Config: q.Config, + Requiredids: q.Requiredids, + }) +} + +func ParseFullActiveQuery(q *repository.Fullactivequery) (*Query, error) { + var reqQueryIDs *[]uuid.UUID + if len(q.Requiredids) > 0 { + rQ, err := database.ToUUIDArray(q.Requiredids) + if err != nil { + return nil, err + } + reqQueryIDs = &rQ + } + + qType, err := resultprocessor.ParseDBType(q.Type) + if err != nil { + return nil, err + } + + var scfg *string + if q.Config != nil && string(q.Config) != "" { + s := string(q.Config) + scfg = &s + } + + return &Query{ + ID: database.MustToUUID(q.ID), + ActiveVersion: q.Activeversion, + LatestVersion: q.Latestversion, + Type: qType, + RequiredQueryIDs: reqQueryIDs, + Config: scfg, + }, nil +} + +func ParseFullActiveQueryArray(q []*repository.Fullactivequery) ([]*Query, error) { + queries := make([]*Query, len(q)) + for index, query := range q { + q, err := ParseFullActiveQuery(query) + if err != nil { + return nil, err + } + + queries[index] = q + } + + return queries, nil +} diff --git a/internal/query/parse_test.go b/internal/query/parse_test.go index 7f76faeb..b8afcf46 100644 --- a/internal/query/parse_test.go +++ b/internal/query/parse_test.go @@ -4,7 +4,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -16,7 +16,7 @@ func TestParseQuery(t *testing.T) { cfg := "example" q := &query.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, ActiveVersion: int32(1), LatestVersion: int32(2), RequiredQueryIDs: &[]uuid.UUID{ @@ -26,7 +26,7 @@ func TestParseQuery(t *testing.T) { } out := query.ParseQuery(q) - assert.EqualExportedValues(t, queryprocessor.Query{ + assert.EqualExportedValues(t, resultprocessor.Query{ ID: q.ID, Type: q.Type, Version: q.ActiveVersion, @@ -52,7 +52,7 @@ func TestParseFullActiveQuery(t *testing.T) { bcfg := string(q.Config) assert.EqualExportedValues(t, query.Query{ ID: database.MustToUUID(q.ID), - Type: queryprocessor.TypeContextFull, + Type: resultprocessor.TypeContextFull, ActiveVersion: q.Activeversion, LatestVersion: q.Latestversion, RequiredQueryIDs: &[]uuid.UUID{ @@ -61,3 +61,61 @@ func TestParseFullActiveQuery(t *testing.T) { Config: &bcfg, }, *out) } + +func TestFullActiveQueryEmpty(t *testing.T) { + dbQuery := &repository.Fullactivequery{ + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: int32(1), + Latestversion: int32(2), + } + + out, err := query.ParseFullActiveQuery(dbQuery) + assert.Nil(t, err) + assert.EqualExportedValues(t, query.Query{ + ID: database.MustToUUID(dbQuery.ID), + Type: resultprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), + }, *out) +} + +func TestFullActiveQueryWithNullUUID(t *testing.T) { + dbQuery := &repository.Fullactivequery{ + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: int32(1), + Latestversion: int32(2), + } + + out, err := query.ParseFullActiveQuery(dbQuery) + assert.Nil(t, err) + assert.EqualExportedValues(t, query.Query{ + ID: database.MustToUUID(dbQuery.ID), + Type: resultprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), + }, *out) +} + +func TestFullActiveQueryArray(t *testing.T) { + dbQueries := []*repository.Fullactivequery{ + { + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: int32(1), + Latestversion: int32(2), + }, + } + + out, err := query.ParseFullActiveQueryArray(dbQueries) + assert.Nil(t, err) + assert.EqualExportedValues(t, []*query.Query{ + { + ID: database.MustToUUID(dbQueries[0].ID), + Type: resultprocessor.TypeContextFull, + ActiveVersion: int32(1), + LatestVersion: int32(2), + }, + }, out) +} diff --git a/internal/query/processor/parse_test.go b/internal/query/processor/parse_test.go deleted file mode 100644 index 714172f8..00000000 --- a/internal/query/processor/parse_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package queryprocessor_test - -import ( - "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/stretchr/testify/assert" -) - -func TestParseDBCollectorQuery(t *testing.T) { - dbResult := repository.Collectorquerydependencytree{ - Collectorid: pgtype.UUID{}, - Queryid: pgtype.UUID{}, - Requiredids: []pgtype.UUID{}, - Type: repository.QuerytypeJsonExtractor, - Queryversion: 1, - } - value, err := queryprocessor.ParseDBCollectorQuery(&dbResult) - assert.Nil(t, err) - assert.Equal(t, uuid.Nil, value.ID) - assert.Nil(t, value.RequiredQueryIDs) - assert.Equal(t, int32(1), value.Version) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value.Type) - - dbResult.Type = repository.Querytype("") - _, err = queryprocessor.ParseDBCollectorQuery(&dbResult) - assert.EqualError(t, err, "invalid database query type") -} - -func TestParseDBNullType(t *testing.T) { - qType := repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor} - value, err := queryprocessor.ParseDBNullType(qType) - assert.Nil(t, err) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value) - - qType = repository.NullQuerytype{} - _, err = queryprocessor.ParseDBNullType(qType) - assert.EqualError(t, err, "invalid database query type") - - qType = repository.NullQuerytype{Valid: true} - _, err = queryprocessor.ParseDBNullType(qType) - assert.EqualError(t, err, "invalid database query type") -} - -func TestParseDBType(t *testing.T) { - qType := repository.QuerytypeJsonExtractor - value, err := queryprocessor.ParseDBType(qType) - assert.Nil(t, err) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value) - - qType = repository.QuerytypeContextFull - value, err = queryprocessor.ParseDBType(qType) - assert.Nil(t, err) - assert.Equal(t, queryprocessor.Type(queryprocessor.TypeContextFull), value) -} - -func TestToDBQueryType(t *testing.T) { - dbQueryType := queryprocessor.Type(queryprocessor.TypeJsonExtractor) - value, err := queryprocessor.ToDBQueryType(dbQueryType) - assert.Nil(t, err) - assert.Equal(t, repository.Querytype(repository.QuerytypeJsonExtractor), value) - - dbQueryType = queryprocessor.Type(-1) - _, err = queryprocessor.ToDBQueryType(dbQueryType) - assert.EqualError(t, err, "invalid database query type") - - dbQueryType = queryprocessor.Type(queryprocessor.TypeContextFull) - value, err = queryprocessor.ToDBQueryType(dbQueryType) - assert.Nil(t, err) - assert.Equal(t, repository.Querytype(repository.QuerytypeContextFull), value) -} - -func TestToDBQueryTypeArray(t *testing.T) { - inArr := []queryprocessor.Type{ - queryprocessor.TypeJsonExtractor, - queryprocessor.TypeContextFull, - } - value, err := queryprocessor.ToDBQueryTypeArray(inArr) - assert.Nil(t, err) - assert.Equal(t, []repository.Querytype{ - repository.QuerytypeJsonExtractor, - repository.QuerytypeContextFull, - }, value) - - inArr = []queryprocessor.Type{ - queryprocessor.Type(-1), - } - _, err = queryprocessor.ToDBQueryTypeArray(inArr) - assert.EqualError(t, err, "invalid database query type") -} - -func TestToDBNullQueryType(t *testing.T) { - dbQueryType := queryprocessor.Type(queryprocessor.TypeJsonExtractor) - value, err := queryprocessor.ToDBNullQueryType(dbQueryType) - assert.Nil(t, err) - assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor}, value) - - dbQueryType = queryprocessor.Type(-1) - _, err = queryprocessor.ToDBNullQueryType(dbQueryType) - assert.EqualError(t, err, "invalid database query type") - - dbQueryType = queryprocessor.Type(queryprocessor.TypeContextFull) - value, err = queryprocessor.ToDBNullQueryType(dbQueryType) - assert.Nil(t, err) - assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeContextFull}, value) -} diff --git a/internal/query/queue/create.go b/internal/query/queue/create.go deleted file mode 100644 index 029287fa..00000000 --- a/internal/query/queue/create.go +++ /dev/null @@ -1,98 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" -) - -func (q *Queue) getUnsyncedQueries() { - for _, query := range q.collectorQueries { - if q.isQuerySynced(query) { - continue - } - - q.Add(query) - } -} - -func (q *Queue) isQuerySynced(query *queryprocessor.Query) bool { - for _, result := range q.results { - if result.QueryID == query.ID && result.QueryVersion == query.Version { - return true - } - } - - return false -} - -func (c *Queue) getCollectorQueries(ctx context.Context) error { - if c.collectorQueries != nil { - return nil - } - - id := database.MustToDBUUID(c.collector.ID) - - queries, err := c.db.Queries.ListCollectorQueries(ctx, id) - if err != nil { - return err - } - - cleanQueries := make([]*queryprocessor.Query, len(queries)) - for index, dbQuery := range queries { - cleanQuery, err := queryprocessor.ParseDBCollectorQuery(dbQuery) - if err != nil { - return err - } - - cleanQueries[index] = cleanQuery - } - - c.collectorQueries = cleanQueries - - return nil -} - -func (q *Queue) Add(qu *queryprocessor.Query) { - dependentQueries := []*queryprocessor.Query{} - requiredIndex := -1 - - if q.unsyncedQueue == nil { - q.unsyncedQueue = []*queryprocessor.Query{} - } else { - for index, entry := range q.unsyncedQueue { - if entry.ID == qu.ID { - return - } - if qu.RequiredQueryIDs != nil { - for _, id := range *qu.RequiredQueryIDs { - if entry.ID == id { - requiredIndex = index - break - } - } - } - } - } - - for _, entry := range q.collectorQueries { - if entry.RequiredQueryIDs != nil { - for _, id := range *entry.RequiredQueryIDs { - if qu.ID == id { - dependentQueries = append(dependentQueries, entry) - break - } - } - } - } - - if requiredIndex != -1 { - q.unsyncedQueue = append((q.unsyncedQueue)[:requiredIndex+1], append([]*queryprocessor.Query{qu}, (q.unsyncedQueue)[requiredIndex+1:]...)...) - } else { - q.unsyncedQueue = append([]*queryprocessor.Query{qu}, q.unsyncedQueue...) - } - - for _, entry := range dependentQueries { - q.Add(entry) - } -} diff --git a/internal/query/queue/create_test.go b/internal/query/queue/create_test.go deleted file mode 100644 index e9c2c1f5..00000000 --- a/internal/query/queue/create_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestGetUnsyncedQueries(t *testing.T) { - queryOne := &queryprocessor.Query{ - ID: uuid.New(), - Version: int32(1), - } - svc := Queue{ - collectorQueries: []*queryprocessor.Query{ - queryOne, - }, - results: []*result.Result{ - {ID: uuid.New(), QueryID: queryOne.ID, QueryVersion: queryOne.Version}, - }, - } - - svc.getUnsyncedQueries() - assert.EqualExportedValues(t, []*queryprocessor.Query(nil), svc.unsyncedQueue) - - svc.results = []*result.Result{} - svc.unsyncedQueue = []*queryprocessor.Query{} - - svc.getUnsyncedQueries() - assert.EqualExportedValues(t, []*queryprocessor.Query{queryOne}, svc.unsyncedQueue) -} - -func TestGetCollectorQueries(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - svc := Queue{ - db: db, - collector: &collector.Collector{ - ID: uuid.New(), - }, - } - dbCollectorID := database.MustToDBUUID(svc.collector.ID) - - collectorQueries := []*queryprocessor.Query{ - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)}, - } - - rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) - for _, q := range collectorQueries { - dbID := database.MustToDBUUID(q.ID) - dbReqIDs := []pgtype.UUID{} - if q.RequiredQueryIDs != nil { - dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) - } - ty, err := queryprocessor.ToDBQueryType(q.Type) - assert.Nil(t, err) - rows = rows. - AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs) - } - - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows) - - err = svc.getCollectorQueries(ctx) - assert.Nil(t, err) - assert.EqualExportedValues(t, collectorQueries, svc.collectorQueries) -} - -func TestIsQuerySynced(t *testing.T) { - query := &queryprocessor.Query{ - ID: uuid.New(), - Version: int32(1), - } - svc := Queue{ - results: []*result.Result{ - {QueryID: query.ID, QueryVersion: query.Version}, - }, - } - - isSynced := svc.isQuerySynced(query) - assert.True(t, isSynced) -} - -func TestIsQuerySyncedNoResult(t *testing.T) { - query := &queryprocessor.Query{ - ID: uuid.New(), - Version: int32(1), - } - svc := Queue{ - results: []*result.Result{}, - } - - isSynced := svc.isQuerySynced(query) - assert.False(t, isSynced) -} - -func TestIsQuerySyncedOldResult(t *testing.T) { - query := &queryprocessor.Query{ - ID: uuid.New(), - Version: int32(1), - } - svc := Queue{ - results: []*result.Result{ - {QueryID: query.ID, QueryVersion: query.Version - 1}, - }, - } - - isSynced := svc.isQuerySynced(query) - assert.False(t, isSynced) -} - -func TestIsQuerySyncedNewResult(t *testing.T) { - query := &queryprocessor.Query{ - ID: uuid.New(), - Version: int32(1), - } - svc := Queue{ - results: []*result.Result{ - {QueryID: query.ID, QueryVersion: query.Version + 1}, - }, - } - - isSynced := svc.isQuerySynced(query) - assert.False(t, isSynced) -} diff --git a/internal/query/queue/execute.go b/internal/query/queue/execute.go deleted file mode 100644 index 6f09f591..00000000 --- a/internal/query/queue/execute.go +++ /dev/null @@ -1,84 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" -) - -func (q *Queue) Execute(ctx context.Context) error { - if q.unsyncedQueue == nil { - return nil - } - - for _, query := range q.unsyncedQueue { - err := q.executeQuery(ctx, query) - if err != nil { - return err - } - } - - q.unsyncedQueue = nil - - return nil -} - -func (q *Queue) executeQuery(ctx context.Context, qu *queryprocessor.Query) error { - values, err := q.getRequiredResults(ctx, qu.RequiredQueryIDs) - if err != nil { - return err - } - - err = q.setResult(ctx, qu, values) - if err != nil { - return err - } - - return nil -} - -func (q *Queue) getRequiredResults(ctx context.Context, requiredQueryIDs *[]uuid.UUID) (*[]result.Value, error) { - var values *[]result.Value - if requiredQueryIDs != nil { - resultIDs := make([]pgtype.UUID, len(*requiredQueryIDs)) - for index, id := range *requiredQueryIDs { - var queryVersion int32 - for _, entry := range q.collectorQueries { - if entry.ID == id { - queryVersion = entry.Version - break - } - } - - for _, entry := range q.results { - if entry.QueryID == id && entry.QueryVersion == queryVersion { - resultIDs[index] = database.MustToDBUUID(entry.ID) - break - } - } - } - - resultValues, err := q.db.Queries.ListResultValuesByID(ctx, resultIDs) - if err != nil { - return nil, err - } - - rValues := make([]result.Value, len(resultValues)) - for index, r := range resultValues { - cleanValue, err := q.getResultValue(r) - if err != nil { - return nil, err - } - - rValues[index] = cleanValue - } - - values = &rValues - } - - return values, nil -} diff --git a/internal/query/queue/execute_test.go b/internal/query/queue/execute_test.go deleted file mode 100644 index 7099ad1f..00000000 --- a/internal/query/queue/execute_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package queryqueue_test - -import ( - "context" - "fmt" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - queryprocessor "queryorchestration/internal/query/processor" - queryqueue "queryorchestration/internal/query/queue" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestExecute(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - coll := collector.Collector{ - ID: uuid.New(), - } - dbCollectorID := database.MustToDBUUID(coll.ID) - - queryOneID := uuid.New() - queryOneVersion := int32(1) - queryTwoID := uuid.New() - queryTwoVersion := int32(2) - queryThreeID := uuid.New() - queryThreeVersion := int32(3) - queryFourID := uuid.New() - queryFourVersion := int32(4) - queryFiveID := uuid.New() - queryFiveVersion := int32(5) - querySixID := uuid.New() - querySixVersion := int32(6) - contextID := uuid.New() - contextVersion := int32(1) - collectorQueries := []queryprocessor.Query{ - {ID: contextID, Type: queryprocessor.TypeContextFull, Version: contextVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryFourVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, - } - - rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) - for _, q := range collectorQueries { - dbID := database.MustToDBUUID(q.ID) - dbReqIDs := []pgtype.UUID{} - if q.RequiredQueryIDs != nil { - dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) - } - ty, err := queryprocessor.ToDBQueryType(q.Type) - assert.Nil(t, err) - rows = rows. - AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs) - } - - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows) - - contextResultID := uuid.New() - results := []*result.Result{ - {ID: contextResultID, QueryID: contextID, QueryVersion: contextVersion}, - {ID: uuid.New(), QueryID: queryFourID, QueryVersion: queryFourVersion}, - {ID: uuid.New(), QueryID: querySixID, QueryVersion: querySixVersion - 1}, - {ID: uuid.New(), QueryID: queryOneID, QueryVersion: queryOneVersion - 1}, - } - - expectedQueries := []*queryprocessor.Query{ - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, - } - - docID := uuid.New() - cleanVersion := int32(1) - textVersion := int32(1) - - q, err := queryqueue.New(ctx, &queryqueue.NewConfig{ - db, &coll, results, docID, cleanVersion, textVersion, - }) - assert.Nil(t, err) - assert.Equal(t, expectedQueries, q.GetQueue()) - - keyLayerOne := "key" - keyLayerTwo := "key5" - valueLayerTwo := "value" - valueLayerOne := fmt.Sprintf("{\"%s\":\"%s\"}", keyLayerTwo, valueLayerTwo) - valueContext := fmt.Sprintf("{\"%s\":%s}", keyLayerOne, valueLayerOne) - - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{database.MustToDBUUID(contextResultID)}).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}). - AddRow(database.MustToDBUUID(contextResultID), database.MustToDBUUID(contextID), valueContext), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(querySixID), querySixVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(fmt.Sprintf("{\"path\":\"%s\"}", keyLayerOne))), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(querySixID), database.MustToDBUUID(docID), valueLayerOne, cleanVersion, textVersion, querySixVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs(pgxmock.AnyArg()).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}). - AddRow(pgtype.UUID{}, database.MustToDBUUID(querySixID), valueLayerOne), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(queryFiveID), queryFiveVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(fmt.Sprintf("{\"path\":\"%s\"}", keyLayerTwo))), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(queryFiveID), database.MustToDBUUID(docID), valueLayerTwo, cleanVersion, textVersion, queryFiveVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs(pgxmock.AnyArg()).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}). - AddRow(database.MustToDBUUID(contextResultID), database.MustToDBUUID(contextID), valueContext), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(queryOneID), queryOneVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(fmt.Sprintf("{\"path\":\"%s\"}", keyLayerOne))), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(queryOneID), database.MustToDBUUID(docID), valueLayerOne, cleanVersion, textVersion, queryOneVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs(pgxmock.AnyArg()).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}). - AddRow(pgtype.UUID{}, database.MustToDBUUID(queryOneID), valueLayerOne), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(queryThreeID), queryThreeVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(fmt.Sprintf("{\"path\":\"%s\"}", keyLayerTwo))), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(queryThreeID), database.MustToDBUUID(docID), valueLayerTwo, cleanVersion, textVersion, queryThreeVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs(pgxmock.AnyArg()).WillReturnRows( - pgxmock.NewRows([]string{"id", "queryId", "value"}). - AddRow(pgtype.UUID{}, database.MustToDBUUID(queryOneID), valueLayerOne), - ) - pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(queryTwoID), queryTwoVersion).WillReturnRows( - pgxmock.NewRows([]string{"id", "config"}). - AddRow(pgtype.UUID{}, []byte(fmt.Sprintf("{\"path\":\"%s\"}", keyLayerTwo))), - ) - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(queryTwoID), database.MustToDBUUID(docID), valueLayerTwo, cleanVersion, textVersion, queryTwoVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - err = q.Execute(ctx) - assert.Nil(t, err) -} diff --git a/internal/query/queue/executeprivate_test.go b/internal/query/queue/executeprivate_test.go deleted file mode 100644 index f2a291ce..00000000 --- a/internal/query/queue/executeprivate_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestExecute(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - expectedQueries := []*queryprocessor.Query{ - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)}, - {ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(2)}, - } - - q := &Queue{ - unsyncedQueue: expectedQueries, - db: db, - documentId: uuid.New(), - cleanVersion: int32(1), - textVersion: int32(2), - } - - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(expectedQueries[0].ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, expectedQueries[0].Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(expectedQueries[1].ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, expectedQueries[1].Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - err = q.Execute(ctx) - assert.Nil(t, err) -} - -func TestExecuteQuery(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)} - - q := &Queue{ - db: db, - documentId: uuid.New(), - cleanVersion: int32(1), - textVersion: int32(2), - } - - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qu.ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, qu.Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - err = q.executeQuery(ctx, qu) - assert.Nil(t, err) -} diff --git a/internal/query/queue/result.go b/internal/query/queue/result.go deleted file mode 100644 index 0906f4f2..00000000 --- a/internal/query/queue/result.go +++ /dev/null @@ -1,73 +0,0 @@ -package queryqueue - -import ( - "context" - "fmt" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" - contextfull "queryorchestration/internal/query/types/contextFull" - jsonextractor "queryorchestration/internal/query/types/jsonExtractor" -) - -func (q *Queue) setResult(ctx context.Context, qu *queryprocessor.Query, resultValues *[]result.Value) error { - processor, err := q.getProcessor(qu.Type) - if err != nil { - return err - } - - value, err := processor.Process(ctx, qu, resultValues) - if err != nil { - return err - } - - id, err := result.Store(ctx, q.db.Queries, &result.ResultStore{ - QueryID: qu.ID, - DocumentID: q.documentId, - Value: value, - CleanVersion: q.cleanVersion, - TextVersion: q.textVersion, - QueryVersion: qu.Version, - }) - if err != nil { - return err - } - - q.results = append(q.results, &result.Result{ - ID: id, - QueryID: qu.ID, - QueryVersion: qu.Version, - }) - - return nil -} - -func (q *Queue) getProcessor(queryType queryprocessor.Type) (queryprocessor.Processor, error) { - switch queryType { - case queryprocessor.TypeJsonExtractor: - return jsonextractor.NewExtractor(q.db), nil - case queryprocessor.TypeContextFull: - return contextfull.NewExtractor(), nil - default: - return nil, fmt.Errorf("attempting to process invalid query type") - } -} - -func (q *Queue) getResultValue(res *repository.ListResultValuesByIDRow) (result.Value, error) { - var queryType queryprocessor.Type - for _, qu := range q.collectorQueries { - if qu.ID == database.MustToUUID(res.Queryid) { - queryType = qu.Type - } - } - - switch queryType { - case queryprocessor.TypeJsonExtractor: - return jsonextractor.NewResult(res.Value), nil - case queryprocessor.TypeContextFull: - return contextfull.NewResult(res.Value), nil - default: - return nil, fmt.Errorf("attempting to process invalid query type") - } -} diff --git a/internal/query/queue/resultprivate_test.go b/internal/query/queue/resultprivate_test.go deleted file mode 100644 index 7c200781..00000000 --- a/internal/query/queue/resultprivate_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestSetResult(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - q := &Queue{ - db: db, - documentId: uuid.New(), - cleanVersion: int32(1), - textVersion: int32(2), - } - - qu := &queryprocessor.Query{ID: uuid.New(), Type: queryprocessor.TypeContextFull, Version: int32(1)} - resultValues := []result.Value{} - - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qu.ID), database.MustToDBUUID(q.documentId), pgxmock.AnyArg(), q.cleanVersion, q.textVersion, qu.Version). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - err = q.setResult(ctx, qu, &resultValues) - assert.Nil(t, err) -} - -func TestGetProcessor(t *testing.T) { - q := Queue{} - - qType := queryprocessor.Type(queryprocessor.TypeContextFull) - - processor, err := q.getProcessor(qType) - assert.Nil(t, err) - assert.NotNil(t, processor) -} - -func TestGetResultValue(t *testing.T) { - result := &repository.ListResultValuesByIDRow{ - Queryid: database.MustToDBUUID(uuid.New()), - Value: "EXAMPLE_VALUE", - } - - q := &Queue{ - collectorQueries: []*queryprocessor.Query{ - {ID: database.MustToUUID(result.Queryid), Type: queryprocessor.TypeContextFull}, - }, - } - - value, err := q.getResultValue(result) - assert.Nil(t, err) - assert.NotNil(t, value) -} diff --git a/internal/query/queue/service.go b/internal/query/queue/service.go deleted file mode 100644 index 715803e6..00000000 --- a/internal/query/queue/service.go +++ /dev/null @@ -1,55 +0,0 @@ -package queryqueue - -import ( - "context" - "queryorchestration/internal/database" - "queryorchestration/internal/job/collector" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" - - "github.com/google/uuid" -) - -type Queue struct { - unsyncedQueue []*queryprocessor.Query - collectorQueries []*queryprocessor.Query - results []*result.Result - collector *collector.Collector - db *database.Connection - documentId uuid.UUID - cleanVersion int32 - textVersion int32 -} - -type NewConfig struct { - DB *database.Connection - Collector *collector.Collector - Results []*result.Result - DocumentID uuid.UUID - CleanVersion int32 - TextVersion int32 -} - -func New(ctx context.Context, cfg *NewConfig) (*Queue, error) { - queue := Queue{ - db: cfg.DB, - results: cfg.Results, - collector: cfg.Collector, - documentId: cfg.DocumentID, - cleanVersion: cfg.CleanVersion, - textVersion: cfg.TextVersion, - } - - err := queue.getCollectorQueries(ctx) - if err != nil { - return nil, err - } - - queue.getUnsyncedQueries() - - return &queue, nil -} - -func (q *Queue) GetQueue() []*queryprocessor.Query { - return q.unsyncedQueue -} diff --git a/internal/query/queue/service_test.go b/internal/query/queue/service_test.go deleted file mode 100644 index f7b10555..00000000 --- a/internal/query/queue/service_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package queryqueue_test - -import ( - "context" - "errors" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/job/collector" - queryprocessor "queryorchestration/internal/query/processor" - queryqueue "queryorchestration/internal/query/queue" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - - coll := collector.Collector{ - ID: uuid.New(), - } - dbCollectorID := database.MustToDBUUID(coll.ID) - - queryOneID := uuid.New() - queryOneVersion := int32(1) - queryTwoID := uuid.New() - queryTwoVersion := int32(2) - queryThreeID := uuid.New() - queryThreeVersion := int32(3) - queryFourID := uuid.New() - queryFourVersion := int32(4) - queryFiveID := uuid.New() - queryFiveVersion := int32(5) - querySixID := uuid.New() - querySixVersion := int32(6) - contextID := uuid.New() - contextVersion := int32(1) - collectorQueries := []queryprocessor.Query{ - {ID: contextID, Type: queryprocessor.TypeContextFull, Version: contextVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryFourID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryFourVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, - } - - rows := pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}) - for _, q := range collectorQueries { - dbID := database.MustToDBUUID(q.ID) - dbReqIDs := []pgtype.UUID{} - if q.RequiredQueryIDs != nil { - dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs) - } - ty, err := queryprocessor.ToDBQueryType(q.Type) - assert.Nil(t, err) - rows = rows. - AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs) - } - - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows) - - contextResultID := uuid.New() - results := []*result.Result{ - {ID: contextResultID, QueryID: contextID, QueryVersion: contextVersion}, - {ID: uuid.New(), QueryID: queryFourID, QueryVersion: queryFourVersion}, - {ID: uuid.New(), QueryID: querySixID, QueryVersion: querySixVersion - 1}, - {ID: uuid.New(), QueryID: queryOneID, QueryVersion: queryOneVersion - 1}, - } - - expectedQueries := []*queryprocessor.Query{ - {ID: querySixID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: querySixVersion}, - {ID: queryFiveID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{querySixID}, Version: queryFiveVersion}, - {ID: queryOneID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{contextID}, Version: queryOneVersion}, - {ID: queryThreeID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryThreeVersion}, - {ID: queryTwoID, Type: queryprocessor.TypeJsonExtractor, RequiredQueryIDs: &[]uuid.UUID{queryOneID}, Version: queryTwoVersion}, - } - - docID := uuid.New() - cleanVersion := int32(1) - textVersion := int32(1) - - q, err := queryqueue.New(ctx, &queryqueue.NewConfig{ - db, &coll, results, docID, cleanVersion, textVersion, - }) - assert.Nil(t, err) - assert.Equal(t, expectedQueries, q.GetQueue()) -} - -func TestQueueFail(t *testing.T) { - ctx := context.Background() - - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - db := &database.Connection{ - Queries: queries, - Pool: pool, - } - coll := collector.Collector{ - ID: uuid.New(), - } - dbCollectorID := database.MustToDBUUID(coll.ID) - - dbErr := "database failure" - pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID). - WillReturnError(errors.New(dbErr)) - - results := []*result.Result{} - - docID := uuid.New() - cleanVersion := int32(1) - textVersion := int32(1) - - _, err = queryqueue.New(ctx, &queryqueue.NewConfig{ - db, &coll, results, docID, cleanVersion, textVersion, - }) - assert.EqualError(t, err, dbErr) -} diff --git a/internal/query/result/get.go b/internal/query/result/get.go new file mode 100644 index 00000000..07831b45 --- /dev/null +++ b/internal/query/result/get.go @@ -0,0 +1,48 @@ +package result + +import ( + "context" + "fmt" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + contextfull "queryorchestration/internal/query/types/contextFull" + jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + + "github.com/google/uuid" +) + +type GetValueWithVersionParams struct { + Type resultprocessor.Type + QueryID uuid.UUID + DocumentID uuid.UUID + QueryVersion int32 + MinCleanVersion int32 + MinTextVersion int32 +} + +func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) { + res, err := s.db.Queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{ + Queryid: database.MustToDBUUID(params.QueryID), + Queryversion: params.QueryVersion, + Documentid: database.MustToDBUUID(params.DocumentID), + Cleanversion: params.MinCleanVersion, + Textversion: params.MinTextVersion, + }) + if err != nil { + return nil, err + } + + return getValueByType(params.Type, res.Value) +} + +func getValueByType(t resultprocessor.Type, value string) (resultprocessor.Value, error) { + switch t { + case resultprocessor.TypeJsonExtractor: + return jsonextractor.NewResult(value), nil + case resultprocessor.TypeContextFull: + return contextfull.NewResult(value), nil + default: + return nil, fmt.Errorf("attempting to process invalid query type") + } +} diff --git a/internal/query/result/get_test.go b/internal/query/result/get_test.go new file mode 100644 index 00000000..d6fb8b51 --- /dev/null +++ b/internal/query/result/get_test.go @@ -0,0 +1,64 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestGetValueByType(t *testing.T) { + _, err := getValueByType(resultprocessor.Type(-1), "example_val") + assert.Error(t, err) + pro, err := getValueByType(resultprocessor.TypeContextFull, "example_context") + assert.Nil(t, err) + assert.NotNil(t, pro) + assert.Equal(t, "example_context", pro.GetStoreValue()) + pro, err = getValueByType(resultprocessor.TypeJsonExtractor, "example_json") + assert.Nil(t, err) + assert.NotNil(t, pro) + assert.Equal(t, "example_json", pro.GetStoreValue()) +} + +func TestGetValueWithVersion(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := New(db) + + params := &GetValueWithVersionParams{ + Type: resultprocessor.TypeJsonExtractor, + QueryID: uuid.New(), + DocumentID: uuid.New(), + QueryVersion: 1, + MinCleanVersion: 2, + MinTextVersion: 3, + } + + pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"id", "value"}). + AddRow(pgtype.UUID{}, "example_value"), + ) + + val, err := svc.GetValueWithVersion(ctx, params) + assert.Nil(t, err) + v := jsonextractor.NewResult("example_value") + assert.Equal(t, v, val) + assert.Equal(t, "example_value", v.GetStoreValue()) +} diff --git a/internal/query/result/list.go b/internal/query/result/list.go new file mode 100644 index 00000000..1908dada --- /dev/null +++ b/internal/query/result/list.go @@ -0,0 +1,52 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + + "github.com/google/uuid" +) + +type ListQueryRequirementValuesParams struct { + QueryID uuid.UUID + DocumentID uuid.UUID + QueryVersion int32 + MinCleanVersion int32 + MinTextVersion int32 +} + +func (s *Service) ListQueryRequirementValues(ctx context.Context, params *ListQueryRequirementValuesParams) (*[]resultprocessor.Value, error) { + qResults, err := s.db.Queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{ + Queryid: database.MustToDBUUID(params.QueryID), + Documentid: database.MustToDBUUID(params.DocumentID), + Addedversion: params.QueryVersion, + Cleanversion: params.MinCleanVersion, + Textversion: params.MinTextVersion, + }) + if err != nil { + return nil, err + } + + return parseQueryRequirementValueArray(qResults) +} + +func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesRow) (*[]resultprocessor.Value, error) { + values := make([]resultprocessor.Value, len(v)) + for index, r := range v { + qType, err := resultprocessor.ParseDBType(r.Type) + if err != nil { + return nil, err + } + + cleanValue, err := getValueByType(qType, r.Value) + if err != nil { + return nil, err + } + + values[index] = cleanValue + } + + return &values, nil +} diff --git a/internal/query/result/list_test.go b/internal/query/result/list_test.go new file mode 100644 index 00000000..705cc3ce --- /dev/null +++ b/internal/query/result/list_test.go @@ -0,0 +1,68 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestListQueryRequirementValues(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := Service{ + db: db, + } + + params := &ListQueryRequirementValuesParams{ + QueryID: uuid.New(), + QueryVersion: 1, + DocumentID: uuid.New(), + MinCleanVersion: 2, + MinTextVersion: 3, + } + + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(pgtype.UUID{}, "example_value", repository.QuerytypeJsonExtractor), + ) + + val, err := svc.ListQueryRequirementValues(ctx, params) + assert.Nil(t, err) + assert.ElementsMatch(t, []resultprocessor.Value{ + jsonextractor.NewResult("example_value"), + }, *val) +} + +func TestParseQueryRequirementValueArray(t *testing.T) { + in := []*repository.ListQueryRequirementValuesRow{ + { + Queryid: database.MustToDBUUID(uuid.New()), + Value: "example_value", + Type: repository.QuerytypeJsonExtractor, + }, + } + + out, err := parseQueryRequirementValueArray(in) + assert.Nil(t, err) + assert.ElementsMatch(t, []resultprocessor.Value{ + jsonextractor.NewResult("example_value"), + }, *out) +} diff --git a/internal/query/result/parse.go b/internal/query/result/parse.go deleted file mode 100644 index 1ed70d7f..00000000 --- a/internal/query/result/parse.go +++ /dev/null @@ -1,14 +0,0 @@ -package result - -import ( - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" -) - -func Parse(dbQuery *repository.ListResultsByDocumentIDRow) *Result { - return &Result{ - ID: database.MustToUUID(dbQuery.ID), - QueryID: database.MustToUUID(dbQuery.Queryid), - QueryVersion: dbQuery.Queryversion, - } -} diff --git a/internal/query/result/parse_test.go b/internal/query/result/parse_test.go deleted file mode 100644 index 18edfd98..00000000 --- a/internal/query/result/parse_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package result_test - -import ( - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/stretchr/testify/assert" -) - -func TestParseResultValue(t *testing.T) { - dbResult := repository.ListResultsByDocumentIDRow{ - ID: pgtype.UUID{}, - Queryid: pgtype.UUID{}, - Queryversion: int32(1), - } - - value := result.Parse(&dbResult) - assert.Equal(t, dbResult.Queryversion, value.QueryVersion) - assert.Equal(t, uuid.Nil, value.ID) - assert.Equal(t, uuid.Nil, value.QueryID) -} diff --git a/internal/query/result/process.go b/internal/query/result/process.go new file mode 100644 index 00000000..49438a4a --- /dev/null +++ b/internal/query/result/process.go @@ -0,0 +1,70 @@ +package result + +import ( + "context" + "errors" + "fmt" + resultprocessor "queryorchestration/internal/query/result/processor" + contextfull "queryorchestration/internal/query/types/contextFull" + jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + + "github.com/google/uuid" +) + +type Process struct { + DocumentID uuid.UUID + MinCleanVersion int32 + MinTextVersion int32 + Query *resultprocessor.Query +} + +func (s *Service) Process(ctx context.Context, p *Process) (resultprocessor.Value, error) { + values, err := s.listRequiredValues(ctx, p) + if err != nil { + return nil, err + } + + processor, err := s.getProcessor(p.Query.Type) + if err != nil { + return nil, err + } + + val, err := processor.Process(ctx, p.Query, values) + if err != nil { + return nil, err + } + + return getValueByType(p.Query.Type, val) +} + +func (s *Service) listRequiredValues(ctx context.Context, p *Process) (*[]resultprocessor.Value, error) { + if p == nil || p.Query == nil || p.Query.RequiredQueryIDs == nil || len(*p.Query.RequiredQueryIDs) == 0 { + return nil, nil + } + + values, err := s.ListQueryRequirementValues(ctx, &ListQueryRequirementValuesParams{ + QueryID: p.Query.ID, + QueryVersion: p.Query.Version, + DocumentID: p.DocumentID, + MinCleanVersion: p.MinCleanVersion, + MinTextVersion: p.MinTextVersion, + }) + if err != nil { + return nil, err + } else if values == nil || len(*values) != len(*p.Query.RequiredQueryIDs) { + return nil, errors.New("document not synced") + } + + return values, nil +} + +func (s *Service) getProcessor(queryType resultprocessor.Type) (resultprocessor.Processor, error) { + switch queryType { + case resultprocessor.TypeJsonExtractor: + return jsonextractor.NewExtractor(s.db), nil + case resultprocessor.TypeContextFull: + return contextfull.NewExtractor(), nil + default: + return nil, fmt.Errorf("attempting to process invalid query type") + } +} diff --git a/internal/query/result/process_test.go b/internal/query/result/process_test.go new file mode 100644 index 00000000..e749326d --- /dev/null +++ b/internal/query/result/process_test.go @@ -0,0 +1,143 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + jsonextractor "queryorchestration/internal/query/types/jsonExtractor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestProcess(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + + svc := Service{ + db: db, + } + + cfg := "{\"path\":\"examplekey\"}" + params := Process{ + DocumentID: uuid.New(), + MinCleanVersion: 1, + MinTextVersion: 3, + Query: &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + Config: &cfg, + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + }, + }, + } + + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "{\"examplekey\":\"example_value\"}", repository.QuerytypeContextFull), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte(cfg)), + ) + + val, err := svc.Process(ctx, ¶ms) + assert.Nil(t, err) + assert.NotNil(t, val) + assert.Equal(t, "example_value", val.GetStoreValue()) +} + +func TestListRequiredValue(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + + svc := Service{ + db: db, + } + + pr, err := svc.listRequiredValues(ctx, nil) + assert.Nil(t, err) + assert.Nil(t, pr) + + pr, err = svc.listRequiredValues(ctx, &Process{}) + assert.Nil(t, err) + assert.Nil(t, pr) + + pr, err = svc.listRequiredValues(ctx, &Process{ + Query: &resultprocessor.Query{}, + }) + assert.Nil(t, err) + assert.Nil(t, pr) + + pr, err = svc.listRequiredValues(ctx, &Process{ + Query: &resultprocessor.Query{ + RequiredQueryIDs: &[]uuid.UUID{}, + }, + }) + assert.Nil(t, err) + assert.Nil(t, pr) + + params := Process{ + DocumentID: uuid.New(), + MinCleanVersion: 1, + MinTextVersion: 3, + Query: &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + }, + }, + } + + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "example_value", repository.QuerytypeJsonExtractor), + ) + + pr, err = svc.listRequiredValues(ctx, ¶ms) + assert.Nil(t, err) + assert.ElementsMatch(t, []resultprocessor.Value{ + jsonextractor.NewResult("example_value"), + }, *pr) +} + +func TestGetProcessor(t *testing.T) { + svc := Service{} + + pr, err := svc.getProcessor(resultprocessor.TypeJsonExtractor) + assert.Nil(t, err) + assert.NotNil(t, pr) + + pr, err = svc.getProcessor(resultprocessor.TypeContextFull) + assert.Nil(t, err) + assert.NotNil(t, pr) + + _, err = svc.getProcessor(resultprocessor.Type(-1)) + assert.Error(t, err) +} diff --git a/internal/query/processor/parse.go b/internal/query/result/processor/parse.go similarity index 70% rename from internal/query/processor/parse.go rename to internal/query/result/processor/parse.go index 887f64b8..4cd73166 100644 --- a/internal/query/processor/parse.go +++ b/internal/query/result/processor/parse.go @@ -1,4 +1,4 @@ -package queryprocessor +package resultprocessor import ( "fmt" @@ -84,3 +84,48 @@ func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, RequiredQueryIDs: reqQueryIDs, }, nil } + +func ParseFullQuery(qs *repository.Fullactivequery) (*Query, error) { + if qs == nil { + return nil, nil + } + + qt, err := ParseDBType(qs.Type) + if err != nil { + return nil, err + } + + var rids *[]uuid.UUID + if len(qs.Requiredids) > 0 { + r := database.MustToUUIDArray(qs.Requiredids) + rids = &r + } + + var cfg *string + if qs.Config != nil { + c := string(qs.Config) + cfg = &c + } + + return &Query{ + ID: database.MustToUUID(qs.ID), + Type: qt, + Version: qs.Activeversion, + RequiredQueryIDs: rids, + Config: cfg, + }, nil +} + +func ParseFullQueryArray(qs []*repository.Fullactivequery) ([]*Query, error) { + parsed := make([]*Query, len(qs)) + for i, q := range qs { + p, err := ParseFullQuery(q) + if err != nil { + return nil, err + } + + parsed[i] = p + } + + return parsed, nil +} diff --git a/internal/query/result/processor/parse_test.go b/internal/query/result/processor/parse_test.go new file mode 100644 index 00000000..0e9495aa --- /dev/null +++ b/internal/query/result/processor/parse_test.go @@ -0,0 +1,197 @@ +package resultprocessor_test + +import ( + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/stretchr/testify/assert" +) + +func TestParseDBCollectorQuery(t *testing.T) { + dbResult := repository.Collectorquerydependencytree{ + Collectorid: pgtype.UUID{}, + Queryid: pgtype.UUID{}, + Requiredids: []pgtype.UUID{}, + Type: repository.QuerytypeJsonExtractor, + Queryversion: 1, + } + value, err := resultprocessor.ParseDBCollectorQuery(&dbResult) + assert.Nil(t, err) + assert.Equal(t, uuid.Nil, value.ID) + assert.Nil(t, value.RequiredQueryIDs) + assert.Equal(t, int32(1), value.Version) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value.Type) + + dbResult.Type = repository.Querytype("") + _, err = resultprocessor.ParseDBCollectorQuery(&dbResult) + assert.EqualError(t, err, "invalid database query type") +} + +func TestParseDBNullType(t *testing.T) { + qType := repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor} + value, err := resultprocessor.ParseDBNullType(qType) + assert.Nil(t, err) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value) + + qType = repository.NullQuerytype{} + _, err = resultprocessor.ParseDBNullType(qType) + assert.EqualError(t, err, "invalid database query type") + + qType = repository.NullQuerytype{Valid: true} + _, err = resultprocessor.ParseDBNullType(qType) + assert.EqualError(t, err, "invalid database query type") +} + +func TestParseDBType(t *testing.T) { + qType := repository.QuerytypeJsonExtractor + value, err := resultprocessor.ParseDBType(qType) + assert.Nil(t, err) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), value) + + qType = repository.QuerytypeContextFull + value, err = resultprocessor.ParseDBType(qType) + assert.Nil(t, err) + assert.Equal(t, resultprocessor.Type(resultprocessor.TypeContextFull), value) +} + +func TestToDBQueryType(t *testing.T) { + dbQueryType := resultprocessor.Type(resultprocessor.TypeJsonExtractor) + value, err := resultprocessor.ToDBQueryType(dbQueryType) + assert.Nil(t, err) + assert.Equal(t, repository.Querytype(repository.QuerytypeJsonExtractor), value) + + dbQueryType = resultprocessor.Type(-1) + _, err = resultprocessor.ToDBQueryType(dbQueryType) + assert.EqualError(t, err, "invalid database query type") + + dbQueryType = resultprocessor.Type(resultprocessor.TypeContextFull) + value, err = resultprocessor.ToDBQueryType(dbQueryType) + assert.Nil(t, err) + assert.Equal(t, repository.Querytype(repository.QuerytypeContextFull), value) +} + +func TestToDBQueryTypeArray(t *testing.T) { + inArr := []resultprocessor.Type{ + resultprocessor.TypeJsonExtractor, + resultprocessor.TypeContextFull, + } + value, err := resultprocessor.ToDBQueryTypeArray(inArr) + assert.Nil(t, err) + assert.Equal(t, []repository.Querytype{ + repository.QuerytypeJsonExtractor, + repository.QuerytypeContextFull, + }, value) + + inArr = []resultprocessor.Type{ + resultprocessor.Type(-1), + } + _, err = resultprocessor.ToDBQueryTypeArray(inArr) + assert.EqualError(t, err, "invalid database query type") +} + +func TestToDBNullQueryType(t *testing.T) { + dbQueryType := resultprocessor.Type(resultprocessor.TypeJsonExtractor) + value, err := resultprocessor.ToDBNullQueryType(dbQueryType) + assert.Nil(t, err) + assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor}, value) + + dbQueryType = resultprocessor.Type(-1) + _, err = resultprocessor.ToDBNullQueryType(dbQueryType) + assert.EqualError(t, err, "invalid database query type") + + dbQueryType = resultprocessor.Type(resultprocessor.TypeContextFull) + value, err = resultprocessor.ToDBNullQueryType(dbQueryType) + assert.Nil(t, err) + assert.Equal(t, repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeContextFull}, value) +} + +func TestParseFullQuery(t *testing.T) { + var q *repository.Fullactivequery + out, err := resultprocessor.ParseFullQuery(q) + assert.Nil(t, err) + assert.Nil(t, out) + + q = &repository.Fullactivequery{ + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: 1, + Latestversion: 2, + } + + out, err = resultprocessor.ParseFullQuery(q) + assert.Nil(t, err) + assert.EqualExportedValues(t, &resultprocessor.Query{ + ID: database.MustToUUID(q.ID), + Type: resultprocessor.TypeContextFull, + Version: 1, + }, out) + + q = &repository.Fullactivequery{ + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: 1, + Latestversion: 2, + Requiredids: []pgtype.UUID{}, + } + + out, err = resultprocessor.ParseFullQuery(q) + assert.Nil(t, err) + assert.EqualExportedValues(t, &resultprocessor.Query{ + ID: database.MustToUUID(q.ID), + Type: resultprocessor.TypeContextFull, + Version: 1, + }, out) + + q = &repository.Fullactivequery{ + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: 1, + Latestversion: 2, + Config: []byte("hello"), + Requiredids: []pgtype.UUID{ + database.MustToDBUUID(uuid.New()), + }, + } + + out, err = resultprocessor.ParseFullQuery(q) + assert.Nil(t, err) + cfg := "hello" + assert.EqualExportedValues(t, &resultprocessor.Query{ + ID: database.MustToUUID(q.ID), + Type: resultprocessor.TypeContextFull, + Version: 1, + Config: &cfg, + RequiredQueryIDs: &[]uuid.UUID{database.MustToUUID(q.Requiredids[0])}, + }, out) +} + +func TestParseFullQueryArray(t *testing.T) { + var q []*repository.Fullactivequery + out, err := resultprocessor.ParseFullQueryArray(q) + assert.Nil(t, err) + assert.ElementsMatch(t, []*resultprocessor.Query{}, out) + + q = []*repository.Fullactivequery{ + { + ID: database.MustToDBUUID(uuid.New()), + Type: repository.QuerytypeContextFull, + Activeversion: 1, + Latestversion: 2, + }, + } + + out, err = resultprocessor.ParseFullQueryArray(q) + assert.Nil(t, err) + assert.EqualExportedValues(t, []*resultprocessor.Query{ + { + ID: database.MustToUUID(q[0].ID), + Type: resultprocessor.TypeContextFull, + Version: 1, + }, + }, out) + +} diff --git a/internal/query/processor/service.go b/internal/query/result/processor/service.go similarity index 86% rename from internal/query/processor/service.go rename to internal/query/result/processor/service.go index 599a3bdc..ea36b844 100644 --- a/internal/query/processor/service.go +++ b/internal/query/result/processor/service.go @@ -1,12 +1,16 @@ -package queryprocessor +package resultprocessor import ( "context" - "queryorchestration/internal/query/result" "github.com/google/uuid" ) +type Value interface { + GetValue(ctx context.Context) (string, error) + GetStoreValue() string +} + type Type int const ( @@ -76,5 +80,5 @@ type Updator interface { } type Processor interface { - Process(ctx context.Context, query *Query, values *[]result.Value) (string, error) + Process(ctx context.Context, query *Query, values *[]Value) (string, error) } diff --git a/internal/query/processor/service_test.go b/internal/query/result/processor/service_test.go similarity index 82% rename from internal/query/processor/service_test.go rename to internal/query/result/processor/service_test.go index d4f2c73b..f5eedec2 100644 --- a/internal/query/processor/service_test.go +++ b/internal/query/result/processor/service_test.go @@ -1,7 +1,7 @@ -package queryprocessor_test +package resultprocessor_test import ( - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -9,7 +9,7 @@ import ( ) func TestCreateGetConfig(t *testing.T) { - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} assert.Nil(t, entity.GetConfig()) @@ -20,7 +20,7 @@ func TestCreateGetConfig(t *testing.T) { } func TestCreateSetConfig(t *testing.T) { - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} assert.Nil(t, entity.Config) @@ -30,7 +30,7 @@ func TestCreateSetConfig(t *testing.T) { } func TestUpdateGetConfig(t *testing.T) { - entity := queryprocessor.Update{} + entity := resultprocessor.Update{} assert.Nil(t, entity.GetConfig()) @@ -41,7 +41,7 @@ func TestUpdateGetConfig(t *testing.T) { } func TestUpdateSetConfig(t *testing.T) { - entity := queryprocessor.Update{} + entity := resultprocessor.Update{} assert.Nil(t, entity.Config) @@ -51,7 +51,7 @@ func TestUpdateSetConfig(t *testing.T) { } func TestCreateGetRequiredQueryIDs(t *testing.T) { - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} assert.Nil(t, entity.GetRequiredQueryIDs()) @@ -62,7 +62,7 @@ func TestCreateGetRequiredQueryIDs(t *testing.T) { } func TestCreateSetRequiredQueryIDs(t *testing.T) { - entity := queryprocessor.Create{} + entity := resultprocessor.Create{} assert.Nil(t, entity.RequiredQueryIDs) @@ -72,7 +72,7 @@ func TestCreateSetRequiredQueryIDs(t *testing.T) { } func TestUpdateGetRequiredQueryIDs(t *testing.T) { - entity := queryprocessor.Update{} + entity := resultprocessor.Update{} assert.Nil(t, entity.GetRequiredQueryIDs()) @@ -83,7 +83,7 @@ func TestUpdateGetRequiredQueryIDs(t *testing.T) { } func TestUpdateSetRequiredQueryIDs(t *testing.T) { - entity := queryprocessor.Update{} + entity := resultprocessor.Update{} assert.Nil(t, entity.RequiredQueryIDs) diff --git a/internal/query/result/service.go b/internal/query/result/service.go index bdf6b795..9810a15f 100644 --- a/internal/query/result/service.go +++ b/internal/query/result/service.go @@ -1,9 +1,7 @@ package result import ( - "context" "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" "github.com/google/uuid" ) @@ -14,33 +12,12 @@ type Result struct { QueryVersion int32 } -type Value interface { - GetValue(ctx context.Context) (string, error) +type Service struct { + db *database.Connection } -type ResultStore struct { - QueryID uuid.UUID - DocumentID uuid.UUID - Value string - CleanVersion int32 - TextVersion int32 - QueryVersion int32 -} - -func Store(ctx context.Context, dbQueries *repository.Queries, res *ResultStore) (uuid.UUID, error) { - dbId, err := dbQueries.SetResult(ctx, &repository.SetResultParams{ - Queryid: database.MustToDBUUID(res.QueryID), - Documentid: database.MustToDBUUID(res.DocumentID), - Value: res.Value, - Cleanversion: res.CleanVersion, - Textversion: res.TextVersion, - Queryversion: res.QueryVersion, - }) - if err != nil { - return uuid.Nil, err +func New(db *database.Connection) *Service { + return &Service{ + db, } - - id := database.MustToUUID(dbId) - - return id, nil } diff --git a/internal/query/result/service_test.go b/internal/query/result/service_test.go new file mode 100644 index 00000000..65093aeb --- /dev/null +++ b/internal/query/result/service_test.go @@ -0,0 +1,25 @@ +package result_test + +import ( + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query/result" + "testing" + + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestService(t *testing.T) { + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := result.New(db) + assert.NotNil(t, svc) +} diff --git a/internal/query/result/set.go b/internal/query/result/set.go new file mode 100644 index 00000000..d45e167d --- /dev/null +++ b/internal/query/result/set.go @@ -0,0 +1,45 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + + "github.com/google/uuid" +) + +type Set struct { + DocumentID uuid.UUID + CleanVersion int32 + TextVersion int32 + Query *resultprocessor.Query +} + +func (s *Service) Set(ctx context.Context, params *Set) (uuid.UUID, error) { + value, err := s.Process(ctx, &Process{ + DocumentID: params.DocumentID, + MinCleanVersion: params.CleanVersion, + MinTextVersion: params.TextVersion, + Query: params.Query, + }) + if err != nil { + return uuid.Nil, err + } + + dbId, err := s.db.Queries.SetResult(ctx, &repository.SetResultParams{ + Queryid: database.MustToDBUUID(params.Query.ID), + Documentid: database.MustToDBUUID(params.DocumentID), + Value: value.GetStoreValue(), + Cleanversion: params.CleanVersion, + Textversion: params.TextVersion, + Queryversion: params.Query.Version, + }) + if err != nil { + return uuid.Nil, err + } + + id := database.MustToUUID(dbId) + + return id, nil +} diff --git a/internal/query/result/set_test.go b/internal/query/result/set_test.go new file mode 100644 index 00000000..c2e94b3a --- /dev/null +++ b/internal/query/result/set_test.go @@ -0,0 +1,67 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + resultprocessor "queryorchestration/internal/query/result/processor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestSet(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + + svc := Service{ + db: db, + } + + rid := uuid.New() + cfg := "{\"path\":\"examplekey\"}" + params := Set{ + DocumentID: uuid.New(), + CleanVersion: 1, + TextVersion: 3, + Query: &resultprocessor.Query{ + ID: uuid.New(), + Version: 2, + Config: &cfg, + RequiredQueryIDs: &[]uuid.UUID{ + uuid.New(), + }, + }, + } + + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version, database.MustToDBUUID(params.DocumentID), params.CleanVersion, params.TextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(database.MustToDBUUID((*params.Query.RequiredQueryIDs)[0]), "{\"examplekey\":\"example_value\"}", repository.QuerytypeContextFull), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte(cfg)), + ) + pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(params.Query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), params.CleanVersion, params.TextVersion, params.Query.Version). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(database.MustToDBUUID(rid)), + ) + + id, err := svc.Set(ctx, ¶ms) + assert.Nil(t, err) + assert.Equal(t, rid, id) +} diff --git a/internal/query/result/store_test.go b/internal/query/result/store_test.go deleted file mode 100644 index a95ffd70..00000000 --- a/internal/query/result/store_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package result_test - -import ( - "context" - "errors" - "queryorchestration/internal/database" - "queryorchestration/internal/database/repository" - "queryorchestration/internal/query/result" - "testing" - - "github.com/google/uuid" - "github.com/jackc/pgx/v5/pgtype" - "github.com/pashagolub/pgxmock/v3" - "github.com/stretchr/testify/assert" -) - -func TestStore(t *testing.T) { - ctx := context.Background() - pool, err := pgxmock.NewPool() - if err != nil { - t.Fatalf("failed to open pgxmock database: %v", err) - } - queries := repository.New(pool) - - resultStore := result.ResultStore{ - QueryID: uuid.New(), - DocumentID: uuid.New(), - Value: "val", - CleanVersion: int32(1), - TextVersion: int32(1), - QueryVersion: int32(1), - } - - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(resultStore.QueryID), database.MustToDBUUID(resultStore.DocumentID), resultStore.Value, resultStore.CleanVersion, resultStore.TextVersion, resultStore.QueryVersion). - WillReturnRows( - pgxmock.NewRows([]string{"id"}).AddRow(pgtype.UUID{}), - ) - - id, err := result.Store(ctx, queries, &resultStore) - assert.Nil(t, err) - assert.NotNil(t, id) - - dbErr := "database failing" - pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(resultStore.QueryID), database.MustToDBUUID(resultStore.DocumentID), resultStore.Value, resultStore.CleanVersion, resultStore.TextVersion, resultStore.QueryVersion). - WillReturnError(errors.New(dbErr)) - - id, err = result.Store(ctx, queries, &resultStore) - assert.EqualError(t, err, dbErr) - assert.Equal(t, uuid.Nil, id) -} diff --git a/internal/query/result/sync.go b/internal/query/result/sync.go new file mode 100644 index 00000000..84e36a8a --- /dev/null +++ b/internal/query/result/sync.go @@ -0,0 +1,18 @@ +package result + +import ( + "context" + "queryorchestration/internal/database" + resultprocessor "queryorchestration/internal/query/result/processor" + + "github.com/google/uuid" +) + +func (s *Service) ListUnsyncedQueriesByDocId(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) { + qs, err := s.db.Queries.ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id)) + if err != nil { + return nil, err + } + + return resultprocessor.ParseFullQueryArray(qs) +} diff --git a/internal/query/result/sync_test.go b/internal/query/result/sync_test.go new file mode 100644 index 00000000..6f963b04 --- /dev/null +++ b/internal/query/result/sync_test.go @@ -0,0 +1,49 @@ +package result_test + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestListUnsyncedQueriesByDocId(t *testing.T) { + ctx := context.Background() + + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := result.New(db) + + documentId := uuid.New() + actualQs := []*resultprocessor.Query{ + { + ID: uuid.New(), + Type: resultprocessor.TypeJsonExtractor, + Version: 2, + }, + } + + pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(documentId)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(actualQs[0].ID), repository.QuerytypeJsonExtractor, actualQs[0].Version, actualQs[0].Version, nil, []pgtype.UUID{}), + ) + + val, err := svc.ListUnsyncedQueriesByDocId(ctx, documentId) + assert.Nil(t, err) + assert.ElementsMatch(t, actualQs, val) +} diff --git a/internal/query/service.go b/internal/query/service.go index 75dca704..f9aad383 100644 --- a/internal/query/service.go +++ b/internal/query/service.go @@ -1,13 +1,37 @@ package query -import "queryorchestration/internal/database" +import ( + "queryorchestration/internal/database" + "queryorchestration/internal/document" + documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/job/collector" + "queryorchestration/internal/query/result" + + "github.com/google/uuid" +) + +type Services struct { + Text *documenttext.Service + Result *result.Service + Collector *collector.Service + Document *document.Service +} type Service struct { - db *database.Connection + db *database.Connection + svc *Services } -func New(db *database.Connection) *Service { +func New(db *database.Connection, svc *Services) *Service { return &Service{ db, + svc, } } + +type Document struct { + ID uuid.UUID `json:"id" validate:"required,uuid"` + JobID uuid.UUID `json:"jobId" validate:"required,uuid"` + CleanVersion int32 `json:"cleanVersion" validate:"required,gt=0"` + TextVersion int32 `json:"textVersion" validate:"required,gt=0"` +} diff --git a/internal/query/service_test.go b/internal/query/service_test.go index 59252891..eebf8452 100644 --- a/internal/query/service_test.go +++ b/internal/query/service_test.go @@ -20,6 +20,6 @@ func TestService(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) assert.NotNil(t, svc) } diff --git a/internal/query/sync.go b/internal/query/sync.go new file mode 100644 index 00000000..23ad7250 --- /dev/null +++ b/internal/query/sync.go @@ -0,0 +1,163 @@ +package query + +import ( + "context" + "errors" + documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" + "sync" + + "github.com/google/uuid" +) + +func (s *Service) Sync(ctx context.Context, doc *Document) error { + err := s.svc.Text.IsExtracted(&documenttext.IsExtractedParams{ + DocumentID: doc.ID, + MinCleanVersion: doc.CleanVersion, + MinTextVersion: doc.TextVersion, + }) + if err != nil { + return err + } + + unsyncedQueries, err := s.svc.Result.ListUnsyncedQueriesByDocId(ctx, doc.ID) + if err != nil { + return err + } + + batchedQueries := s.batchQueries(unsyncedQueries) + + for _, queries := range batchedQueries { + err := s.processBatch(ctx, doc, queries) + if err != nil { + return err + } + } + + return nil +} + +func (s *Service) batchQueries(queries []*resultprocessor.Query) [][]*resultprocessor.Query { + n := len(queries) + if n == 0 { + return nil + } + + idToIndex := make(map[uuid.UUID]int, n) + for i, q := range queries { + idToIndex[q.ID] = i + } + + result := make([][]*resultprocessor.Query, 0, n) + assigned := make([]bool, n) + deps := make([][]int, n) + remaining := n + + for i, q := range queries { + toAppend := false + if q.RequiredQueryIDs == nil || len(*q.RequiredQueryIDs) == 0 { + toAppend = true + } else { + deps[i] = make([]int, 0, len(*q.RequiredQueryIDs)) + + for _, reqID := range *q.RequiredQueryIDs { + if idx, exists := idToIndex[reqID]; exists { + deps[i] = append(deps[i], idx) + } + } + + toAppend = len(deps[i]) == 0 + } + + if toAppend { + if len(result) == 0 { + result = [][]*resultprocessor.Query{{q}} + } else { + result[0] = append(result[0], q) + } + assigned[idToIndex[q.ID]] = true + remaining-- + } + } + + for remaining > 0 { + currentLayer := make([]*resultprocessor.Query, 0, remaining) + + for i, q := range queries { + if assigned[i] { + continue + } + + allSatisfied := true + for _, depIdx := range deps[i] { + if !assigned[depIdx] { + allSatisfied = false + break + } + } + + if allSatisfied { + currentLayer = append(currentLayer, q) + } + } + + for _, v := range currentLayer { + assigned[idToIndex[v.ID]] = true + remaining-- + } + + result = append(result, currentLayer) + } + + return result +} + +func (s *Service) processBatch(ctx context.Context, doc *Document, queries []*resultprocessor.Query) error { + if doc == nil { + return errors.New("document required") + } + + errChan := make(chan error, len(queries)) + var wg sync.WaitGroup + sem := make(chan struct{}, 10) + + for _, query := range queries { + wg.Add(1) + go func(q *resultprocessor.Query) { + defer wg.Done() + + sem <- struct{}{} + defer func() { + <-sem + }() + + select { + case <-ctx.Done(): + errChan <- ctx.Err() + return + default: + } + + _, err := s.svc.Result.Set(ctx, &result.Set{ + DocumentID: doc.ID, + CleanVersion: doc.CleanVersion, + TextVersion: doc.TextVersion, + Query: query, + }) + if err != nil { + errChan <- err + } + }(query) + } + + wg.Wait() + close(errChan) + for err := range errChan { + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/query/sync_test.go b/internal/query/sync_test.go new file mode 100644 index 00000000..b9d5b3f9 --- /dev/null +++ b/internal/query/sync_test.go @@ -0,0 +1,307 @@ +package query + +import ( + "context" + "queryorchestration/internal/database" + "queryorchestration/internal/database/repository" + "queryorchestration/internal/job/collector" + "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" + "testing" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v3" + "github.com/stretchr/testify/assert" +) + +func TestBatchQueries(t *testing.T) { + svc := Service{} + + in := []*resultprocessor.Query{} + out := svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{}, out) + + in = []*resultprocessor.Query{ + { + ID: uuid.New(), + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + }, + }, out) + + in = []*resultprocessor.Query{ + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{uuid.New()}, + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + }, + }, out) + + in = []*resultprocessor.Query{ + { + ID: uuid.New(), + }, + { + ID: uuid.New(), + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + in[1], + }, + }, out) + + idone := uuid.New() + in = []*resultprocessor.Query{ + { + ID: idone, + }, + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + idone, + }, + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + }, + { + in[1], + }, + }, out) + + in = []*resultprocessor.Query{ + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + idone, + }, + }, + { + ID: idone, + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[1], + }, + { + in[0], + }, + }, out) + + in = []*resultprocessor.Query{ + { + ID: idone, + }, + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + idone, + }, + }, + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + idone, + }, + }, + { + ID: uuid.New(), + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + in[3], + }, + { + in[1], + in[2], + }, + }, out) + + idtwo := uuid.New() + idthree := uuid.New() + in = []*resultprocessor.Query{ + { + ID: idone, + }, + { + ID: idtwo, + RequiredQueryIDs: &[]uuid.UUID{ + idone, + }, + }, + { + ID: idthree, + RequiredQueryIDs: &[]uuid.UUID{ + idtwo, + }, + }, + { + ID: uuid.New(), + RequiredQueryIDs: &[]uuid.UUID{ + idthree, + }, + }, + } + out = svc.batchQueries(in) + assert.ElementsMatch(t, [][]*resultprocessor.Query{ + { + in[0], + }, + { + in[1], + }, + { + in[2], + }, + { + in[3], + }, + }, out) +} + +func TestProcessBatch(t *testing.T) { + ctx := context.Background() + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := Service{ + svc: &Services{ + Result: result.New(db), + }, + } + + assert.Error(t, svc.processBatch(ctx, nil, nil)) + + doc := &Document{ + ID: uuid.New(), + CleanVersion: 1, + TextVersion: 1, + } + + assert.Nil(t, svc.processBatch(ctx, doc, nil)) + + qs := []*resultprocessor.Query{} + assert.Nil(t, svc.processBatch(ctx, doc, qs)) + + coll := &collector.Collector{ + ID: uuid.New(), + MinCleanVersion: 1, + MinTextVersion: 1, + } + qs = []*resultprocessor.Query{ + { + ID: uuid.New(), + Type: resultprocessor.TypeContextFull, + Version: 1, + }, + } + pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qs[0].ID), database.MustToDBUUID(doc.ID), pgxmock.AnyArg(), coll.MinCleanVersion, coll.MinTextVersion, qs[0].Version). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(pgtype.UUID{}), + ) + pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + ) + pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, []pgtype.UUID{}), + ) + assert.Nil(t, svc.processBatch(ctx, doc, qs)) +} + +func TestSync(t *testing.T) { + ctx := context.Background() + pool, err := pgxmock.NewPool() + if err != nil { + t.Fatalf("failed to open pgxmock database: %v", err) + } + queries := repository.New(pool) + db := &database.Connection{ + Queries: queries, + Pool: pool, + } + svc := Service{ + svc: &Services{ + Result: result.New(db), + }, + } + + assert.Error(t, svc.processBatch(ctx, nil, nil)) + + doc := &Document{ + ID: uuid.New(), + CleanVersion: 1, + TextVersion: 1, + } + + assert.Nil(t, svc.processBatch(ctx, doc, nil)) + + qs := []*resultprocessor.Query{} + assert.Nil(t, svc.processBatch(ctx, doc, qs)) + + coll := &collector.Collector{ + ID: uuid.New(), + MinCleanVersion: 1, + MinTextVersion: 1, + } + qs = []*resultprocessor.Query{ + { + ID: uuid.New(), + Type: resultprocessor.TypeContextFull, + Version: 1, + }, + } + + pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, qs[0].Version, nil, []pgtype.UUID{}), + ) + pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(qs[0].ID), database.MustToDBUUID(doc.ID), pgxmock.AnyArg(), coll.MinCleanVersion, coll.MinTextVersion, qs[0].Version). + WillReturnRows( + pgxmock.NewRows([]string{"id"}). + AddRow(pgtype.UUID{}), + ) + pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + ) + pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(qs[0].ID), repository.QuerytypeContextFull, qs[0].Version, []pgtype.UUID{}), + ) + assert.Nil(t, svc.Sync(ctx, doc)) +} diff --git a/internal/query/test.go b/internal/query/test.go index 5eed7887..77d841ac 100644 --- a/internal/query/test.go +++ b/internal/query/test.go @@ -2,6 +2,8 @@ package query import ( "context" + "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/google/uuid" ) @@ -13,9 +15,47 @@ type Test struct { } func (s *Service) Test(ctx context.Context, filters Test) (string, error) { - // TODO - // Sync doc - documentID - // Run test - queryID, queryVersion + doc, err := s.svc.Document.Get(ctx, filters.DocumentID) + if err != nil { + return "", err + } - return "", nil + col, err := s.svc.Collector.GetByJobID(ctx, doc.JobID) + if err != nil { + return "", err + } + + res, err := s.svc.Result.GetValueWithVersion(ctx, &result.GetValueWithVersionParams{ + QueryID: filters.QueryID, + DocumentID: filters.DocumentID, + QueryVersion: filters.QueryVersion, + MinCleanVersion: col.MinCleanVersion, + MinTextVersion: col.MinTextVersion, + }) + if err == nil { + return res.GetValue(ctx) + } + + q, err := s.GetWithVersion(ctx, filters.QueryID, filters.QueryVersion) + if err != nil { + return "", err + } + + value, err := s.svc.Result.Process(ctx, &result.Process{ + DocumentID: filters.DocumentID, + MinCleanVersion: col.MinCleanVersion, + MinTextVersion: col.MinTextVersion, + Query: &resultprocessor.Query{ + ID: q.ID, + Type: q.Type, + Version: filters.QueryVersion, + RequiredQueryIDs: q.RequiredQueryIDs, + Config: q.Config, + }, + }) + if err != nil { + return "", err + } + + return value.GetValue(ctx) } diff --git a/internal/query/test_test.go b/internal/query/test_test.go index a6dc40b0..b08809e2 100644 --- a/internal/query/test_test.go +++ b/internal/query/test_test.go @@ -4,10 +4,16 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" + "queryorchestration/internal/document" + documentclean "queryorchestration/internal/document/clean" + documenttext "queryorchestration/internal/document/text" + "queryorchestration/internal/job/collector" "queryorchestration/internal/query" + "queryorchestration/internal/query/result" "testing" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" "github.com/pashagolub/pgxmock/v3" "github.com/stretchr/testify/assert" ) @@ -24,15 +30,62 @@ func TestTest(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + text := documenttext.New() + clean := documentclean.New() + col := collector.New(db, &collector.Services{ + Text: text, + Clean: clean, + }) + docsvc := document.New(db) + svc := query.New(db, &query.Services{ + Text: text, + Document: docsvc, + Collector: col, + Result: result.New(db), + }) + coll := collector.Collector{ + ID: uuid.New(), + JobID: uuid.New(), + } + doc := document.Document{ + ID: uuid.New(), + JobID: coll.JobID, + Hash: "example_hash", + Location: "example_location", + } params := &query.Test{ QueryID: uuid.New(), - DocumentID: uuid.New(), + DocumentID: doc.ID, QueryVersion: int32(1), } + reqID := database.MustToDBUUID(uuid.New()) + pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}). + AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location), + ) + pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)). + WillReturnRows( + pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}). + AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")), + ) + pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( + pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}). + AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}), + ) + pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion). + WillReturnRows( + pgxmock.NewRows([]string{"queryId", "value", "type"}). + AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull), + ) + pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows( + pgxmock.NewRows([]string{"id", "config"}). + AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")), + ) + result, err := svc.Test(ctx, *params) assert.Nil(t, err) - assert.Empty(t, result) + assert.Equal(t, "old_value", result) } diff --git a/internal/query/types/contextFull/creator.go b/internal/query/types/contextFull/creator.go index 3e08fac4..a0311669 100644 --- a/internal/query/types/contextFull/creator.go +++ b/internal/query/types/contextFull/creator.go @@ -3,7 +3,7 @@ package contextfull import ( "context" "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" ) type Creator struct { @@ -14,7 +14,7 @@ func NewCreator(db *database.Connection) Creator { return Creator{db} } -func (s Creator) Validate(ctx context.Context, entity *queryprocessor.Create) error { +func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { // TODO // Type, RequiredQueryIDs, Config return nil diff --git a/internal/query/types/contextFull/creator_test.go b/internal/query/types/contextFull/creator_test.go index 1332c8e3..3f14de3f 100644 --- a/internal/query/types/contextFull/creator_test.go +++ b/internal/query/types/contextFull/creator_test.go @@ -4,7 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" "testing" @@ -28,8 +28,8 @@ func TestCreatorValidate(t *testing.T) { svc := contextfull.NewCreator(db) assert.NotNil(t, svc) - entity := &queryprocessor.Create{ - Type: queryprocessor.TypeContextFull, + entity := &resultprocessor.Create{ + Type: resultprocessor.TypeContextFull, } err = svc.Validate(ctx, entity) diff --git a/internal/query/types/contextFull/process_test.go b/internal/query/types/contextFull/process_test.go index f1738ae9..fb0583f1 100644 --- a/internal/query/types/contextFull/process_test.go +++ b/internal/query/types/contextFull/process_test.go @@ -2,8 +2,7 @@ package contextfull_test import ( "context" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" "testing" @@ -16,19 +15,19 @@ func TestContextFull(t *testing.T) { extractor := contextfull.NewExtractor() - query := &queryprocessor.Query{ + query := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, Version: int32(1), } - values := []result.Value{} + values := []resultprocessor.Value{} value, err := extractor.Process(ctx, query, &values) assert.Nil(t, err) assert.Equal(t, "", value) - values = []result.Value{ + values = []resultprocessor.Value{ contextfull.NewResult("example_result"), } diff --git a/internal/query/types/contextFull/result.go b/internal/query/types/contextFull/result.go index 0ad30bf1..32abfb22 100644 --- a/internal/query/types/contextFull/result.go +++ b/internal/query/types/contextFull/result.go @@ -16,3 +16,7 @@ func (r Result) GetValue(ctx context.Context) (string, error) { // TODO - get value from s3 return r.value, nil } + +func (r Result) GetStoreValue() string { + return r.value +} diff --git a/internal/query/types/contextFull/service.go b/internal/query/types/contextFull/service.go index 1ae7d351..2649610e 100644 --- a/internal/query/types/contextFull/service.go +++ b/internal/query/types/contextFull/service.go @@ -3,8 +3,7 @@ package contextfull import ( "context" "errors" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" ) type Extractor struct { @@ -14,7 +13,7 @@ func NewExtractor() Extractor { return Extractor{} } -func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values *[]result.Value) (string, error) { +func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { if values != nil && len(*values) > 0 { return "", errors.New("no requirements expected") } diff --git a/internal/query/types/contextFull/updator.go b/internal/query/types/contextFull/updator.go index b0f47515..ddb1749e 100644 --- a/internal/query/types/contextFull/updator.go +++ b/internal/query/types/contextFull/updator.go @@ -3,7 +3,7 @@ package contextfull import ( "context" "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" ) type Updator struct { @@ -14,7 +14,7 @@ func NewUpdator(db *database.Connection) Updator { return Updator{db} } -func (s Updator) Validate(ctx context.Context, current *queryprocessor.Query, entity *queryprocessor.Update) error { +func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { // TODO // Type, RequiredQueryIDs, Config return nil diff --git a/internal/query/types/contextFull/updator_test.go b/internal/query/types/contextFull/updator_test.go index 83a12b38..d7e5fedc 100644 --- a/internal/query/types/contextFull/updator_test.go +++ b/internal/query/types/contextFull/updator_test.go @@ -4,7 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" "testing" @@ -29,13 +29,13 @@ func TestUpdatorValidate(t *testing.T) { svc := contextfull.NewUpdator(db) assert.NotNil(t, svc) - current := &queryprocessor.Query{ + current := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeContextFull, + Type: resultprocessor.TypeContextFull, Version: int32(1), } - entity := &queryprocessor.Update{ + entity := &resultprocessor.Update{ ID: current.ID, } diff --git a/internal/query/types/jsonExtractor/creator.go b/internal/query/types/jsonExtractor/creator.go index 6ef0f5b4..fa147462 100644 --- a/internal/query/types/jsonExtractor/creator.go +++ b/internal/query/types/jsonExtractor/creator.go @@ -3,7 +3,7 @@ package jsonextractor import ( "context" "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" ) type Creator struct { @@ -14,7 +14,7 @@ func NewCreator(db *database.Connection) Creator { return Creator{db} } -func (s Creator) Validate(ctx context.Context, entity *queryprocessor.Create) error { +func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error { // TODO return nil } diff --git a/internal/query/types/jsonExtractor/creator_test.go b/internal/query/types/jsonExtractor/creator_test.go index b7a12dff..0bde4356 100644 --- a/internal/query/types/jsonExtractor/creator_test.go +++ b/internal/query/types/jsonExtractor/creator_test.go @@ -4,7 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "testing" @@ -28,8 +28,8 @@ func TestCreatorValidate(t *testing.T) { svc := jsonextractor.NewCreator(db) assert.NotNil(t, svc) - entity := &queryprocessor.Create{ - Type: queryprocessor.TypeJsonExtractor, + entity := &resultprocessor.Create{ + Type: resultprocessor.TypeJsonExtractor, } err = svc.Validate(ctx, entity) diff --git a/internal/query/types/jsonExtractor/process_test.go b/internal/query/types/jsonExtractor/process_test.go index debeedf2..b72875aa 100644 --- a/internal/query/types/jsonExtractor/process_test.go +++ b/internal/query/types/jsonExtractor/process_test.go @@ -5,8 +5,7 @@ import ( "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "testing" @@ -32,15 +31,15 @@ func TestJSONProcess(t *testing.T) { extractor := jsonextractor.NewExtractor(db) - query := &queryprocessor.Query{ + query := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, Version: int32(1), } entryValue := "value" jsonString := fmt.Sprintf("{\"key\": \"%s\"}", entryValue) - values := []result.Value{ + values := []resultprocessor.Value{ contextfull.NewResult(jsonString), } @@ -58,7 +57,7 @@ func TestJSONProcess(t *testing.T) { entryValue = "" jsonString = fmt.Sprintf("{\"key\": \"%s\"}", entryValue) - values = []result.Value{ + values = []resultprocessor.Value{ contextfull.NewResult(jsonString), } pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). @@ -73,7 +72,7 @@ func TestJSONProcess(t *testing.T) { entryValue = "1" jsonString = fmt.Sprintf("{\"key\": %s", entryValue) - values = []result.Value{ + values = []resultprocessor.Value{ contextfull.NewResult(jsonString), } pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version). @@ -102,15 +101,15 @@ func TestJSONProcessJSON(t *testing.T) { extractor := jsonextractor.NewExtractor(db) - query := &queryprocessor.Query{ + query := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, Version: int32(1), } entryValue := "value" jsonString := fmt.Sprintf("{\"key\": \"%s\"}", entryValue) - values := []result.Value{ + values := []resultprocessor.Value{ contextfull.NewResult(jsonString), } @@ -187,18 +186,18 @@ func TestJSONProcessResults(t *testing.T) { extractor := jsonextractor.NewExtractor(db) - query := &queryprocessor.Query{ + query := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, Version: int32(1), } - results := []result.Value{} + results := []resultprocessor.Value{} value, err := extractor.Process(ctx, query, &results) assert.EqualError(t, err, "JSON Extraction requires 1 result") assert.Empty(t, value) - results = []result.Value{ + results = []resultprocessor.Value{ contextfull.NewResult(""), contextfull.NewResult(""), } @@ -206,7 +205,7 @@ func TestJSONProcessResults(t *testing.T) { assert.EqualError(t, err, "JSON Extraction requires 1 result") assert.Empty(t, value) - results = []result.Value{ + results = []resultprocessor.Value{ contextfull.NewResult(""), contextfull.NewResult(""), contextfull.NewResult(""), diff --git a/internal/query/types/jsonExtractor/result.go b/internal/query/types/jsonExtractor/result.go index 7ac82ef0..682763d3 100644 --- a/internal/query/types/jsonExtractor/result.go +++ b/internal/query/types/jsonExtractor/result.go @@ -15,3 +15,7 @@ func NewResult(value string) Result { func (r Result) GetValue(ctx context.Context) (string, error) { return r.value, nil } + +func (r Result) GetStoreValue() string { + return r.value +} diff --git a/internal/query/types/jsonExtractor/service.go b/internal/query/types/jsonExtractor/service.go index 928c0266..a8ec6639 100644 --- a/internal/query/types/jsonExtractor/service.go +++ b/internal/query/types/jsonExtractor/service.go @@ -6,8 +6,7 @@ import ( "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" - "queryorchestration/internal/query/result" + resultprocessor "queryorchestration/internal/query/result/processor" "github.com/tidwall/gjson" ) @@ -24,7 +23,7 @@ func NewExtractor(db *database.Connection) Extractor { return Extractor{db} } -func (e Extractor) Process(ctx context.Context, query *queryprocessor.Query, values *[]result.Value) (string, error) { +func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) { if values == nil || len(*values) != 1 { return "", fmt.Errorf("JSON Extraction requires 1 result") } diff --git a/internal/query/types/jsonExtractor/updator.go b/internal/query/types/jsonExtractor/updator.go index 0c656dda..ce8a35b5 100644 --- a/internal/query/types/jsonExtractor/updator.go +++ b/internal/query/types/jsonExtractor/updator.go @@ -3,7 +3,7 @@ package jsonextractor import ( "context" "queryorchestration/internal/database" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" ) type Updator struct { @@ -14,7 +14,7 @@ func NewUpdator(db *database.Connection) Updator { return Updator{db} } -func (s Updator) Validate(ctx context.Context, current *queryprocessor.Query, entity *queryprocessor.Update) error { +func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error { // TODO return nil } diff --git a/internal/query/types/jsonExtractor/updator_test.go b/internal/query/types/jsonExtractor/updator_test.go index 0d801d5c..ba5427cd 100644 --- a/internal/query/types/jsonExtractor/updator_test.go +++ b/internal/query/types/jsonExtractor/updator_test.go @@ -4,7 +4,7 @@ import ( "context" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "testing" @@ -29,13 +29,13 @@ func TestUpdatorValidate(t *testing.T) { svc := jsonextractor.NewUpdator(db) assert.NotNil(t, svc) - current := &queryprocessor.Query{ + current := &resultprocessor.Query{ ID: uuid.New(), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, Version: int32(1), } - entity := &queryprocessor.Update{ + entity := &resultprocessor.Update{ ID: current.ID, } diff --git a/internal/query/update.go b/internal/query/update.go index bf0801c4..cca38af0 100644 --- a/internal/query/update.go +++ b/internal/query/update.go @@ -6,14 +6,14 @@ import ( "fmt" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" contextfull "queryorchestration/internal/query/types/contextFull" jsonextractor "queryorchestration/internal/query/types/jsonExtractor" "github.com/google/uuid" ) -func (s *Service) Update(ctx context.Context, entity *queryprocessor.Update) error { +func (s *Service) Update(ctx context.Context, entity *resultprocessor.Update) error { current, err := s.Get(ctx, entity.ID) if err != nil { return err @@ -32,7 +32,7 @@ func (s *Service) Update(ctx context.Context, entity *queryprocessor.Update) err return nil } -func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *queryprocessor.Update) error { +func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *resultprocessor.Update) error { err := s.normalizeActiveVersion(current, entity) if err != nil { return err @@ -79,7 +79,7 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *q return nil } -func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *queryprocessor.Update) error { +func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *resultprocessor.Update) error { tx, err := s.db.Pool.Begin(ctx) if err != nil { return err @@ -182,19 +182,19 @@ func getSetDifference(setA *[]uuid.UUID, setB *[]uuid.UUID) []uuid.UUID { return diff } -func (s *Service) getUpdator(qType queryprocessor.Type) (queryprocessor.Updator, error) { +func (s *Service) getUpdator(qType resultprocessor.Type) (resultprocessor.Updator, error) { switch qType { - case queryprocessor.TypeJsonExtractor: + case resultprocessor.TypeJsonExtractor: return jsonextractor.NewUpdator(s.db), nil - case queryprocessor.TypeContextFull: + case resultprocessor.TypeContextFull: return contextfull.NewUpdator(s.db), nil default: return nil, fmt.Errorf("attempting to process invalid query type") } } -func ParseQuery(q *Query) *queryprocessor.Query { - return &queryprocessor.Query{ +func ParseQuery(q *Query) *resultprocessor.Query { + return &resultprocessor.Query{ ID: q.ID, Type: q.Type, Version: q.ActiveVersion, diff --git a/internal/query/update_test.go b/internal/query/update_test.go index c5c02833..654404fb 100644 --- a/internal/query/update_test.go +++ b/internal/query/update_test.go @@ -5,7 +5,7 @@ import ( "queryorchestration/internal/database" "queryorchestration/internal/database/repository" "queryorchestration/internal/query" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -26,7 +26,7 @@ func TestUpdate(t *testing.T) { Queries: queries, Pool: pool, } - svc := query.New(db) + svc := query.New(db, &query.Services{}) config := "{\"path\":\"example_path\"}" existing := query.Query{ @@ -34,7 +34,7 @@ func TestUpdate(t *testing.T) { ActiveVersion: int32(1), LatestVersion: int32(1), } - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: existing.ID, } diff --git a/internal/query/updateprivate_test.go b/internal/query/updateprivate_test.go index 42eeb6a2..558b4bb7 100644 --- a/internal/query/updateprivate_test.go +++ b/internal/query/updateprivate_test.go @@ -5,7 +5,7 @@ import ( "errors" "queryorchestration/internal/database" "queryorchestration/internal/database/repository" - queryprocessor "queryorchestration/internal/query/processor" + resultprocessor "queryorchestration/internal/query/result/processor" "testing" "github.com/google/uuid" @@ -24,19 +24,19 @@ func TestGetUpdator(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) - queryType := queryprocessor.Type(queryprocessor.TypeContextFull) + queryType := resultprocessor.Type(resultprocessor.TypeContextFull) updator, err := svc.getUpdator(queryType) assert.Nil(t, err) assert.NotNil(t, updator) - queryType = queryprocessor.Type(queryprocessor.TypeJsonExtractor) + queryType = resultprocessor.Type(resultprocessor.TypeJsonExtractor) updator, err = svc.getUpdator(queryType) assert.Nil(t, err) assert.NotNil(t, updator) - queryType = queryprocessor.Type(-1) + queryType = resultprocessor.Type(-1) _, err = svc.getUpdator(queryType) assert.NotNil(t, err) } @@ -53,7 +53,7 @@ func TestSubmitUpdate(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ @@ -66,7 +66,7 @@ func TestSubmitUpdate(t *testing.T) { LatestVersion: int32(2), } aV := int32(10) - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: q.ID, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), @@ -104,7 +104,7 @@ func TestSubmitUpdateRollback(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) q := Query{ ID: uuid.New(), @@ -112,7 +112,7 @@ func TestSubmitUpdateRollback(t *testing.T) { LatestVersion: int32(2), } aV := int32(10) - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: q.ID, ActiveVersion: &aV, } @@ -139,7 +139,7 @@ func TestSubmitUpdateRequiredQueries(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) config := "{\"path\":\"example_path\"}" q := Query{ @@ -153,7 +153,7 @@ func TestSubmitUpdateRequiredQueries(t *testing.T) { ActiveVersion: int32(1), LatestVersion: int32(2), } - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: q.ID, RequiredQueryIDs: &[]uuid.UUID{ uuid.New(), @@ -191,7 +191,7 @@ func TestSubmitUpdateActiveVersion(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) q := Query{ ID: uuid.New(), @@ -199,7 +199,7 @@ func TestSubmitUpdateActiveVersion(t *testing.T) { LatestVersion: int32(2), } aV := int32(10) - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: q.ID, ActiveVersion: &aV, } @@ -238,17 +238,17 @@ func TestNormalizeUpdate(t *testing.T) { Queries: queries, Pool: pool, } - svc := New(db) + svc := New(db, &Services{}) current := &Query{ ID: uuid.New(), ActiveVersion: int32(1), LatestVersion: int32(2), - Type: queryprocessor.TypeJsonExtractor, + Type: resultprocessor.TypeJsonExtractor, } cfg := "{}" aV := int32(2) - update := &queryprocessor.Update{ + update := &resultprocessor.Update{ ID: current.ID, Config: &cfg, ActiveVersion: &aV, @@ -264,7 +264,7 @@ func TestNormalizeUpdate(t *testing.T) { err = svc.normalizeUpdate(ctx, current, update) assert.Nil(t, err) - assert.EqualExportedValues(t, queryprocessor.Update{ + assert.EqualExportedValues(t, resultprocessor.Update{ ID: current.ID, ActiveVersion: &aV, Config: &cfg, @@ -283,7 +283,7 @@ func TestNormalizeUpdate(t *testing.T) { err = svc.normalizeUpdate(ctx, current, update) assert.Error(t, err) - update = &queryprocessor.Update{ + update = &resultprocessor.Update{ ID: current.ID, } err = svc.normalizeUpdate(ctx, current, update) diff --git a/internal/test/container.go b/internal/test/container.go index 32701f43..bd145768 100644 --- a/internal/test/container.go +++ b/internal/test/container.go @@ -50,8 +50,8 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t } req.ExposedPorts = ports req.WaitingFor = wait.ForAll( - wait.ForExposedPort(), - wait.ForListeningPort(cfg.ExposedPorts[0]), + // wait.ForExposedPort(), + // wait.ForListeningPort(cfg.ExposedPorts[0]), wait.ForLog(cfg.WaitForMsg), ) } diff --git a/internal/text_extraction/service.go b/internal/text_extraction/service.go deleted file mode 100644 index 35dc5794..00000000 --- a/internal/text_extraction/service.go +++ /dev/null @@ -1,18 +0,0 @@ -package textextraction - -import "errors" - -type Service struct { -} - -func New() *Service { - return &Service{} -} - -func (s *Service) IsValidVersion(v int32) error { - if v <= 0 { - return errors.New("document clean code version must be > 0") - } - - return nil -} diff --git a/internal/text_extraction/service_test.go b/internal/text_extraction/service_test.go deleted file mode 100644 index 5de541ec..00000000 --- a/internal/text_extraction/service_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package textextraction_test - -import ( - documentclean "queryorchestration/internal/document/clean" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestService(t *testing.T) { - svc := documentclean.New() - assert.NotNil(t, svc) -} diff --git a/test/queryRunner/queryrunner_test.go b/test/queryRunner/queryrunner_test.go index 4ac714e0..ec8d8228 100644 --- a/test/queryRunner/queryrunner_test.go +++ b/test/queryRunner/queryrunner_test.go @@ -3,7 +3,7 @@ package integration_test import ( "context" "encoding/json" - documentsync "queryorchestration/internal/document/sync" + "queryorchestration/internal/query" "queryorchestration/internal/server/queue" "queryorchestration/internal/test" "testing" @@ -19,7 +19,7 @@ func TestQueryRunner(t *testing.T) { qCfg, cleanup := test.CreateQueueWithDependencies(t, ctx, "queryRunner") defer cleanup() - document := documentsync.Document{ + document := query.Document{ ID: uuid.New(), JobID: uuid.New(), CleanVersion: int32(1), diff --git a/test/queryService/testquery_test.go b/test/queryService/testquery_test.go index 05b75197..2c224a16 100644 --- a/test/queryService/testquery_test.go +++ b/test/queryService/testquery_test.go @@ -34,4 +34,6 @@ func TestQueryServiceTest(t *testing.T) { }) assert.Nil(t, err) assert.NotNil(t, testRes) + // assert.NotNil(t, testRes.JSON200) + // assert.Equal(t, "hey", testRes.JSON200.Value) }