Merged in feature/testquery (pull request #39)

Test Query

* depstextandclean

* startedcleaningresult

* resulttidyup

* roundone

* cleaning

* unsyncedquery

* startedtestsandsimplification

* api

* querytests

* resultprocessortests

* unittests

* cleanup

api/queryRunner/queryrunner.go

@@ -3,7 +3,7 @@ package controllers
 import (
 	"context"
 	"encoding/json"
-	documentsync "queryorchestration/internal/document/sync"
+	"queryorchestration/internal/query"
 
 	"github.com/go-playground/validator/v10"
 
@@ -12,7 +12,7 @@ import (
 )
 
 type Services struct {
-	DocumentSync *documentsync.Service
+	Query *query.Service
 }
 
 type QueryRunner struct {
@@ -32,7 +32,7 @@ type DocumentQueryEvent struct {
 }
 
 func (s QueryRunner) Process(ctx context.Context, req *types.Message) error {
-	var body documentsync.Document
+	var body query.Document
 	err := json.Unmarshal([]byte(*req.Body), &body)
 	if err != nil {
 		return err
@@ -43,7 +43,7 @@ func (s QueryRunner) Process(ctx context.Context, req *types.Message) error {
 		return err
 	}
 
-	err = s.svc.DocumentSync.Sync(ctx, &body)
+	err = s.svc.Query.Sync(ctx, &body)
 	if err != nil {
 		return err
 	}

api/queryRunner/queryrunner_test.go

@@ -6,15 +6,15 @@ import (
 	controllers "queryorchestration/api/queryRunner"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
-	documentsync "queryorchestration/internal/document/sync"
+	documenttext "queryorchestration/internal/document/text"
 	"queryorchestration/internal/job/collector"
 	"queryorchestration/internal/query"
+	"queryorchestration/internal/query/result"
 	"testing"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
-	"github.com/jackc/pgx/v5/pgtype"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
 )
@@ -32,18 +32,18 @@ func TestQueryRunner(t *testing.T) {
 		Pool:    pool,
 	}
 
-	svc := documentsync.New(db, &documentsync.Services{
-		Collector: collector.New(db, &collector.Services{
-			Query: query.New(db),
-		}),
+	svc := query.New(db, &query.Services{
+		Result:    result.New(db),
+		Text:      documenttext.New(),
+		Collector: collector.New(db, &collector.Services{}),
 	})
 
 	runner := controllers.NewQueryRunner(validator.New(), &controllers.Services{
-		DocumentSync: svc,
+		Query: svc,
 	})
 	assert.NotNil(t, runner)
 
-	doc := documentsync.Document{
+	doc := query.Document{
 		ID:           uuid.New(),
 		JobID:        uuid.New(),
 		CleanVersion: 1,
@@ -55,27 +55,10 @@ func TestQueryRunner(t *testing.T) {
 	msg := &types.Message{
 		Body: &body,
 	}
-	collectorID := database.MustToDBUUID(uuid.New())
-	queryID := database.MustToDBUUID(uuid.New())
-	minCleanVersion := int32(1)
-	minTextVersion := int32(1)
 
-	qV := int32(1)
-
-	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
+	pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
-				AddRow(collectorID, database.MustToDBUUID(doc.JobID), &minCleanVersion, &minTextVersion, int32(1), int32(2), []byte("")),
-		)
-	pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
-		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
-				AddRow(collectorID, queryID, qV),
-		)
-	pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(collectorID).
-		WillReturnRows(
-			pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
-				AddRow(collectorID, queryID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}),
+			pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}),
 		)
 
 	err = runner.Process(ctx, msg)

api/queryService/controllers.go

@@ -11,11 +11,11 @@ import (
 )
 
 type Services struct {
-	Export       *export.Service
-	JobCollector *collector.Service
-	Query        *query.Service
-	Client       *client.Service
-	Job          *job.Service
+	Export    *export.Service
+	Collector *collector.Service
+	Query     *query.Service
+	Client    *client.Service
+	Job       *job.Service
 }
 
 type Controllers struct {

api/queryService/job_test.go

@@ -10,10 +10,9 @@ import (
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
 	documentclean "queryorchestration/internal/document/clean"
+	documenttext "queryorchestration/internal/document/text"
 	"queryorchestration/internal/job"
 	"queryorchestration/internal/job/collector"
-	"queryorchestration/internal/query"
-	textextraction "queryorchestration/internal/text_extraction"
 	"strings"
 	"testing"
 
@@ -36,12 +35,12 @@ func TestCreateJob(t *testing.T) {
 		Pool:    pool,
 	}
 
+	extract := documenttext.New()
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
 		Job: job.New(db, &job.Services{
 			Collector: collector.New(db, &collector.Services{
-				Query:          query.New(db),
-				DocumentClean:  documentclean.New(),
-				TextExtraction: textextraction.New(),
+				Clean: documentclean.New(),
+				Text:  extract,
 			}),
 		}),
 	})

api/queryService/jobcollector.go

@@ -11,7 +11,7 @@ import (
 )
 
 func (s *Controllers) GetJobCollectorByJobId(ctx echo.Context, jobId types.UUID) error {
-	coll, err := s.svc.JobCollector.GetByJobID(ctx.Request().Context(), jobId)
+	coll, err := s.svc.Collector.GetByJobID(ctx.Request().Context(), jobId)
 	if err != nil {
 		return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to get collector: %s", err))
 	}
@@ -50,7 +50,7 @@ func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UU
 		}
 	}
 
-	err := s.svc.JobCollector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{
+	err := s.svc.Collector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{
 		JobID:           jobId,
 		ActiveVersion:   req.ActiveVersion,
 		MinCleanVersion: req.MinimumCleanerVersion,

api/queryService/jobcollector_test.go

@@ -8,9 +8,8 @@ import (
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
 	documentclean "queryorchestration/internal/document/clean"
+	documenttext "queryorchestration/internal/document/text"
 	"queryorchestration/internal/job/collector"
-	"queryorchestration/internal/query"
-	textextraction "queryorchestration/internal/text_extraction"
 	"strings"
 	"testing"
 
@@ -46,11 +45,11 @@ func TestUpdateJobCollector(t *testing.T) {
 	rec := httptest.NewRecorder()
 	ctx := e.NewContext(req, rec)
 
+	extract := documenttext.New()
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		JobCollector: collector.New(db, &collector.Services{
-			Query:          query.New(db),
-			DocumentClean:  documentclean.New(),
-			TextExtraction: textextraction.New(),
+		Collector: collector.New(db, &collector.Services{
+			Clean: documentclean.New(),
+			Text:  extract,
 		}),
 	})
 
@@ -66,7 +65,7 @@ func TestUpdateJobCollector(t *testing.T) {
 	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
-				AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), &current.MinCleanVersion, &current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
+				AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
 		)
 	pool.ExpectBeginTx(pgx.TxOptions{})
 	pool.ExpectExec("name: UpdateCollector :exec").WithArgs(int32(5), int32(2), database.MustToDBUUID(current.ID)).
@@ -95,11 +94,13 @@ func TestGetJobCollectorByJobId(t *testing.T) {
 	rec := httptest.NewRecorder()
 	ctx := e.NewContext(req, rec)
 
+	extract := documenttext.New()
 	svc := collector.New(db, &collector.Services{
-		Query: query.New(db),
+		Clean: documentclean.New(),
+		Text:  extract,
 	})
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		JobCollector: svc,
+		Collector: svc,
 	})
 
 	coll := collector.Collector{
@@ -112,7 +113,7 @@ func TestGetJobCollectorByJobId(t *testing.T) {
 	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(coll.JobID)).
 		WillReturnRows(
 			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
-				AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), &coll.MinCleanVersion, &coll.MinTextVersion, int32(1), int32(2), []byte("")),
+				AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
 		)
 
 	err = cons.GetJobCollectorByJobId(ctx, coll.JobID)

api/queryService/parse.go

@@ -3,7 +3,7 @@ package queryservice
 import (
 	"errors"
 	"queryorchestration/internal/query"
-	queryprocessor "queryorchestration/internal/query/processor"
+	resultprocessor "queryorchestration/internal/query/result/processor"
 
 	"github.com/google/uuid"
 )
@@ -44,25 +44,25 @@ func parseQuery(query *query.Query) (*Query, error) {
 	return q, nil
 }
 
-func parseQueryType(qType queryprocessor.Type) (QueryType, error) {
+func parseQueryType(qType resultprocessor.Type) (QueryType, error) {
 	switch qType {
-	case queryprocessor.TypeJsonExtractor:
+	case resultprocessor.TypeJsonExtractor:
 		return JSONEXTRACTOR, nil
-	case queryprocessor.TypeContextFull:
+	case resultprocessor.TypeContextFull:
 		return CONTEXTFULL, nil
 	default:
 		return "", errors.New("invalid query type")
 	}
 }
 
-func parseSpecQueryType(qType QueryType) (queryprocessor.Type, error) {
+func parseSpecQueryType(qType QueryType) (resultprocessor.Type, error) {
 	switch qType {
 	case JSONEXTRACTOR:
-		return queryprocessor.TypeJsonExtractor, nil
+		return resultprocessor.TypeJsonExtractor, nil
 	case CONTEXTFULL:
-		return queryprocessor.TypeContextFull, nil
+		return resultprocessor.TypeContextFull, nil
 	default:
-		return queryprocessor.Type(-1), errors.New("invalid query type")
+		return resultprocessor.Type(-1), errors.New("invalid query type")
 	}
 }
 

api/queryService/parse_test.go

@@ -2,7 +2,7 @@ package queryservice
 
 import (
 	"queryorchestration/internal/query"
-	queryprocessor "queryorchestration/internal/query/processor"
+	resultprocessor "queryorchestration/internal/query/result/processor"
 	"testing"
 
 	"github.com/google/uuid"
@@ -15,7 +15,7 @@ func TestParseQueries(t *testing.T) {
 	in := []*query.Query{
 		{
 			ID:            uuid.New(),
-			Type:          queryprocessor.TypeContextFull,
+			Type:          resultprocessor.TypeContextFull,
 			ActiveVersion: 1,
 			LatestVersion: 2,
 			RequiredQueryIDs: &[]uuid.UUID{
@@ -45,7 +45,7 @@ func TestParseQuery(t *testing.T) {
 	cfg := "hey"
 	in := &query.Query{
 		ID:            uuid.New(),
-		Type:          queryprocessor.TypeContextFull,
+		Type:          resultprocessor.TypeContextFull,
 		ActiveVersion: 1,
 		LatestVersion: 2,
 		RequiredQueryIDs: &[]uuid.UUID{
@@ -71,7 +71,7 @@ func TestParseQuery(t *testing.T) {
 func TestParseQueryMinimal(t *testing.T) {
 	in := &query.Query{
 		ID:            uuid.New(),
-		Type:          queryprocessor.TypeContextFull,
+		Type:          resultprocessor.TypeContextFull,
 		ActiveVersion: 1,
 		LatestVersion: 2,
 	}
@@ -87,26 +87,26 @@ func TestParseQueryMinimal(t *testing.T) {
 }
 
 func TestParseQueryType(t *testing.T) {
-	qt, err := parseQueryType(queryprocessor.TypeContextFull)
+	qt, err := parseQueryType(resultprocessor.TypeContextFull)
 	assert.Nil(t, err)
 	assert.Equal(t, CONTEXTFULL, qt)
 
-	qt, err = parseQueryType(queryprocessor.TypeJsonExtractor)
+	qt, err = parseQueryType(resultprocessor.TypeJsonExtractor)
 	assert.Nil(t, err)
 	assert.Equal(t, JSONEXTRACTOR, qt)
 
-	_, err = parseQueryType(queryprocessor.Type(-1))
+	_, err = parseQueryType(resultprocessor.Type(-1))
 	assert.NotNil(t, err)
 }
 
 func TestParseSpecQueryType(t *testing.T) {
 	qt, err := parseSpecQueryType(CONTEXTFULL)
 	assert.Nil(t, err)
-	assert.Equal(t, queryprocessor.Type(queryprocessor.TypeContextFull), qt)
+	assert.Equal(t, resultprocessor.Type(resultprocessor.TypeContextFull), qt)
 
 	qt, err = parseSpecQueryType(JSONEXTRACTOR)
 	assert.Nil(t, err)
-	assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), qt)
+	assert.Equal(t, resultprocessor.Type(resultprocessor.TypeJsonExtractor), qt)
 
 	_, err = parseSpecQueryType("invalid")
 	assert.NotNil(t, err)

api/queryService/query.go

@@ -4,16 +4,14 @@ import (
 	"fmt"
 	"net/http"
 	"queryorchestration/internal/query"
-	queryprocessor "queryorchestration/internal/query/processor"
+	resultprocessor "queryorchestration/internal/query/result/processor"
 
 	"github.com/labstack/echo/v4"
 	"github.com/oapi-codegen/runtime/types"
 )
 
 func (s *Controllers) ListQueries(ctx echo.Context) error {
-	filters := query.ListFilters{}
-
-	queries, err := s.svc.Query.List(ctx.Request().Context(), filters)
+	queries, err := s.svc.Query.List(ctx.Request().Context())
 	if err != nil {
 		return echo.NewHTTPError(http.StatusNotFound, fmt.Sprintf("Unable to list query: %s", err))
 	}
@@ -53,7 +51,7 @@ func (s *Controllers) CreateQuery(ctx echo.Context) error {
 		return echo.NewHTTPError(http.StatusBadRequest, "Invalid Query Type")
 	}
 
-	id, err := s.svc.Query.Create(ctx.Request().Context(), &queryprocessor.Create{
+	id, err := s.svc.Query.Create(ctx.Request().Context(), &resultprocessor.Create{
 		Type:             qt,
 		RequiredQueryIDs: req.RequiredQueries,
 		Config:           req.Config,
@@ -73,7 +71,7 @@ func (s *Controllers) UpdateQuery(ctx echo.Context, id types.UUID) error {
 		return echo.NewHTTPError(http.StatusBadRequest, err)
 	}
 
-	err := s.svc.Query.Update(ctx.Request().Context(), &queryprocessor.Update{
+	err := s.svc.Query.Update(ctx.Request().Context(), &resultprocessor.Update{
 		ActiveVersion:    req.ActiveVersion,
 		Config:           req.Config,
 		RequiredQueryIDs: req.RequiredQueries,

api/queryService/query_test.go

@@ -8,7 +8,12 @@ import (
 	queryservice "queryorchestration/api/queryService"
 	"queryorchestration/internal/database"
 	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	documentclean "queryorchestration/internal/document/clean"
+	documenttext "queryorchestration/internal/document/text"
+	"queryorchestration/internal/job/collector"
 	"queryorchestration/internal/query"
+	"queryorchestration/internal/query/result"
 	"strings"
 	"testing"
 
@@ -33,7 +38,7 @@ func TestCreateQuery(t *testing.T) {
 	}
 
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		Query: query.New(db),
+		Query: query.New(db, &query.Services{}),
 	})
 
 	body := queryservice.QueryCreate{
@@ -75,7 +80,7 @@ func TestListQueries(t *testing.T) {
 	}
 
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		Query: query.New(db),
+		Query: query.New(db, &query.Services{}),
 	})
 
 	e := echo.New()
@@ -85,7 +90,7 @@ func TestListQueries(t *testing.T) {
 
 	id := uuid.New()
 
-	pool.ExpectQuery("-- name: ListQueries :many").WithArgs().WillReturnRows(
+	pool.ExpectQuery("name: ListQueries :many").WithArgs().WillReturnRows(
 		pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
 			AddRow(database.MustToDBUUID(id), repository.QuerytypeContextFull, int32(1), int32(2), []byte(""), []pgtype.UUID{}),
 	)
@@ -120,7 +125,7 @@ func TestGetQuery(t *testing.T) {
 	}
 
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		Query: query.New(db),
+		Query: query.New(db, &query.Services{}),
 	})
 
 	e := echo.New()
@@ -164,7 +169,7 @@ func TestUpdateQuery(t *testing.T) {
 	}
 
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		Query: query.New(db),
+		Query: query.New(db, &query.Services{}),
 	})
 
 	av := int32(2)
@@ -211,12 +216,38 @@ func TestTestQuery(t *testing.T) {
 		Pool:    pool,
 	}
 
+	docsvc := document.New(db)
+	col := collector.New(db, &collector.Services{
+		Clean: documentclean.New(),
+		Text:  documenttext.New(),
+	})
 	cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
-		Query: query.New(db),
+		Collector: col,
+		Query: query.New(db, &query.Services{
+			Document:  docsvc,
+			Collector: col,
+			Result:    result.New(db),
+		}),
 	})
 
+	coll := collector.Collector{
+		ID:    uuid.New(),
+		JobID: uuid.New(),
+	}
+	doc := document.Document{
+		ID:       uuid.New(),
+		JobID:    coll.JobID,
+		Hash:     "example_hash",
+		Location: "example_location",
+	}
+	params := &query.Test{
+		QueryID:      uuid.New(),
+		DocumentID:   doc.ID,
+		QueryVersion: int32(3),
+	}
 	body := queryservice.QueryTestRequest{
-		DocumentId: uuid.New(),
+		DocumentId:   params.DocumentID,
+		QueryVersion: params.QueryVersion,
 	}
 	bodyBytes, err := json.Marshal(body)
 	assert.Nil(t, err)
@@ -227,9 +258,32 @@ func TestTestQuery(t *testing.T) {
 	rec := httptest.NewRecorder()
 	ctx := e.NewContext(req, rec)
 
-	id := uuid.New()
+	reqID := database.MustToDBUUID(uuid.New())
+	pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
+				AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
+		)
+	pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
+				AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(doc.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
+		)
+	pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows(
+		pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
+			AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}),
+	)
+	pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"queryId", "value", "type"}).
+				AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull),
+		)
+	pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows(
+		pgxmock.NewRows([]string{"id", "config"}).
+			AddRow(pgtype.UUID{}, []byte("{\"path\":\"oldkey\"}")),
+	)
 
-	err = cons.TestQuery(ctx, id)
+	err = cons.TestQuery(ctx, params.QueryID)
 	assert.Nil(t, err)
 	assert.Equal(t, http.StatusOK, rec.Code)
 
@@ -237,6 +291,6 @@ func TestTestQuery(t *testing.T) {
 	err = json.Unmarshal(rec.Body.Bytes(), &res)
 	assert.Nil(t, err)
 	assert.EqualExportedValues(t, queryservice.QueryTestResponse{
-		Value: "",
+		Value: "old_value",
 	}, res)
 }