jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/queryRunner/runner.go
+7 -4
View File
@@ -3,7 +3,7 @@ package queryrunner
import (
"context"
"encoding/json"
"queryorchestration/internal/query"
resultset "queryorchestration/internal/query/result/set"
"github.com/go-playground/validator/v10"
@@ -13,7 +13,7 @@ import (
const Name = "queryRunner"
type Services struct {
Query *query.Service
ResultSet *resultset.Service
}
type Runner struct {
@@ -29,7 +29,7 @@ func New(validator *validator.Validate, svc *Services) Runner {
}
func (s *Runner) Process(ctx context.Context, req *types.Message) error {
var body query.Document
var body resultset.Set
err := json.Unmarshal([]byte(*req.Body), &body)
if err != nil {
return err
@@ -40,7 +40,10 @@ func (s *Runner) Process(ctx context.Context, req *types.Message) error {
return err
}
err = s.svc.Query.Sync(ctx, &body)
err = s.svc.ResultSet.Set(ctx, &resultset.Set{
DocumentID: body.DocumentID,
QueryID: body.QueryID,
})
if err != nil {
return err
}
api/queryRunner/runner_test.go
+83 -20
View File
@@ -3,23 +3,34 @@ package queryrunner_test
import (
"context"
"encoding/json"
controllers "queryorchestration/api/queryRunner"
"fmt"
queryrunner "queryorchestration/api/queryRunner"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
"queryorchestration/internal/serviceconfig"
resultprocessor "queryorchestration/internal/query/result/processor"
resultset "queryorchestration/internal/query/result/set"
"queryorchestration/internal/server/runner"
queryc "queryorchestration/internal/serviceconfig/queue/query"
queuemock "queryorchestration/mocks/queue"
"testing"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
type QueryConfig struct {
runner.BaseConfig
queryc.QueryConfig
}
func TestQueryRunner(t *testing.T) {
ctx := context.Background()
@@ -28,26 +39,27 @@ func TestQueryRunner(t *testing.T) {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &serviceconfig.BaseConfig{}
cfg := &QueryConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockSQS := queuemock.NewMockSQSClient(t)
cfg.QueueClient = mockSQS
cfg.QueryURL = "/i/am/here"
svc := query.New(cfg, &query.Services{
Result: result.New(cfg),
Collector: collector.New(cfg, &collector.Services{}),
Document: document.New(cfg),
})
runner := controllers.New(validator.New(), &controllers.Services{
Query: svc,
que := query.New(cfg)
runner := queryrunner.New(validator.New(), &queryrunner.Services{
ResultSet: resultset.New(cfg, &resultset.Services{
Query: que,
Result: result.New(cfg, &result.Services{
Query: que,
}),
}),
})
assert.NotNil(t, runner)
doc := query.Document{
ID: uuid.New(),
JobID: uuid.New(),
CleanVersion: 1,
TextVersion: 1,
doc := resultset.Set{
DocumentID: uuid.New(),
QueryID: uuid.New(),
}
bodyBytes, err := json.Marshal(doc)
assert.NoError(t, err)
@@ -56,10 +68,61 @@ func TestQueryRunner(t *testing.T) {
Body: &body,
}
pool.ExpectQuery("name: ListUnsyncedQueriesByDocId :many").WithArgs(database.MustToDBUUID(doc.ID)).
qcfg := "{\"path\":\"examplekey\"}"
query := &resultprocessor.Query{
ID: doc.QueryID,
Version: 2,
RequiredQueryIDs: &[]uuid.UUID{uuid.New()},
Config: &qcfg,
}
params := &resultset.Set{
DocumentID: doc.DocumentID,
}
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}),
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
)
pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
)
pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: GetQueryWithVersion :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
)
pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(doc.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0])),
)
mockSQS.EXPECT().
SendMessage(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.QueryURL && *in.MessageBody == fmt.Sprintf("{\"document_id\":\"%s\",\"query_id\":\"%s\"}", params.DocumentID.String(), (*query.RequiredQueryIDs)[0].String())
}),
mock.Anything,
).
Return(&sqs.SendMessageOutput{}, nil)
err = runner.Process(ctx, msg)
assert.NoError(t, err)