jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/queryService/controllers.go
+2
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/job"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query"
querytest "queryorchestration/internal/query/test"
"github.com/go-playground/validator/v10"
)
@@ -18,6 +19,7 @@ type Services struct {
Query *query.Service
Client *client.Service
Job *job.Service
QueryTest *querytest.Service
}
type Controllers struct {
api/queryService/jobcollector.go
+3 -2
View File
@@ -44,10 +44,11 @@ func (s *Controllers) UpdateJobCollectorByJobId(ctx echo.Context, jobId types.UU
var fields *map[string]uuid.UUID
if req.Fields != nil {
fields := &map[string]uuid.UUID{}
fs := map[string]uuid.UUID{}
for _, field := range *req.Fields {
(*fields)[field.Name] = field.QueryId
fs[field.Name] = field.QueryId
}
fields = &fs
}
err := s.svc.Collector.UpdateByJobId(ctx.Request().Context(), &collector.UpdateParams{
api/queryService/jobcollector_test.go
+21 -7
View File
@@ -16,6 +16,7 @@ import (
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/labstack/echo/v4"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
@@ -30,9 +31,22 @@ func TestUpdateJobCollector(t *testing.T) {
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 4,
}
av := int32(2)
body := queryservice.JobCollectorUpdate{
ActiveVersion: &av,
Fields: &[]queryservice.JobCollectorField{
{
Name: "a",
QueryId: uuid.New(),
},
},
}
bodyBytes, err := json.Marshal(body)
assert.NoError(t, err)
@@ -49,13 +63,6 @@ func TestUpdateJobCollector(t *testing.T) {
}),
})
current := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
ActiveVersion: 1,
LatestVersion: 4,
}
ctx.Set("id", current.JobID)
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(current.JobID)).
@@ -63,7 +70,14 @@ func TestUpdateJobCollector(t *testing.T) {
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(current.ID), database.MustToDBUUID(current.JobID), current.MinCleanVersion, current.MinTextVersion, current.ActiveVersion, current.LatestVersion, []byte("")),
)
pool.ExpectQuery("name: AllQueriesExist :one").WithArgs([]pgtype.UUID{database.MustToDBUUID((*body.Fields)[0].QueryId)}).
WillReturnRows(
pgxmock.NewRows([]string{"exist"}).
AddRow(true),
)
pool.ExpectBeginTx(pgx.TxOptions{})
pool.ExpectExec("name: AddCollectorQuery :exec").WithArgs(database.MustToDBUUID(current.ID), "a", database.MustToDBUUID((*body.Fields)[0].QueryId), int32(5)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectExec("name: UpdateCollector :exec").WithArgs(int32(5), int32(2), database.MustToDBUUID(current.ID)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
api/queryService/query.go
+2 -2
View File
@@ -3,7 +3,7 @@ package queryservice
import (
"fmt"
"net/http"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
resultprocessor "queryorchestration/internal/query/result/processor"
"github.com/labstack/echo/v4"
@@ -90,7 +90,7 @@ func (s *Controllers) TestQuery(ctx echo.Context, id types.UUID) error {
return echo.NewHTTPError(http.StatusBadRequest, err)
}
value, err := s.svc.Query.Test(ctx.Request().Context(), query.Test{
value, err := s.svc.QueryTest.Test(ctx.Request().Context(), result.Process{
QueryID: id,
QueryVersion: req.QueryVersion,
DocumentID: req.DocumentId,
api/queryService/query_test.go
+15 -10
View File
@@ -12,6 +12,7 @@ import (
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
querytest "queryorchestration/internal/query/test"
"queryorchestration/internal/serviceconfig"
"strings"
"testing"
@@ -35,7 +36,7 @@ func TestCreateQuery(t *testing.T) {
cfg.DBQueries = repository.New(pool)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
Query: query.New(cfg, &query.Services{}),
Query: query.New(cfg),
})
body := queryservice.QueryCreate{
@@ -75,7 +76,7 @@ func TestListQueries(t *testing.T) {
cfg.DBQueries = repository.New(pool)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
Query: query.New(cfg, &query.Services{}),
Query: query.New(cfg),
})
e := echo.New()
@@ -118,7 +119,7 @@ func TestGetQuery(t *testing.T) {
cfg.DBQueries = repository.New(pool)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
Query: query.New(cfg, &query.Services{}),
Query: query.New(cfg),
})
e := echo.New()
@@ -160,7 +161,7 @@ func TestUpdateQuery(t *testing.T) {
cfg.DBQueries = repository.New(pool)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
Query: query.New(cfg, &query.Services{}),
Query: query.New(cfg),
})
av := int32(2)
@@ -209,12 +210,16 @@ func TestTestQuery(t *testing.T) {
col := collector.New(cfg, &collector.Services{
Document: docsvc,
})
que := query.New(cfg)
cons := queryservice.NewControllers(validator.New(), &queryservice.Services{
Collector: col,
Query: query.New(cfg, &query.Services{
Query: que,
QueryTest: querytest.New(cfg, &querytest.Services{
Document: docsvc,
Collector: col,
Result: result.New(cfg),
Result: result.New(cfg, &result.Services{
Query: que,
}),
}),
})
@@ -227,7 +232,7 @@ func TestTestQuery(t *testing.T) {
JobID: coll.JobID,
Hash: "example_hash",
}
params := &query.Test{
params := &result.Process{
QueryID: uuid.New(),
DocumentID: doc.ID,
QueryVersion: int32(3),
@@ -260,10 +265,10 @@ func TestTestQuery(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(params.QueryID), repository.QuerytypeJsonExtractor, int32(1), params.QueryVersion+1, []byte("{\"path\":\"oldkey\"}"), []pgtype.UUID{reqID}),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), coll.MinCleanVersion, coll.MinTextVersion).
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(params.QueryID), params.QueryVersion).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "value", "type"}).
AddRow(reqID, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}", repository.QuerytypeContextFull),
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(reqID, repository.QuerytypeContextFull, "{\"mykey\":\"example_value\",\"oldkey\":\"old_value\"}"),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).