jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobsynced (pull request #72)

Browse Source 
Job Status Get and DB tidy up

* initalquery

* tests

* shorttests

* testing queries

* job

* solvedthequery

* updatingdb

* fixingtests

* repotests

* shorttests

* docker

* testspassed
This commit is contained in:
Michael McGuinness
2025-02-20 19:02:44 +00:00
parent 0ea544926b
commit 3d434eedb8
74 changed files with 2395 additions and 1335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/query/result/get.go
+11 -10
View File
@@ -2,6 +2,7 @@ package result
import (
"context"
"errors"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
@@ -13,27 +14,27 @@ import (
)
type GetValueWithVersionParams struct {
Type resultprocessor.Type
QueryID uuid.UUID
DocumentID uuid.UUID
QueryVersion int32
MinCleanVersion int32
MinTextVersion int32
Type resultprocessor.Type
QueryID uuid.UUID
DocumentID uuid.UUID
QueryVersion int32
}
func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) {
res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: database.MustToDBUUID(params.QueryID),
Queryversion: params.QueryVersion,
Queryversion: &params.QueryVersion,
Documentid: database.MustToDBUUID(params.DocumentID),
Cleanversion: params.MinCleanVersion,
Textversion: params.MinTextVersion,
})
if err != nil {
return nil, err
}
return getValueByType(params.Type, res.Value)
if res.Value == nil {
return nil, errors.New("no value found")
}
return getValueByType(params.Type, *res.Value)
}
func getValueByType(t resultprocessor.Type, value string) (resultprocessor.Value, error) {
internal/query/result/get_test.go
+9 -10
View File
@@ -42,23 +42,22 @@ func TestGetValueWithVersion(t *testing.T) {
svc := New(cfg, &Services{})
params := &GetValueWithVersionParams{
Type: resultprocessor.TypeJsonExtractor,
QueryID: uuid.New(),
DocumentID: uuid.New(),
QueryVersion: 1,
MinCleanVersion: 2,
MinTextVersion: 3,
Type: resultprocessor.TypeJsonExtractor,
QueryID: uuid.New(),
DocumentID: uuid.New(),
QueryVersion: 1,
}
pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), params.QueryVersion, database.MustToDBUUID(params.DocumentID), params.MinCleanVersion, params.MinTextVersion).
value := "exaple_value"
pool.ExpectQuery("name: GetResultValueWithVersion :one").WithArgs(database.MustToDBUUID(params.QueryID), &params.QueryVersion, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "value"}).
AddRow(pgtype.UUID{}, "example_value"),
AddRow(pgtype.UUID{}, &value),
)
val, err := svc.GetValueWithVersion(ctx, params)
assert.NoError(t, err)
v := jsonextractor.NewResult("example_value")
v := jsonextractor.NewResult(value)
assert.Equal(t, v, val)
assert.Equal(t, "example_value", v.GetStoreValue())
assert.Equal(t, value, v.GetStoreValue())
}
internal/query/result/process.go
+2 -2
View File
@@ -71,7 +71,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process, query *res
qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: database.MustToDBUUID(p.QueryID),
Documentid: database.MustToDBUUID(p.DocumentID),
Version: p.QueryVersion,
Version: &p.QueryVersion,
})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
@@ -92,7 +92,7 @@ func parseQueryRequirementValueArray(v []*repository.ListQueryRequirementValuesR
return nil, err
}
cleanValue, err := getValueByType(qType, r.Value)
cleanValue, err := getValueByType(qType, *r.Value)
if err != nil {
return nil, err
}
internal/query/result/process_test.go
+16 -13
View File
@@ -11,7 +11,6 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -51,14 +50,15 @@ func TestProcess(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
strVal := `{"examplekey":"example_value"}`
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID(query.ID), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(qcfg)),
)
val, err := svc.Process(ctx, &params)
@@ -109,16 +109,18 @@ func TestListRequiredValue(t *testing.T) {
QueryVersion: query.Version,
}
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
strVal := "axe_value"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, "example_value"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(uuid.New()), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeJsonExtractor, &strVal),
)
pr, err = svc.listRequiredValues(ctx, &params, query)
assert.NoError(t, err)
assert.ElementsMatch(t, []resultprocessor.Value{
jsonextractor.NewResult("example_value"),
jsonextractor.NewResult(strVal),
}, pr)
}
@@ -138,10 +140,11 @@ func TestGetProcessor(t *testing.T) {
}
func TestParseQueryRequirementValueArray(t *testing.T) {
exval := "exampleval"
in := []*repository.ListQueryRequirementValuesRow{
{
Queryid: database.MustToDBUUID(uuid.New()),
Value: "example_value",
Value: &exval,
Type: repository.QuerytypeJsonExtractor,
},
}
@@ -149,6 +152,6 @@ func TestParseQueryRequirementValueArray(t *testing.T) {
out, err := parseQueryRequirementValueArray(in)
assert.NoError(t, err)
assert.ElementsMatch(t, []resultprocessor.Value{
jsonextractor.NewResult("example_value"),
jsonextractor.NewResult(exval),
}, out)
}
internal/query/result/processor/parse_test.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
func TestParseDBCollectorQuery(t *testing.T) {
dbResult := repository.Collectorquerydependencytree{
Collectorid: pgtype.UUID{},
Jobid: pgtype.UUID{},
Queryid: pgtype.UUID{},
Requiredids: []pgtype.UUID{},
Type: repository.QuerytypeJsonExtractor,
internal/query/result/set/set.go
+35 -14
View File
@@ -2,6 +2,8 @@ package resultset
import (
"context"
"database/sql"
"errors"
"log/slog"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
@@ -17,11 +19,6 @@ type Set struct {
func (s *Service) Set(ctx context.Context, params *Set) error {
dbid := database.MustToDBUUID(params.DocumentID)
cleanVersion, err := s.cfg.GetDBQueries().GetDocumentCleanEntry(ctx, dbid)
if err != nil {
return err
}
textVersion, err := s.cfg.GetDBQueries().GetDocumentTextEntry(ctx, dbid)
if err != nil {
return err
@@ -41,13 +38,37 @@ func (s *Service) Set(ctx context.Context, params *Set) error {
return err
}
err = s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{
Queryid: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Value: value.GetStoreValue(),
Cleanversion: cleanVersion.Version,
Textversion: textVersion.Version,
Queryversion: query.ActiveVersion,
err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
id, err := q.AddResult(ctx, &repository.AddResultParams{
Queryid: database.MustToDBUUID(params.QueryID),
Value: value.GetStoreValue(),
Queryversion: query.ActiveVersion,
Textentryid: textVersion.ID,
})
if err != nil {
return err
}
qResults, err := q.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Version: &query.ActiveVersion,
})
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
for _, result := range qResults {
err = q.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: id,
Requiredresultid: result.ID,
})
if err != nil {
return err
}
}
return nil
})
if err != nil {
return err
@@ -60,8 +81,8 @@ func (s *Service) Set(ctx context.Context, params *Set) error {
func (s *Service) informQueryDependents(ctx context.Context, params *Set) error {
ids, err := s.cfg.GetDBQueries().ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: database.MustToDBUUID(params.DocumentID),
Requiredids: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Queryid: database.MustToDBUUID(params.QueryID),
})
if err != nil {
return err
internal/query/result/set/set_test.go
+25 -15
View File
@@ -16,7 +16,6 @@ import (
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
@@ -65,15 +64,11 @@ func TestSet(t *testing.T) {
QueryID: query.ID,
}
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
)
textEntryId := uuid.New()
pool.ExpectQuery("name: GetDocumentTextEntry :one").WithArgs(database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key", "version"}).
AddRow(database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1)),
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "cleanEntryId"}).
AddRow(database.MustToDBUUID(textEntryId), database.MustToDBUUID(params.DocumentID), "buket", "/i/am/here", int32(1), database.MustToDBUUID(uuid.New())),
)
pool.ExpectQuery("name: GetQuery :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
@@ -83,17 +78,32 @@ func TestSet(t *testing.T) {
pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).
AddRow(database.MustToDBUUID(query.ID), repository.QuerytypeJsonExtractor, query.Version, query.Version, []byte(*query.Config), database.MustToDBUUIDArray(*query.RequiredQueryIDs)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(params.DocumentID), database.MustToDBUUID(query.ID), query.Version).
requiredResultId := uuid.New()
strVal := "{\"examplekey\":\"example_value\"}"
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId", "type", "value"}).
AddRow(database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, "{\"examplekey\":\"example_value\"}"),
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID), query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(qcfg)),
pool.ExpectQuery("name: GetActiveQueryConfig :one").WithArgs(database.MustToDBUUID(query.ID)).WillReturnRows(
pgxmock.NewRows([]string{"config"}).
AddRow([]byte(qcfg)),
)
pool.ExpectExec("name: SetResult :exec").WithArgs(database.MustToDBUUID(query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), int32(1), int32(1), query.Version).
pool.ExpectBegin()
resultId := uuid.New()
pool.ExpectQuery("name: AddResult :one").WithArgs(database.MustToDBUUID(query.ID), pgxmock.AnyArg(), database.MustToDBUUID(textEntryId), query.Version).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(resultId)),
)
pool.ExpectQuery("name: ListQueryRequirementValues :many").WithArgs(database.MustToDBUUID(query.ID), &query.Version, database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "type", "value"}).
AddRow(database.MustToDBUUID(requiredResultId), database.MustToDBUUID((*query.RequiredQueryIDs)[0]), repository.QuerytypeContextFull, &strVal),
)
pool.ExpectExec("name: AddResultDependency :exec").WithArgs(database.MustToDBUUID(resultId), database.MustToDBUUID(requiredResultId)).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
pool.ExpectQuery("name: ListQueryDirectDependentsByDocumentID :many").WithArgs(database.MustToDBUUID(params.QueryID), database.MustToDBUUID(params.DocumentID)).
WillReturnRows(
pgxmock.NewRows([]string{"queryId"}).