Merged in feature/serviceconfig-integration (pull request #38)

DRAFT PR : WIP working through ideas for integration

* movearound

* attempttwo

* openapi

* further sanding

* fix

* start on tests

* runthroughsingleconfig

* somechanges

* reflectissue

* removeerrs

* mostlyremovepanic

* removeenv

* noncfgtests

* go

* repo

* fix service config test

* add PWD to all

* test fix

* fix lint

* todo for later

* passingunittests

* alltests

* testlogger

* testloggername

* clean
This commit is contained in:
Jay Brown
2025-01-31 13:43:55 +00:00
parent 058f8dba7f
commit 15adaebfcd
138 changed files with 1864 additions and 1695 deletions
+3 -3
View File
@@ -58,7 +58,7 @@ func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Crea
}
var dbID pgtype.UUID
err = database.ExecuteTransaction(ctx, s.db, func(ctx context.Context, qtx *repository.Queries) error {
err = s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, qtx *repository.Queries) error {
dbID, err = qtx.CreateQuery(ctx, query.Type)
if err != nil {
return err
@@ -106,9 +106,9 @@ func (s *Service) submitCreate(ctx context.Context, entity *resultprocessor.Crea
func (s *Service) getCreator(qType resultprocessor.Type) (resultprocessor.Creator, error) {
switch qType {
case resultprocessor.TypeJsonExtractor:
return jsonextractor.NewCreator(s.db), nil
return jsonextractor.NewCreator(), nil
case resultprocessor.TypeContextFull:
return contextfull.NewCreator(s.db), nil
return contextfull.NewCreator(), nil
default:
return nil, fmt.Errorf("attempting to process invalid query type")
}
+13 -18
View File
@@ -7,6 +7,7 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -22,12 +23,10 @@ func TestCreate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
q := query.Query{
@@ -76,12 +75,10 @@ func TestCreateMinimal(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
q := query.Query{
ID: uuid.New(),
@@ -113,12 +110,10 @@ func TestCreateRollback(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
create := &resultprocessor.Create{
Type: resultprocessor.TypeJsonExtractor,
+17 -24
View File
@@ -5,6 +5,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -18,12 +19,10 @@ func TestGetCreator(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
queryType := resultprocessor.Type(resultprocessor.TypeContextFull)
creator, err := svc.getCreator(queryType)
@@ -82,12 +81,10 @@ func TestSubmitCreate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
config := "{\"path\":\"example_path\"}"
q := Query{
@@ -132,12 +129,10 @@ func TestSubmitCreateNoReqsOrConfig(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
q := Query{
ID: uuid.New(),
@@ -169,12 +164,10 @@ func TestNormalizeCreate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
create := &resultprocessor.Create{
Type: resultprocessor.TypeJsonExtractor,
+2 -2
View File
@@ -19,7 +19,7 @@ type Query struct {
}
func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int32) (*Query, error) {
query, err := s.db.Queries.GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
query, err := s.cfg.GetDBQueries().GetQueryWithVersion(ctx, &repository.GetQueryWithVersionParams{
ID: database.MustToDBUUID(id),
Addedversion: version,
})
@@ -31,7 +31,7 @@ func (s *Service) GetWithVersion(ctx context.Context, id uuid.UUID, version int3
}
func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Query, error) {
query, err := s.db.Queries.GetQuery(ctx, database.MustToDBUUID(id))
query, err := s.cfg.GetDBQueries().GetQuery(ctx, database.MustToDBUUID(id))
if err != nil {
return nil, err
}
+9 -12
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -20,12 +21,10 @@ func TestGet(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
query := query.Query{
@@ -59,12 +58,10 @@ func TestGetWithVersion(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
query := query.Query{
+2 -2
View File
@@ -8,7 +8,7 @@ import (
)
func (s *Service) List(ctx context.Context) ([]*Query, error) {
dbQueries, err := s.db.Queries.ListQueries(ctx)
dbQueries, err := s.cfg.GetDBQueries().ListQueries(ctx)
if err != nil {
return nil, err
}
@@ -22,7 +22,7 @@ func (s *Service) List(ctx context.Context) ([]*Query, error) {
}
func (s *Service) ListById(ctx context.Context, ids []uuid.UUID) ([]*Query, error) {
dbQueries, err := s.db.Queries.ListQueriesById(ctx, database.MustToDBUUIDArray(ids))
dbQueries, err := s.cfg.GetDBQueries().ListQueriesById(ctx, database.MustToDBUUIDArray(ids))
if err != nil {
return nil, err
}
+9 -12
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -21,12 +22,10 @@ func TestList(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
q := &query.Query{
@@ -60,12 +59,10 @@ func TestListById(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
q := &query.Query{
+1 -1
View File
@@ -73,7 +73,7 @@ func (s *Service) NormalizeQueryIDs(ctx context.Context, ids RequiredQueryIDs) e
dbids := database.MustToDBUUIDArray(dedup)
exist, err := s.db.Queries.AllQueriesExist(ctx, dbids)
exist, err := s.cfg.GetDBQueries().AllQueriesExist(ctx, dbids)
if err != nil {
return err
} else if !exist {
+5 -6
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -74,12 +75,10 @@ func TestNormalizeQueryIDs(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
s := Service{db: db}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
s := Service{cfg: cfg}
err = s.NormalizeQueryIDs(ctx, nil)
assert.Nil(t, err)
+1 -1
View File
@@ -22,7 +22,7 @@ type GetValueWithVersionParams struct {
}
func (s *Service) GetValueWithVersion(ctx context.Context, params *GetValueWithVersionParams) (resultprocessor.Value, error) {
res, err := s.db.Queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
res, err := s.cfg.GetDBQueries().GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: database.MustToDBUUID(params.QueryID),
Queryversion: params.QueryVersion,
Documentid: database.MustToDBUUID(params.DocumentID),
+6 -6
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -34,12 +35,11 @@ func TestGetValueWithVersion(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db)
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg)
params := &GetValueWithVersionParams{
Type: resultprocessor.TypeJsonExtractor,
+1 -1
View File
@@ -18,7 +18,7 @@ type ListQueryRequirementValuesParams struct {
}
func (s *Service) ListQueryRequirementValues(ctx context.Context, params *ListQueryRequirementValuesParams) (*[]resultprocessor.Value, error) {
qResults, err := s.db.Queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
qResults, err := s.cfg.GetDBQueries().ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: database.MustToDBUUID(params.QueryID),
Documentid: database.MustToDBUUID(params.DocumentID),
Addedversion: params.QueryVersion,
+5 -6
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -21,13 +22,11 @@ func TestListQueryRequirementValues(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
db: db,
cfg: cfg,
}
params := &ListQueryRequirementValuesParams{
+1 -1
View File
@@ -61,7 +61,7 @@ func (s *Service) listRequiredValues(ctx context.Context, p *Process) (*[]result
func (s *Service) getProcessor(queryType resultprocessor.Type) (resultprocessor.Processor, error) {
switch queryType {
case resultprocessor.TypeJsonExtractor:
return jsonextractor.NewExtractor(s.db), nil
return jsonextractor.NewExtractor(s.cfg), nil
case resultprocessor.TypeContextFull:
return contextfull.NewExtractor(), nil
default:
+12 -15
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -21,17 +22,15 @@ func TestProcess(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
db: db,
cfg: cfg,
}
cfg := "{\"path\":\"examplekey\"}"
qcfg := "{\"path\":\"examplekey\"}"
params := Process{
DocumentID: uuid.New(),
MinCleanVersion: 1,
@@ -39,7 +38,7 @@ func TestProcess(t *testing.T) {
Query: &resultprocessor.Query{
ID: uuid.New(),
Version: 2,
Config: &cfg,
Config: &qcfg,
RequiredQueryIDs: &[]uuid.UUID{
uuid.New(),
},
@@ -53,7 +52,7 @@ func TestProcess(t *testing.T) {
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(cfg)),
AddRow(pgtype.UUID{}, []byte(qcfg)),
)
val, err := svc.Process(ctx, &params)
@@ -69,14 +68,12 @@ func TestListRequiredValue(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
db: db,
cfg: cfg,
}
pr, err := svc.listRequiredValues(ctx, nil)
+4 -4
View File
@@ -1,7 +1,7 @@
package result
import (
"queryorchestration/internal/database"
"queryorchestration/internal/serviceconfig"
"github.com/google/uuid"
)
@@ -13,11 +13,11 @@ type Result struct {
}
type Service struct {
db *database.Connection
cfg serviceconfig.ConfigProvider
}
func New(db *database.Connection) *Service {
func New(cfg serviceconfig.ConfigProvider) *Service {
return &Service{
db,
cfg,
}
}
+5 -7
View File
@@ -1,9 +1,9 @@
package result_test
import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query/result"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/pashagolub/pgxmock/v3"
@@ -15,11 +15,9 @@ func TestService(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := result.New(db)
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := result.New(cfg)
assert.NotNil(t, svc)
}
+1 -1
View File
@@ -27,7 +27,7 @@ func (s *Service) Set(ctx context.Context, params *Set) (uuid.UUID, error) {
return uuid.Nil, err
}
dbId, err := s.db.Queries.SetResult(ctx, &repository.SetResultParams{
dbId, err := s.cfg.GetDBQueries().SetResult(ctx, &repository.SetResultParams{
Queryid: database.MustToDBUUID(params.Query.ID),
Documentid: database.MustToDBUUID(params.DocumentID),
Value: value.GetStoreValue(),
+8 -9
View File
@@ -5,6 +5,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -20,18 +21,16 @@ func TestSet(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
db: db,
cfg: cfg,
}
rid := uuid.New()
cfg := "{\"path\":\"examplekey\"}"
qcfg := "{\"path\":\"examplekey\"}"
params := Set{
DocumentID: uuid.New(),
CleanVersion: 1,
@@ -39,7 +38,7 @@ func TestSet(t *testing.T) {
Query: &resultprocessor.Query{
ID: uuid.New(),
Version: 2,
Config: &cfg,
Config: &qcfg,
RequiredQueryIDs: &[]uuid.UUID{
uuid.New(),
},
@@ -53,7 +52,7 @@ func TestSet(t *testing.T) {
)
pool.ExpectQuery("name: GetQueryConfig :one").WithArgs(database.MustToDBUUID(params.Query.ID), params.Query.Version).WillReturnRows(
pgxmock.NewRows([]string{"id", "config"}).
AddRow(pgtype.UUID{}, []byte(cfg)),
AddRow(pgtype.UUID{}, []byte(qcfg)),
)
pool.ExpectQuery("name: SetResult :one").WithArgs(database.MustToDBUUID(params.Query.ID), database.MustToDBUUID(params.DocumentID), pgxmock.AnyArg(), params.CleanVersion, params.TextVersion, params.Query.Version).
WillReturnRows(
+1 -1
View File
@@ -9,7 +9,7 @@ import (
)
func (s *Service) ListUnsyncedQueriesByDocId(ctx context.Context, id uuid.UUID) ([]*resultprocessor.Query, error) {
qs, err := s.db.Queries.ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id))
qs, err := s.cfg.GetDBQueries().ListUnsyncedQueriesByDocId(ctx, database.MustToDBUUID(id))
if err != nil {
return nil, err
}
+5 -6
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query/result"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -21,12 +22,10 @@ func TestListUnsyncedQueriesByDocId(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := result.New(db)
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := result.New(cfg)
documentId := uuid.New()
actualQs := []*resultprocessor.Query{
+4 -4
View File
@@ -1,11 +1,11 @@
package query
import (
"queryorchestration/internal/database"
"queryorchestration/internal/document"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/result"
"queryorchestration/internal/serviceconfig"
"github.com/google/uuid"
)
@@ -18,13 +18,13 @@ type Services struct {
}
type Service struct {
db *database.Connection
cfg serviceconfig.ConfigProvider
svc *Services
}
func New(db *database.Connection, svc *Services) *Service {
func New(cfg serviceconfig.ConfigProvider, svc *Services) *Service {
return &Service{
db,
cfg,
svc,
}
}
+5 -7
View File
@@ -1,9 +1,9 @@
package query_test
import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/pashagolub/pgxmock/v3"
@@ -15,11 +15,9 @@ func TestService(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
assert.NotNil(t, svc)
}
+9 -12
View File
@@ -7,6 +7,7 @@ import (
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/result"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -186,14 +187,12 @@ func TestProcessBatch(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
svc: &Services{
Result: result.New(db),
Result: result.New(cfg),
},
}
@@ -246,14 +245,12 @@ func TestSync(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
svc: &Services{
Result: result.New(db),
Result: result.New(cfg),
},
}
+8 -9
View File
@@ -10,6 +10,7 @@ import (
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query"
"queryorchestration/internal/query/result"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -25,23 +26,21 @@ func TestTest(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
text := documenttext.New()
clean := documentclean.New()
col := collector.New(db, &collector.Services{
col := collector.New(cfg, &collector.Services{
Text: text,
Clean: clean,
})
docsvc := document.New(db)
svc := query.New(db, &query.Services{
docsvc := document.New(cfg)
svc := query.New(cfg, &query.Services{
Text: text,
Document: docsvc,
Collector: col,
Result: result.New(db),
Result: result.New(cfg),
})
coll := collector.Collector{
+3 -5
View File
@@ -2,19 +2,17 @@ package contextfull
import (
"context"
"queryorchestration/internal/database"
resultprocessor "queryorchestration/internal/query/result/processor"
)
type Creator struct {
db *database.Connection
}
func NewCreator(db *database.Connection) Creator {
return Creator{db}
func NewCreator() *Creator {
return &Creator{}
}
func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
// TODO
// Type, RequiredQueryIDs, Config
return nil
@@ -2,10 +2,10 @@ package contextfull_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
contextfull "queryorchestration/internal/query/types/contextFull"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/pashagolub/pgxmock/v3"
@@ -19,13 +19,11 @@ func TestCreatorValidate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := contextfull.NewCreator(db)
svc := contextfull.NewCreator()
assert.NotNil(t, svc)
entity := &resultprocessor.Create{
+3 -3
View File
@@ -8,11 +8,11 @@ type Result struct {
value string
}
func NewResult(value string) Result {
return Result{value}
func NewResult(value string) *Result {
return &Result{value}
}
func (r Result) GetValue(ctx context.Context) (string, error) {
func (r *Result) GetValue(ctx context.Context) (string, error) {
// TODO - get value from s3
return r.value, nil
}
+3 -3
View File
@@ -9,11 +9,11 @@ import (
type Extractor struct {
}
func NewExtractor() Extractor {
return Extractor{}
func NewExtractor() *Extractor {
return &Extractor{}
}
func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) {
func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) {
if values != nil && len(*values) > 0 {
return "", errors.New("no requirements expected")
}
+5 -8
View File
@@ -2,19 +2,16 @@ package contextfull
import (
"context"
"queryorchestration/internal/database"
resultprocessor "queryorchestration/internal/query/result/processor"
)
type Updator struct {
db *database.Connection
type Updator struct{}
func NewUpdator() *Updator {
return &Updator{}
}
func NewUpdator(db *database.Connection) Updator {
return Updator{db}
}
func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
// TODO
// Type, RequiredQueryIDs, Config
return nil
@@ -2,10 +2,10 @@ package contextfull_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
contextfull "queryorchestration/internal/query/types/contextFull"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -20,13 +20,11 @@ func TestUpdatorValidate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := contextfull.NewUpdator(db)
svc := contextfull.NewUpdator()
assert.NotNil(t, svc)
current := &resultprocessor.Query{
@@ -2,19 +2,16 @@ package jsonextractor
import (
"context"
"queryorchestration/internal/database"
resultprocessor "queryorchestration/internal/query/result/processor"
)
type Creator struct {
db *database.Connection
type Creator struct{}
func NewCreator() *Creator {
return &Creator{}
}
func NewCreator(db *database.Connection) Creator {
return Creator{db}
}
func (s Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
func (s *Creator) Validate(ctx context.Context, entity *resultprocessor.Create) error {
// TODO
return nil
}
@@ -2,10 +2,10 @@ package jsonextractor_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/pashagolub/pgxmock/v3"
@@ -19,13 +19,11 @@ func TestCreatorValidate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := jsonextractor.NewCreator(db)
svc := jsonextractor.NewCreator()
assert.NotNil(t, svc)
entity := &resultprocessor.Create{
@@ -8,6 +8,7 @@ import (
resultprocessor "queryorchestration/internal/query/result/processor"
contextfull "queryorchestration/internal/query/types/contextFull"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -23,13 +24,11 @@ func TestJSONProcess(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
extractor := jsonextractor.NewExtractor(db)
extractor := jsonextractor.NewExtractor(cfg)
query := &resultprocessor.Query{
ID: uuid.New(),
@@ -93,13 +92,11 @@ func TestJSONProcessJSON(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
extractor := jsonextractor.NewExtractor(db)
extractor := jsonextractor.NewExtractor(cfg)
query := &resultprocessor.Query{
ID: uuid.New(),
@@ -178,13 +175,11 @@ func TestJSONProcessResults(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
extractor := jsonextractor.NewExtractor(db)
extractor := jsonextractor.NewExtractor(cfg)
query := &resultprocessor.Query{
ID: uuid.New(),
+3 -3
View File
@@ -8,11 +8,11 @@ type Result struct {
value string
}
func NewResult(value string) Result {
return Result{value}
func NewResult(value string) *Result {
return &Result{value}
}
func (r Result) GetValue(ctx context.Context) (string, error) {
func (r *Result) GetValue(ctx context.Context) (string, error) {
return r.value, nil
}
@@ -7,23 +7,24 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"github.com/tidwall/gjson"
)
type Extractor struct {
db *database.Connection
cfg serviceconfig.ConfigProvider
}
type Config struct {
Path string `json:"path"`
}
func NewExtractor(db *database.Connection) Extractor {
return Extractor{db}
func NewExtractor(cfg serviceconfig.ConfigProvider) *Extractor {
return &Extractor{cfg}
}
func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) {
func (e *Extractor) Process(ctx context.Context, query *resultprocessor.Query, values *[]resultprocessor.Value) (string, error) {
if values == nil || len(*values) != 1 {
return "", fmt.Errorf("JSON Extraction requires 1 result")
}
@@ -33,7 +34,7 @@ func (e Extractor) Process(ctx context.Context, query *resultprocessor.Query, va
return "", err
}
byteConfig, err := e.db.Queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
byteConfig, err := e.cfg.GetDBQueries().GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: database.MustToDBUUID(query.ID),
Addedversion: query.Version,
})
@@ -2,19 +2,17 @@ package jsonextractor
import (
"context"
"queryorchestration/internal/database"
resultprocessor "queryorchestration/internal/query/result/processor"
)
type Updator struct {
db *database.Connection
}
func NewUpdator(db *database.Connection) Updator {
return Updator{db}
func NewUpdator() *Updator {
return &Updator{}
}
func (s Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
func (s *Updator) Validate(ctx context.Context, current *resultprocessor.Query, entity *resultprocessor.Update) error {
// TODO
return nil
}
@@ -2,10 +2,10 @@ package jsonextractor_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
jsonextractor "queryorchestration/internal/query/types/jsonExtractor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -20,13 +20,11 @@ func TestUpdatorValidate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := jsonextractor.NewUpdator(db)
svc := jsonextractor.NewUpdator()
assert.NotNil(t, svc)
current := &resultprocessor.Query{
+55 -63
View File
@@ -44,7 +44,7 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *r
}
if entity.RequiredQueryIDs != nil {
createsloop, err := s.db.Queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
createsloop, err := s.cfg.GetDBQueries().IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: database.MustToDBUUID(current.ID),
ID: database.MustToDBUUIDArray(*entity.RequiredQueryIDs),
})
@@ -80,81 +80,73 @@ func (s *Service) normalizeUpdate(ctx context.Context, current *Query, entity *r
}
func (s *Service) submitUpdate(ctx context.Context, current *Query, entity *resultprocessor.Update) error {
tx, err := s.db.Pool.Begin(ctx)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback(ctx)
}()
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
latestVersion := current.LatestVersion + 1
id := database.MustToDBUUID(entity.ID)
qtx := s.db.Queries.WithTx(tx)
latestVersion := current.LatestVersion + 1
id := database.MustToDBUUID(entity.ID)
addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs)
for _, qID := range addIDs {
err = qtx.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: id,
Requiredqueryid: database.MustToDBUUID(qID),
Addedversion: latestVersion,
})
if err != nil {
return err
addIDs := getSetDifference(entity.RequiredQueryIDs, current.RequiredQueryIDs)
for _, qID := range addIDs {
err := q.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: id,
Requiredqueryid: database.MustToDBUUID(qID),
Addedversion: latestVersion,
})
if err != nil {
return err
}
}
}
removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs)
for _, qID := range removeIDs {
err = qtx.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{
Queryid: id,
Requiredqueryid: database.MustToDBUUID(qID),
Removedversion: &latestVersion,
})
if err != nil {
return err
removeIDs := getSetDifference(current.RequiredQueryIDs, entity.RequiredQueryIDs)
for _, qID := range removeIDs {
err := q.RemoveRequiredQuery(ctx, &repository.RemoveRequiredQueryParams{
Queryid: id,
Requiredqueryid: database.MustToDBUUID(qID),
Removedversion: &latestVersion,
})
if err != nil {
return err
}
}
}
if entity.Config != nil && *entity.Config != "" {
err = qtx.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{
Queryid: id,
Removedversion: &latestVersion,
if entity.Config != nil && *entity.Config != "" {
err := q.RemoveQueryConfig(ctx, &repository.RemoveQueryConfigParams{
Queryid: id,
Removedversion: &latestVersion,
})
if err != nil {
return err
}
err = q.AddQueryConfig(ctx, &repository.AddQueryConfigParams{
Queryid: id,
Config: []byte(*entity.Config),
Addedversion: latestVersion,
})
if err != nil {
return err
}
}
activeVersion := entity.ActiveVersion
if activeVersion == nil {
activeVersion = &current.ActiveVersion
}
err := q.UpdateQuery(ctx, &repository.UpdateQueryParams{
Latestversion: latestVersion,
Activeversion: *activeVersion,
ID: id,
})
if err != nil {
return err
}
err = qtx.AddQueryConfig(ctx, &repository.AddQueryConfigParams{
Queryid: id,
Config: []byte(*entity.Config),
Addedversion: latestVersion,
})
if err != nil {
return err
}
}
activeVersion := entity.ActiveVersion
if activeVersion == nil {
activeVersion = &current.ActiveVersion
}
err = qtx.UpdateQuery(ctx, &repository.UpdateQueryParams{
Latestversion: latestVersion,
Activeversion: *activeVersion,
ID: id,
return nil
})
if err != nil {
return err
}
err = tx.Commit(ctx)
if err != nil {
return err
}
return nil
}
@@ -185,9 +177,9 @@ func getSetDifference(setA *[]uuid.UUID, setB *[]uuid.UUID) []uuid.UUID {
func (s *Service) getUpdator(qType resultprocessor.Type) (resultprocessor.Updator, error) {
switch qType {
case resultprocessor.TypeJsonExtractor:
return jsonextractor.NewUpdator(s.db), nil
return jsonextractor.NewUpdator(), nil
case resultprocessor.TypeContextFull:
return contextfull.NewUpdator(s.db), nil
return contextfull.NewUpdator(), nil
default:
return nil, fmt.Errorf("attempting to process invalid query type")
}
+5 -6
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database/repository"
"queryorchestration/internal/query"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -21,12 +22,10 @@ func TestUpdate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := query.New(db, &query.Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := query.New(cfg, &query.Services{})
config := "{\"path\":\"example_path\"}"
existing := query.Query{
+28 -39
View File
@@ -6,6 +6,7 @@ import (
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
resultprocessor "queryorchestration/internal/query/result/processor"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
@@ -19,12 +20,10 @@ func TestGetUpdator(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
queryType := resultprocessor.Type(resultprocessor.TypeContextFull)
updator, err := svc.getUpdator(queryType)
@@ -48,12 +47,10 @@ func TestSubmitUpdate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
config := "{\"path\":\"example_path\"}"
q := Query{
@@ -99,12 +96,10 @@ func TestSubmitUpdateRollback(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
q := Query{
ID: uuid.New(),
@@ -134,12 +129,10 @@ func TestSubmitUpdateRequiredQueries(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
config := "{\"path\":\"example_path\"}"
q := Query{
@@ -186,12 +179,10 @@ func TestSubmitUpdateActiveVersion(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
q := Query{
ID: uuid.New(),
@@ -233,12 +224,10 @@ func TestNormalizeUpdate(t *testing.T) {
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := New(db, &Services{})
cfg := &serviceconfig.BaseConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{})
current := &Query{
ID: uuid.New(),
@@ -246,11 +235,11 @@ func TestNormalizeUpdate(t *testing.T) {
LatestVersion: int32(2),
Type: resultprocessor.TypeJsonExtractor,
}
cfg := "{}"
qcfg := "{}"
aV := int32(2)
update := &resultprocessor.Update{
ID: current.ID,
Config: &cfg,
Config: &qcfg,
ActiveVersion: &aV,
RequiredQueryIDs: &[]uuid.UUID{},
}
@@ -267,7 +256,7 @@ func TestNormalizeUpdate(t *testing.T) {
assert.EqualExportedValues(t, resultprocessor.Update{
ID: current.ID,
ActiveVersion: &aV,
Config: &cfg,
Config: &qcfg,
RequiredQueryIDs: nil,
}, *update)