jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobcollector (pull request #30)

Browse Source 
Initial Job Collector (changes pending)

* movearroundtocleancollector

* internalgetfunctions

* completecollectorquery

* simplify

* fixtests

* addvendor

* noplaceholder
This commit is contained in:
Michael McGuinness
2025-01-21 12:28:46 +00:00
parent b888e3450f
commit 4ccb980593
46 changed files with 951 additions and 655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/collector.sql.go
+77 -29
View File
@@ -11,62 +11,110 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getCollectorFromJobID = `-- name: GetCollectorFromJobID :one
SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
type GetCollectorFromJobIDRow struct {
ID pgtype.UUID `db:"id"`
type AddCollectorQueryParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
// AddCollectorQuery
//
// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
_, err := q.db.Exec(ctx, addCollectorQuery,
arg.Collectorid,
arg.Name,
arg.Queryid,
arg.Addedversion,
)
return err
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
`
type CreateCollectorParams struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
// GetCollectorFromJobID
// CreateCollector
//
// SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID) (*GetCollectorFromJobIDRow, error) {
row := q.db.QueryRow(ctx, getCollectorFromJobID, jobid)
var i GetCollectorFromJobIDRow
// INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, arg *CreateCollectorParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, arg.Jobid, arg.Mincleanversion, arg.Mintextversion)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getCollector = `-- name: GetCollector :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
`
// GetCollector
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, id)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
const getCollectorQueries = `-- name: GetCollectorQueries :many
SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
FROM collectorQueryDependencyTree
WHERE collectorId = $1
GROUP BY queryId, collectorId, type, queryVersion
const getCollectorByJobID = `-- name: GetCollectorByJobID :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
type GetCollectorQueriesRow struct {
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type NullQuerytype `db:"type"`
Queryversion *int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
// GetCollectorByJobID
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByJobID, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
// GetCollectorQueries
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
`
// ListCollectorQueries
//
// SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
// FROM collectorQueryDependencyTree
// WHERE collectorId = $1
// GROUP BY queryId, collectorId, type, queryVersion
func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*GetCollectorQueriesRow, error) {
rows, err := q.db.Query(ctx, getCollectorQueries, collectorid)
// SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*GetCollectorQueriesRow{}
items := []*Collectorquerydependencytree{}
for rows.Next() {
var i GetCollectorQueriesRow
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Queryid,
internal/database/repository/collector_test.go
+82 -8
View File
@@ -2,6 +2,7 @@ package repository_test
import (
"context"
"fmt"
"os"
"path"
"queryorchestration/internal/database"
@@ -10,6 +11,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -23,15 +25,87 @@ func TestCollector(t *testing.T) {
queries := repository.New(db.Pool)
collectorID := database.MustToDBUUID(uuid.New())
collectorQueries, err := queries.GetCollectorQueries(ctx, collectorID)
contextId, err := queries.CreateQuery(ctx, repository.QuerytypeContextFull)
assert.Nil(t, err)
jsonId, err := queries.CreateQuery(ctx, repository.QuerytypeJsonExtractor)
assert.Nil(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonId,
Requiredqueryid: contextId,
Addedversion: 1,
})
assert.Nil(t, err)
assert.Len(t, collectorQueries, 0)
assert.ElementsMatch(t, []repository.GetCollectorQueriesRow{}, collectorQueries)
jobID := database.MustToDBUUID(uuid.New())
jobId := database.MustToDBUUID(uuid.New())
minCleanVersion := int32(2)
minTextVersion := int32(4)
_, err = queries.GetCollectorFromJobID(ctx, jobID)
assert.NotNil(t, err)
collId, err := queries.CreateCollector(ctx, &repository.CreateCollectorParams{
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
})
assert.Nil(t, err)
coll, err := queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
coll, err = queries.GetCollectorByJobID(ctx, jobId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collId,
Queryid: jsonId,
Addedversion: 1,
Name: "example_key",
})
assert.Nil(t, err)
coll, err = queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
}, *coll)
qs, err := queries.ListCollectorQueries(ctx, collId)
assert.Nil(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Queryid: jsonId,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
Requiredids: []pgtype.UUID{contextId},
},
{
Collectorid: collId,
Queryid: contextId,
Queryversion: 1,
Type: repository.QuerytypeContextFull,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
},
}, qs)
}
internal/database/repository/models.go
+17 -14
View File
@@ -62,17 +62,10 @@ func (e Querytype) Valid() bool {
return false
}
type Activecollectorquery struct {
Collectorid pgtype.UUID `db:"collectorid"`
type Activecollectorswithrequiredid struct {
ID pgtype.UUID `db:"id"`
Activeversion int32 `db:"activeversion"`
Queryid pgtype.UUID `db:"queryid"`
}
type Activequeryrequirement struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Activeversion int32 `db:"activeversion"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryids interface{} `db:"queryids"`
}
type Collector struct {
@@ -94,11 +87,21 @@ type Collectorquery struct {
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Queryversion int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryversion int32 `db:"queryversion"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Fields []byte `db:"fields"`
}
type Fullactivequery struct {
internal/database/repository/query.sql.go
+4 -4
View File
@@ -92,12 +92,12 @@ func (q *Queries) DeprecateQuery(ctx context.Context, queryid pgtype.UUID) error
}
const getQuery = `-- name: GetQuery :one
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
`
// GetQuery
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequery, error) {
row := q.db.QueryRow(ctx, getQuery, id)
var i Fullactivequery
@@ -155,12 +155,12 @@ func (q *Queries) IsQueryDeprecated(ctx context.Context, queryid pgtype.UUID) (b
}
const listQueries = `-- name: ListQueries :many
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
`
// ListQueries
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
rows, err := q.db.Query(ctx, listQueries)
if err != nil {
internal/database/repository/query_test.go
+18
View File
@@ -102,6 +102,24 @@ func TestQueries(t *testing.T) {
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
}, *jsonQuery)
err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
Activeversion: 2,
Latestversion: 2,
ID: jsonQueryID,
})
assert.Nil(t, err)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
}, *jsonQuery)
internal/job/collector/collector.go
-43
View File
@@ -1,43 +0,0 @@
package collector
import (
"context"
"queryorchestration/internal/database"
"github.com/google/uuid"
)
type Collector struct {
ID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
db *database.Connection
}
func NewByJobId(ctx context.Context, db *database.Connection, jobID uuid.UUID) (*Collector, error) {
collector := Collector{
db: db,
}
err := collector.getByJobID(ctx, jobID)
if err != nil {
return nil, err
}
return &collector, nil
}
func (c *Collector) getByJobID(ctx context.Context, jobID uuid.UUID) error {
dbJobID := database.MustToDBUUID(jobID)
dbCollector, err := c.db.Queries.GetCollectorFromJobID(ctx, dbJobID)
if err != nil {
return err
}
c.ID = database.MustToUUID(dbCollector.ID)
c.MinCleanVersion = dbCollector.Mincleanversion
c.MinTextVersion = dbCollector.Mintextversion
return nil
}
internal/job/collector/get.go
+26
View File
@@ -0,0 +1,26 @@
package collector
import (
"context"
"queryorchestration/internal/database"
"github.com/google/uuid"
)
func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Collector, error) {
dbColl, err := s.db.Queries.GetCollector(ctx, database.MustToDBUUID(id))
if err != nil {
return nil, err
}
return parseDBCollector(dbColl)
}
func (s *Service) GetByJobID(ctx context.Context, jobID uuid.UUID) (*Collector, error) {
dbColl, err := s.db.Queries.GetCollectorByJobID(ctx, database.MustToDBUUID(jobID))
if err != nil {
return nil, err
}
return parseDBCollector(dbColl)
}
internal/job/collector/get_test.go
+86
View File
@@ -0,0 +1,86 @@
package collector_test
import (
"context"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
func TestGet(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := collector.New(db)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 2,
MinTextVersion: 4,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
pool.ExpectQuery("name: GetCollector :one").WithArgs(database.MustToDBUUID(ogc.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.Get(ctx, ogc.ID)
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
func TestGetByJobID(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
svc := collector.New(db)
ogc := collector.Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 2,
MinTextVersion: 4,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(ogc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(ogc.ID), database.MustToDBUUID(ogc.JobID), ogc.MinCleanVersion, ogc.MinTextVersion, ogc.ActiveVersion, ogc.LatestVersion, []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"].String()))),
)
coll, err := svc.GetByJobID(ctx, ogc.JobID)
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *coll)
}
internal/job/collector/parse.go
+37
View File
@@ -0,0 +1,37 @@
package collector
import (
"encoding/json"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
)
func parseDBCollector(c *repository.Fullactivecollector) (*Collector, error) {
if c == nil {
return nil, nil
}
var fields map[string]uuid.UUID
if len(c.Fields) > 0 {
err := json.Unmarshal(c.Fields, &fields)
if err != nil {
return nil, fmt.Errorf("error unmarshalling: %s", err)
}
} else {
fields = map[string]uuid.UUID{}
}
return &Collector{
ID: database.MustToUUID(c.ID),
JobID: database.MustToUUID(c.Jobid),
MinCleanVersion: c.Mincleanversion,
MinTextVersion: c.Mintextversion,
ActiveVersion: c.Activeversion,
LatestVersion: c.Latestversion,
Fields: fields,
}, nil
}
internal/job/collector/parse_test.go
+36
View File
@@ -0,0 +1,36 @@
package collector
import (
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func TestParseDBCollector(t *testing.T) {
c, err := parseDBCollector(nil)
assert.Nil(t, err)
assert.Nil(t, c)
ogc := Collector{
ID: uuid.New(),
JobID: uuid.New(),
MinCleanVersion: 1,
MinTextVersion: 2,
Fields: map[string]uuid.UUID{
"example_key": uuid.New(),
},
}
c, err = parseDBCollector(&repository.Fullactivecollector{
ID: database.MustToDBUUID(ogc.ID),
Jobid: database.MustToDBUUID(ogc.JobID),
Mincleanversion: ogc.MinCleanVersion,
Mintextversion: ogc.MinTextVersion,
Fields: []byte(fmt.Sprintf("{\"example_key\":\"%s\"}", ogc.Fields["example_key"])),
})
assert.Nil(t, err)
assert.EqualExportedValues(t, ogc, *c)
}
internal/job/collector/service.go
+12
View File
@@ -2,8 +2,20 @@ package collector
import (
"queryorchestration/internal/database"
"github.com/google/uuid"
)
type Collector struct {
ID uuid.UUID
JobID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
ActiveVersion int32
LatestVersion int32
Fields map[string]uuid.UUID
}
type Service struct {
db *database.Connection
}
internal/job/collector/service_test.go
-41
View File
@@ -1,13 +1,11 @@
package collector_test
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -26,42 +24,3 @@ func TestService(t *testing.T) {
svc := collector.New(db)
assert.NotNil(t, svc)
}
func TestByJobId(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
fullCollector := collector.Collector{
ID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(1),
}
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(jobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(database.MustToDBUUID(fullCollector.ID), database.MustToDBUUID(jobID), fullCollector.MinCleanVersion, fullCollector.MinTextVersion),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
assert.EqualExportedValues(t, fullCollector, *coll)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(database.MustToDBUUID(jobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}),
)
_, err = collector.NewByJobId(ctx, db, jobID)
assert.EqualError(t, err, "no rows in result set")
}
internal/query/document/service.go
+19 -6
View File
@@ -19,28 +19,41 @@ type Document struct {
TextVersion int32 `json:"textVersion" validate:"required,gt=0"`
}
type Service struct {
db *database.Connection
type Services struct {
Collector *collector.Service
}
func New(db *database.Connection) *Service {
type Service struct {
db *database.Connection
svc *Services
}
func New(db *database.Connection, svc *Services) *Service {
return &Service{
db,
svc,
}
}
func (s *Service) Sync(ctx context.Context, doc *Document) error {
collector, err := collector.NewByJobId(ctx, s.db, doc.JobID)
coll, err := s.svc.Collector.GetByJobID(ctx, doc.JobID)
if err != nil {
return err
}
results, err := s.getResults(ctx, doc.ID, collector)
results, err := s.getResults(ctx, doc.ID, coll)
if err != nil {
return err
}
queue, err := queryqueue.New(ctx, s.db, collector, results, doc.ID, doc.CleanVersion, doc.TextVersion)
queue, err := queryqueue.New(ctx, &queryqueue.NewConfig{
DB: s.db,
Collector: coll,
Results: results,
DocumentID: doc.ID,
CleanVersion: doc.CleanVersion,
TextVersion: doc.TextVersion,
})
if err != nil {
return err
}
internal/query/document/sync_test.go
+53 -38
View File
@@ -5,6 +5,7 @@ import (
"errors"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/job/collector"
"queryorchestration/internal/query/document"
"testing"
@@ -33,29 +34,33 @@ func TestSyncIsSynced(t *testing.T) {
Name: "document_name",
}
dbCollectorId := database.MustToDBUUID(uuid.New())
dbJobID := database.MustToDBUUID(doc.JobID)
minCleanVersion := int32(1)
minTextVersion := int32(1)
qV := int32(1)
queryVersion := int32(1)
coll := collector.Collector{
ID: uuid.New(),
JobID: doc.JobID,
MinCleanVersion: int32(1),
MinTextVersion: int32(2),
}
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(database.MustToDBUUID(doc.JobID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(database.MustToDBUUID(coll.ID), database.MustToDBUUID(coll.JobID), coll.MinCleanVersion, coll.MinTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), coll.MinCleanVersion, coll.MinTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}).
AddRow(pgtype.UUID{}, pgtype.UUID{}, qV),
AddRow(pgtype.UUID{}, pgtype.UUID{}, queryVersion),
)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(database.MustToDBUUID(coll.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, pgtype.UUID{}, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(database.MustToDBUUID(coll.ID), pgtype.UUID{}, repository.QuerytypeJsonExtractor, queryVersion, []pgtype.UUID{}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.Nil(t, err)
}
@@ -85,49 +90,55 @@ func TestSyncDBFail(t *testing.T) {
minCleanVersion := int32(1)
minTextVersion := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, "no rows in result set")
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
dbErr := "database failure"
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
dbErr = "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
qV := int32(1)
reqID := database.MustToDBUUID(uuid.New())
@@ -138,16 +149,18 @@ func TestSyncDBFail(t *testing.T) {
AddRow(resID, reqID, qV),
)
dbErr = "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{reqID}).
AddRow(dbCollectorId, reqID, repository.NullQuerytype{Querytype: repository.QuerytypeContextFull, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{reqID}).
AddRow(dbCollectorId, reqID, repository.QuerytypeContextFull, qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{resID}).
WillReturnError(errors.New(dbErr))
docSvc = document.New(db)
docSvc = document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, dbErr)
}
@@ -177,27 +190,29 @@ func TestSync(t *testing.T) {
minCleanVersion := int32(1)
minTextVersion := int32(1)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
pool.ExpectQuery("name: GetCollectorByJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion),
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion", "activeVersion", "latestVersion", "fields"}).
AddRow(dbCollectorId, dbJobID, minCleanVersion, minTextVersion, int32(1), int32(2), []byte("")),
)
pool.ExpectQuery("name: ListResultsByDocumentID :many").WithArgs(database.MustToDBUUID(doc.ID), minCleanVersion, minTextVersion).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "queryVersion"}),
)
qV := int32(1)
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorId).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorId).
WillReturnRows(
pgxmock.NewRows([]string{"collectorId", "queryId", "type", "queryVersion", "requiredIds"}).
AddRow(dbCollectorId, dbQueryID, repository.NullQuerytype{Querytype: repository.QuerytypeJsonExtractor, Valid: true}, &qV, []pgtype.UUID{}),
AddRow(dbCollectorId, dbQueryID, repository.QuerytypeJsonExtractor, qV, []pgtype.UUID{}),
)
pool.ExpectQuery("name: ListResultValuesByID :many").WithArgs([]pgtype.UUID{}).
WillReturnRows(
pgxmock.NewRows([]string{"id", "queryId", "value"}),
)
docSvc := document.New(db)
docSvc := document.New(db, &document.Services{
Collector: collector.New(db),
})
err = docSvc.Sync(ctx, &doc)
assert.EqualError(t, err, "JSON Extraction requires 1 result")
}
internal/query/processor/parse.go
+3 -3
View File
@@ -65,21 +65,21 @@ func ToDBNullQueryType(t Type) (repository.NullQuerytype, error) {
return repository.NullQuerytype{Querytype: dbType, Valid: true}, nil
}
func ParseDBCollectorQuery(q *repository.GetCollectorQueriesRow) (*Query, error) {
func ParseDBCollectorQuery(q *repository.Collectorquerydependencytree) (*Query, error) {
var reqQueryIDs *[]uuid.UUID
if len(q.Requiredids) > 0 {
ids := database.MustToUUIDArray(q.Requiredids)
reqQueryIDs = &ids
}
qType, err := ParseDBNullType(q.Type)
qType, err := ParseDBType(q.Type)
if err != nil {
return nil, err
}
return &Query{
ID: database.MustToUUID(q.Queryid),
Version: *q.Queryversion,
Version: q.Queryversion,
Type: qType,
RequiredQueryIDs: reqQueryIDs,
}, nil
internal/query/processor/parse_test.go
+5 -6
View File
@@ -11,22 +11,21 @@ import (
)
func TestParseDBCollectorQuery(t *testing.T) {
qV := int32(0)
dbResult := repository.GetCollectorQueriesRow{
dbResult := repository.Collectorquerydependencytree{
Collectorid: pgtype.UUID{},
Queryid: pgtype.UUID{},
Requiredids: []pgtype.UUID{},
Type: repository.NullQuerytype{Valid: true, Querytype: repository.QuerytypeJsonExtractor},
Queryversion: &qV,
Type: repository.QuerytypeJsonExtractor,
Queryversion: 1,
}
value, err := queryprocessor.ParseDBCollectorQuery(&dbResult)
assert.Nil(t, err)
assert.Equal(t, uuid.Nil, value.ID)
assert.Nil(t, value.RequiredQueryIDs)
assert.Equal(t, int32(0), value.Version)
assert.Equal(t, int32(1), value.Version)
assert.Equal(t, queryprocessor.Type(queryprocessor.TypeJsonExtractor), value.Type)
dbResult.Type = repository.NullQuerytype{}
dbResult.Type = repository.Querytype("")
_, err = queryprocessor.ParseDBCollectorQuery(&dbResult)
assert.EqualError(t, err, "invalid database query type")
}
internal/query/queue/create.go
+1 -1
View File
@@ -33,7 +33,7 @@ func (c *Queue) getCollectorQueries(ctx context.Context) error {
id := database.MustToDBUUID(c.collector.ID)
queries, err := c.db.Queries.GetCollectorQueries(ctx, id)
queries, err := c.db.Queries.ListCollectorQueries(ctx, id)
if err != nil {
return err
}
internal/query/queue/create_test.go
+3 -3
View File
@@ -71,13 +71,13 @@ func TestGetCollectorQueries(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
err = svc.getCollectorQueries(ctx)
assert.Nil(t, err)
internal/query/queue/execute_test.go
+10 -17
View File
@@ -29,19 +29,10 @@ func TestExecute(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
queryOneID := uuid.New()
queryOneVersion := int32(1)
@@ -74,13 +65,13 @@ func TestExecute(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
contextResultID := uuid.New()
results := []*result.Result{
@@ -102,7 +93,9 @@ func TestExecute(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
q, err := queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
q, err := queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.Nil(t, err)
assert.Equal(t, expectedQueries, q.GetQueue())
internal/query/queue/service.go
+17 -8
View File
@@ -16,19 +16,28 @@ type Queue struct {
results []*result.Result
collector *collector.Collector
db *database.Connection
documentId uuid.UUID
cleanVersion int32
textVersion int32
documentId uuid.UUID
}
func New(ctx context.Context, db *database.Connection, coll *collector.Collector, results []*result.Result, docId uuid.UUID, cleanVersion int32, textVersion int32) (*Queue, error) {
type NewConfig struct {
DB *database.Connection
Collector *collector.Collector
Results []*result.Result
DocumentID uuid.UUID
CleanVersion int32
TextVersion int32
}
func New(ctx context.Context, cfg *NewConfig) (*Queue, error) {
queue := Queue{
db: db,
results: results,
collector: coll,
documentId: docId,
cleanVersion: cleanVersion,
textVersion: textVersion,
db: cfg.DB,
results: cfg.Results,
collector: cfg.Collector,
documentId: cfg.DocumentID,
cleanVersion: cfg.CleanVersion,
textVersion: cfg.TextVersion,
}
err := queue.getCollectorQueries(ctx)
internal/query/queue/service_test.go
+18 -31
View File
@@ -29,19 +29,11 @@ func TestService(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
queryOneID := uuid.New()
queryOneVersion := int32(1)
@@ -74,13 +66,13 @@ func TestService(t *testing.T) {
if q.RequiredQueryIDs != nil {
dbReqIDs = database.MustToDBUUIDArray(*q.RequiredQueryIDs)
}
ty, err := queryprocessor.ToDBNullQueryType(q.Type)
ty, err := queryprocessor.ToDBQueryType(q.Type)
assert.Nil(t, err)
rows = rows.
AddRow(dbCollectorID, dbID, ty, &q.Version, dbReqIDs)
AddRow(dbCollectorID, dbID, ty, q.Version, dbReqIDs)
}
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).WillReturnRows(rows)
contextResultID := uuid.New()
results := []*result.Result{
@@ -102,7 +94,9 @@ func TestService(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
q, err := queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
q, err := queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.Nil(t, err)
assert.Equal(t, expectedQueries, q.GetQueue())
}
@@ -119,22 +113,13 @@ func TestQueueFail(t *testing.T) {
Queries: queries,
Pool: pool,
}
jobID := uuid.New()
dbJobID := database.MustToDBUUID(jobID)
collectorID := uuid.New()
dbCollectorID := database.MustToDBUUID(collectorID)
pool.ExpectQuery("name: GetCollectorFromJobID :one").WithArgs(dbJobID).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "minCleanVersion", "minTextVersion"}).
AddRow(dbCollectorID, dbJobID, int32(1), int32(1)),
)
coll, err := collector.NewByJobId(ctx, db, jobID)
assert.Nil(t, err)
coll := collector.Collector{
ID: uuid.New(),
}
dbCollectorID := database.MustToDBUUID(coll.ID)
dbErr := "database failure"
pool.ExpectQuery("name: GetCollectorQueries :many").WithArgs(dbCollectorID).
pool.ExpectQuery("name: ListCollectorQueries :many").WithArgs(dbCollectorID).
WillReturnError(errors.New(dbErr))
results := []*result.Result{}
@@ -143,6 +128,8 @@ func TestQueueFail(t *testing.T) {
cleanVersion := int32(1)
textVersion := int32(1)
_, err = queryqueue.New(ctx, db, coll, results, docID, cleanVersion, textVersion)
_, err = queryqueue.New(ctx, &queryqueue.NewConfig{
db, &coll, results, docID, cleanVersion, textVersion,
})
assert.EqualError(t, err, dbErr)
}