jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobcollector (pull request #30)

Browse Source 
Initial Job Collector (changes pending)

* movearroundtocleancollector

* internalgetfunctions

* completecollectorquery

* simplify

* fixtests

* addvendor

* noplaceholder
This commit is contained in:
Michael McGuinness
2025-01-21 12:28:46 +00:00
parent b888e3450f
commit 4ccb980593
46 changed files with 951 additions and 655 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/collector.sql.go
+77 -29
View File
@@ -11,62 +11,110 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const getCollectorFromJobID = `-- name: GetCollectorFromJobID :one
SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
type GetCollectorFromJobIDRow struct {
ID pgtype.UUID `db:"id"`
type AddCollectorQueryParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
// AddCollectorQuery
//
// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
_, err := q.db.Exec(ctx, addCollectorQuery,
arg.Collectorid,
arg.Name,
arg.Queryid,
arg.Addedversion,
)
return err
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
`
type CreateCollectorParams struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
// GetCollectorFromJobID
// CreateCollector
//
// SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorFromJobID(ctx context.Context, jobid pgtype.UUID) (*GetCollectorFromJobIDRow, error) {
row := q.db.QueryRow(ctx, getCollectorFromJobID, jobid)
var i GetCollectorFromJobIDRow
// INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, arg *CreateCollectorParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, arg.Jobid, arg.Mincleanversion, arg.Mintextversion)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getCollector = `-- name: GetCollector :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
`
// GetCollector
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, id)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
const getCollectorQueries = `-- name: GetCollectorQueries :many
SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
FROM collectorQueryDependencyTree
WHERE collectorId = $1
GROUP BY queryId, collectorId, type, queryVersion
const getCollectorByJobID = `-- name: GetCollectorByJobID :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
type GetCollectorQueriesRow struct {
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type NullQuerytype `db:"type"`
Queryversion *int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
// GetCollectorByJobID
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByJobID, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
&i.Activeversion,
&i.Latestversion,
&i.Fields,
)
return &i, err
}
// GetCollectorQueries
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
`
// ListCollectorQueries
//
// SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
// FROM collectorQueryDependencyTree
// WHERE collectorId = $1
// GROUP BY queryId, collectorId, type, queryVersion
func (q *Queries) GetCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*GetCollectorQueriesRow, error) {
rows, err := q.db.Query(ctx, getCollectorQueries, collectorid)
// SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*GetCollectorQueriesRow{}
items := []*Collectorquerydependencytree{}
for rows.Next() {
var i GetCollectorQueriesRow
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Queryid,
internal/database/repository/collector_test.go
+82 -8
View File
@@ -2,6 +2,7 @@ package repository_test
import (
"context"
"fmt"
"os"
"path"
"queryorchestration/internal/database"
@@ -10,6 +11,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -23,15 +25,87 @@ func TestCollector(t *testing.T) {
queries := repository.New(db.Pool)
collectorID := database.MustToDBUUID(uuid.New())
collectorQueries, err := queries.GetCollectorQueries(ctx, collectorID)
contextId, err := queries.CreateQuery(ctx, repository.QuerytypeContextFull)
assert.Nil(t, err)
jsonId, err := queries.CreateQuery(ctx, repository.QuerytypeJsonExtractor)
assert.Nil(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonId,
Requiredqueryid: contextId,
Addedversion: 1,
})
assert.Nil(t, err)
assert.Len(t, collectorQueries, 0)
assert.ElementsMatch(t, []repository.GetCollectorQueriesRow{}, collectorQueries)
jobID := database.MustToDBUUID(uuid.New())
jobId := database.MustToDBUUID(uuid.New())
minCleanVersion := int32(2)
minTextVersion := int32(4)
_, err = queries.GetCollectorFromJobID(ctx, jobID)
assert.NotNil(t, err)
collId, err := queries.CreateCollector(ctx, &repository.CreateCollectorParams{
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
})
assert.Nil(t, err)
coll, err := queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
coll, err = queries.GetCollectorByJobID(ctx, jobId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
}, *coll)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collId,
Queryid: jsonId,
Addedversion: 1,
Name: "example_key",
})
assert.Nil(t, err)
coll, err = queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Activeversion: 1,
Latestversion: 1,
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
}, *coll)
qs, err := queries.ListCollectorQueries(ctx, collId)
assert.Nil(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Queryid: jsonId,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
Requiredids: []pgtype.UUID{contextId},
},
{
Collectorid: collId,
Queryid: contextId,
Queryversion: 1,
Type: repository.QuerytypeContextFull,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
},
}, qs)
}
internal/database/repository/models.go
+17 -14
View File
@@ -62,17 +62,10 @@ func (e Querytype) Valid() bool {
return false
}
type Activecollectorquery struct {
Collectorid pgtype.UUID `db:"collectorid"`
type Activecollectorswithrequiredid struct {
ID pgtype.UUID `db:"id"`
Activeversion int32 `db:"activeversion"`
Queryid pgtype.UUID `db:"queryid"`
}
type Activequeryrequirement struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Activeversion int32 `db:"activeversion"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryids interface{} `db:"queryids"`
}
type Collector struct {
@@ -94,11 +87,21 @@ type Collectorquery struct {
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Collectorid pgtype.UUID `db:"collectorid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Queryversion int32 `db:"queryversion"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Type Querytype `db:"type"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Queryversion int32 `db:"queryversion"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Fields []byte `db:"fields"`
}
type Fullactivequery struct {
internal/database/repository/query.sql.go
+4 -4
View File
@@ -92,12 +92,12 @@ func (q *Queries) DeprecateQuery(ctx context.Context, queryid pgtype.UUID) error
}
const getQuery = `-- name: GetQuery :one
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
`
// GetQuery
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequery, error) {
row := q.db.QueryRow(ctx, getQuery, id)
var i Fullactivequery
@@ -155,12 +155,12 @@ func (q *Queries) IsQueryDeprecated(ctx context.Context, queryid pgtype.UUID) (b
}
const listQueries = `-- name: ListQueries :many
SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
`
// ListQueries
//
// SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
func (q *Queries) ListQueries(ctx context.Context) ([]*Fullactivequery, error) {
rows, err := q.db.Query(ctx, listQueries)
if err != nil {
internal/database/repository/query_test.go
+18
View File
@@ -102,6 +102,24 @@ func TestQueries(t *testing.T) {
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
}, *jsonQuery)
err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
Activeversion: 2,
Latestversion: 2,
ID: jsonQueryID,
})
assert.Nil(t, err)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.Nil(t, err)
assert.EqualExportedValues(t, repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
}, *jsonQuery)