Merged in fature/jobs (pull request #34)

Job Collector

* createstructure

* mostupdatevalidation

* repocollectorupdate

* updateoutline

* updatevalidation

* scriptupdate

* cleanupdockerignore

* update

* collectorupdateapi
This commit is contained in:
Michael McGuinness
2025-01-23 14:56:20 +00:00
parent 36967fc946
commit 5b7160fe44
88 changed files with 2181 additions and 297 deletions
+25
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"log"
"net/url"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/server/env"
"github.com/docker/go-connections/nat"
@@ -108,3 +109,27 @@ func GetDBPool(ctx context.Context) *pgxpool.Pool {
return pool
}
func ExecuteTransaction(ctx context.Context, db *Connection, executeQueries func(context.Context, *repository.Queries) error) error {
tx, err := db.Pool.Begin(ctx)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback(ctx)
}()
qtx := db.Queries.WithTx(tx)
err = executeQueries(ctx, qtx)
if err != nil {
return err
}
err = tx.Commit(ctx)
if err != nil {
return err
}
return nil
}
+37
View File
@@ -5,9 +5,12 @@ import (
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
@@ -26,3 +29,37 @@ func TestDBConn(t *testing.T) {
pool := database.GetDBPool(ctx)
assert.NotNil(t, pool)
}
func TestExecuteTransation(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
queries := repository.New(pool)
db := &database.Connection{
Queries: queries,
Pool: pool,
}
clientID := database.MustToDBUUID(uuid.New())
clientName := "example_client"
pool.ExpectBegin()
pool.ExpectQuery("name: CreateClient :one").WithArgs(clientName).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(clientID),
)
pool.ExpectCommit()
err = database.ExecuteTransaction(ctx, db, func(ctx context.Context, q *repository.Queries) error {
id, err := q.CreateClient(ctx, "example_client")
assert.Nil(t, err)
assert.Equal(t, clientID, id)
return nil
})
assert.Nil(t, err)
}
+81 -10
View File
@@ -11,6 +11,30 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const addCollectorCodeVersion = `-- name: AddCollectorCodeVersion :exec
INSERT INTO collectorCodeVersions (collectorId, addedVersion, minCleanVersion, minTextVersion) VALUES ($1, $2, $3, $4)
`
type AddCollectorCodeVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Addedversion int32 `db:"addedversion"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
// AddCollectorCodeVersion
//
// INSERT INTO collectorCodeVersions (collectorId, addedVersion, minCleanVersion, minTextVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorCodeVersion(ctx context.Context, arg *AddCollectorCodeVersionParams) error {
_, err := q.db.Exec(ctx, addCollectorCodeVersion,
arg.Collectorid,
arg.Addedversion,
arg.Mincleanversion,
arg.Mintextversion,
)
return err
}
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
@@ -36,20 +60,14 @@ func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryP
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
`
type CreateCollectorParams struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
}
// CreateCollector
//
// INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, arg *CreateCollectorParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, arg.Jobid, arg.Mincleanversion, arg.Mintextversion)
// INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, jobid)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
@@ -131,3 +149,56 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
}
return items, nil
}
const removeCollectorCodeVersion = `-- name: RemoveCollectorCodeVersion :exec
UPDATE collectorCodeVersions SET removedVersion = $1 WHERE collectorId = $2 and removedVersion is null
`
type RemoveCollectorCodeVersionParams struct {
Removedversion *int32 `db:"removedversion"`
Collectorid pgtype.UUID `db:"collectorid"`
}
// RemoveCollectorCodeVersion
//
// UPDATE collectorCodeVersions SET removedVersion = $1 WHERE collectorId = $2 and removedVersion is null
func (q *Queries) RemoveCollectorCodeVersion(ctx context.Context, arg *RemoveCollectorCodeVersionParams) error {
_, err := q.db.Exec(ctx, removeCollectorCodeVersion, arg.Removedversion, arg.Collectorid)
return err
}
const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
`
type RemoveCollectorQueryParams struct {
Removedversion *int32 `db:"removedversion"`
Queryid pgtype.UUID `db:"queryid"`
Collectorid pgtype.UUID `db:"collectorid"`
}
// RemoveCollectorQuery
//
// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error {
_, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Collectorid)
return err
}
const updateCollector = `-- name: UpdateCollector :exec
UPDATE collectors SET latestVersion = $1, activeVersion = $2 WHERE id = $3
`
type UpdateCollectorParams struct {
Latestversion int32 `db:"latestversion"`
Activeversion int32 `db:"activeversion"`
ID pgtype.UUID `db:"id"`
}
// UpdateCollector
//
// UPDATE collectors SET latestVersion = $1, activeVersion = $2 WHERE id = $3
func (q *Queries) UpdateCollector(ctx context.Context, arg *UpdateCollectorParams) error {
_, err := q.db.Exec(ctx, updateCollector, arg.Latestversion, arg.Activeversion, arg.ID)
return err
}
+52 -12
View File
@@ -36,15 +36,14 @@ func TestCollector(t *testing.T) {
})
assert.Nil(t, err)
jobId := database.MustToDBUUID(uuid.New())
clientId, err := queries.CreateClient(ctx, "example_client")
assert.Nil(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.Nil(t, err)
minCleanVersion := int32(2)
minTextVersion := int32(4)
collId, err := queries.CreateCollector(ctx, &repository.CreateCollectorParams{
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
})
collId, err := queries.CreateCollector(ctx, jobId)
assert.Nil(t, err)
coll, err := queries.GetCollector(ctx, collId)
@@ -52,8 +51,8 @@ func TestCollector(t *testing.T) {
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Mincleanversion: nil,
Mintextversion: nil,
Activeversion: 1,
Latestversion: 1,
}, coll)
@@ -63,8 +62,8 @@ func TestCollector(t *testing.T) {
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Mincleanversion: nil,
Mintextversion: nil,
Activeversion: 1,
Latestversion: 1,
}, coll)
@@ -77,13 +76,21 @@ func TestCollector(t *testing.T) {
})
assert.Nil(t, err)
err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{
Collectorid: collId,
Addedversion: 1,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
})
assert.Nil(t, err)
coll, err = queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
Mincleanversion: &minCleanVersion,
Mintextversion: &minTextVersion,
Activeversion: 1,
Latestversion: 1,
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
@@ -108,4 +115,37 @@ func TestCollector(t *testing.T) {
Requiredids: []pgtype.UUID{database.MustToDBUUID(uuid.Nil)},
},
}, qs)
removeV := int32(2)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Collectorid: collId,
Queryid: jsonId,
Removedversion: &removeV,
})
assert.Nil(t, err)
err = queries.RemoveCollectorCodeVersion(ctx, &repository.RemoveCollectorCodeVersionParams{
Collectorid: collId,
Removedversion: &removeV,
})
assert.Nil(t, err)
err = queries.UpdateCollector(ctx, &repository.UpdateCollectorParams{
ID: collId,
Latestversion: 2,
Activeversion: 2,
})
assert.Nil(t, err)
coll, err = queries.GetCollector(ctx, collId)
assert.Nil(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: nil,
Mintextversion: nil,
Activeversion: 2,
Latestversion: 2,
Fields: []byte(nil),
}, coll)
}
@@ -0,0 +1,40 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.27.0
// source: document.sql
package repository
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createDocument = `-- name: CreateDocument :one
INSERT INTO documents (jobId) VALUES ($1) RETURNING id
`
// CreateDocument
//
// INSERT INTO documents (jobId) VALUES ($1) RETURNING id
func (q *Queries) CreateDocument(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createDocument, jobid)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getDocument = `-- name: GetDocument :one
SELECT id, jobId FROM documents WHERE id = $1 LIMIT 1
`
// GetDocument
//
// SELECT id, jobId FROM documents WHERE id = $1 LIMIT 1
func (q *Queries) GetDocument(ctx context.Context, id pgtype.UUID) (*Document, error) {
row := q.db.QueryRow(ctx, getDocument, id)
var i Document
err := row.Scan(&i.ID, &i.Jobid)
return &i, err
}
@@ -0,0 +1,40 @@
package repository_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
"testing"
"github.com/stretchr/testify/assert"
)
func TestDocument(t *testing.T) {
ctx := context.Background()
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
queries := repository.New(db.Pool)
clientId, err := queries.CreateClient(ctx, "example_client")
assert.Nil(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.Nil(t, err)
id, err := queries.CreateDocument(ctx, jobId)
assert.Nil(t, err)
assert.NotEmpty(t, id)
doc, err := queries.GetDocument(ctx, id)
assert.Nil(t, err)
assert.EqualExportedValues(t, &repository.Document{
ID: id,
Jobid: jobId,
}, doc)
}
+40
View File
@@ -0,0 +1,40 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.27.0
// source: job.sql
package repository
import (
"context"
"github.com/jackc/pgx/v5/pgtype"
)
const createJob = `-- name: CreateJob :one
INSERT INTO jobs (clientId) VALUES ($1) RETURNING id
`
// CreateJob
//
// INSERT INTO jobs (clientId) VALUES ($1) RETURNING id
func (q *Queries) CreateJob(ctx context.Context, clientid pgtype.UUID) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createJob, clientid)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getJob = `-- name: GetJob :one
SELECT id, clientId, canSync FROM jobs WHERE id = $1 LIMIT 1
`
// GetJob
//
// SELECT id, clientId, canSync FROM jobs WHERE id = $1 LIMIT 1
func (q *Queries) GetJob(ctx context.Context, id pgtype.UUID) (*Job, error) {
row := q.db.QueryRow(ctx, getJob, id)
var i Job
err := row.Scan(&i.ID, &i.Clientid, &i.Cansync)
return &i, err
}
+39
View File
@@ -0,0 +1,39 @@
package repository_test
import (
"context"
"os"
"path"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/test"
"testing"
"github.com/stretchr/testify/assert"
)
func TestJob(t *testing.T) {
ctx := context.Background()
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
queries := repository.New(db.Pool)
clientId, err := queries.CreateClient(ctx, "example_client")
assert.Nil(t, err)
id, err := queries.CreateJob(ctx, clientId)
assert.Nil(t, err)
assert.NotEmpty(t, id)
job, err := queries.GetJob(ctx, id)
assert.Nil(t, err)
assert.EqualExportedValues(t, &repository.Job{
ID: id,
Clientid: clientId,
Cansync: false,
}, job)
}
+28 -11
View File
@@ -62,12 +62,6 @@ func (e Querytype) Valid() bool {
return false
}
type Activecollectorswithrequiredid struct {
ID pgtype.UUID `db:"id"`
Activeversion int32 `db:"activeversion"`
Queryids interface{} `db:"queryids"`
}
type Client struct {
ID pgtype.UUID `db:"id"`
Name string `db:"name"`
@@ -75,12 +69,19 @@ type Client struct {
}
type Collector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Latestversion int32 `db:"latestversion"`
Activeversion int32 `db:"activeversion"`
}
type Collectorcodeversion struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Collectorid pgtype.UUID `db:"collectorid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Latestversion int32 `db:"latestversion"`
Activeversion int32 `db:"activeversion"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
}
type Collectorquery struct {
@@ -100,11 +101,16 @@ type Collectorquerydependencytree struct {
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Document struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
Mincleanversion *int32 `db:"mincleanversion"`
Mintextversion *int32 `db:"mintextversion"`
Activeversion int32 `db:"activeversion"`
Latestversion int32 `db:"latestversion"`
Fields []byte `db:"fields"`
@@ -119,6 +125,12 @@ type Fullactivequery struct {
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Job struct {
ID pgtype.UUID `db:"id"`
Clientid pgtype.UUID `db:"clientid"`
Cansync bool `db:"cansync"`
}
type Query struct {
ID pgtype.UUID `db:"id"`
Latestversion int32 `db:"latestversion"`
@@ -126,6 +138,11 @@ type Query struct {
Type Querytype `db:"type"`
}
type Queryactivedependency struct {
ID pgtype.UUID `db:"id"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
}
type Queryconfig struct {
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
+23
View File
@@ -124,6 +124,29 @@ func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams)
return &i, err
}
const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one
SELECT EXISTS (
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
)
`
type IsQueryInDependencyTreeParams struct {
ID []pgtype.UUID `db:"id"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
}
// IsQueryInDependencyTree
//
// SELECT EXISTS (
// SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
// )
func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDependencyTreeParams) (bool, error) {
row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.ID, arg.Requiredqueryid)
var exists bool
err := row.Scan(&exists)
return exists, err
}
const listQueries = `-- name: ListQueries :many
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
`
@@ -145,6 +145,74 @@ func TestQueries(t *testing.T) {
assert.False(t, all_exist)
}
func TestQueryDependencyTree(t *testing.T) {
ctx := context.Background()
db, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Migrations: &database.MigrationConfig{
BasePath: path.Join(os.Getenv("PWD"), "../../.."),
}})
defer cleanup()
queries := repository.New(db.Pool)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.Nil(t, err)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.Nil(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.Nil(t, err)
secondJsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.Nil(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: secondJsonQueryID,
Requiredqueryid: jsonQueryID,
Addedversion: 1,
})
assert.Nil(t, err)
isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{contextQueryID},
})
assert.Nil(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{secondJsonQueryID},
})
assert.Nil(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{jsonQueryID},
})
assert.Nil(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: secondJsonQueryID,
ID: []pgtype.UUID{jsonQueryID, contextQueryID},
})
assert.Nil(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: contextQueryID,
ID: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
})
assert.Nil(t, err)
assert.True(t, isdependent)
}
func TestQueriesList(t *testing.T) {
ctx := context.Background()
+4 -2
View File
@@ -45,7 +45,8 @@ func (q *Queries) ListResultValuesByID(ctx context.Context, id []pgtype.UUID) ([
}
const listResultsByDocumentID = `-- name: ListResultsByDocumentID :many
SELECT id, queryId, queryVersion FROM results where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
SELECT id, queryId, queryVersion FROM results
where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
`
type ListResultsByDocumentIDParams struct {
@@ -62,7 +63,8 @@ type ListResultsByDocumentIDRow struct {
// ListResultsByDocumentID
//
// SELECT id, queryId, queryVersion FROM results where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
// SELECT id, queryId, queryVersion FROM results
// where documentId = $1 and cleanVersion >= $2 and textVersion >= $3
func (q *Queries) ListResultsByDocumentID(ctx context.Context, arg *ListResultsByDocumentIDParams) ([]*ListResultsByDocumentIDRow, error) {
rows, err := q.db.Query(ctx, listResultsByDocumentID, arg.Documentid, arg.Cleanversion, arg.Textversion)
if err != nil {
+6 -2
View File
@@ -9,7 +9,6 @@ import (
"queryorchestration/internal/test"
"testing"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -28,7 +27,12 @@ func TestResults(t *testing.T) {
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.Nil(t, err)
documentID := database.MustToDBUUID(uuid.New())
clientId, err := queries.CreateClient(ctx, "example_client")
assert.Nil(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.Nil(t, err)
documentID, err := queries.CreateDocument(ctx, jobId)
assert.Nil(t, err)
jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)
assert.Nil(t, err)