jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobsynced (pull request #72)

Browse Source 
Job Status Get and DB tidy up

* initalquery

* tests

* shorttests

* testing queries

* job

* solvedthequery

* updatingdb

* fixingtests

* repotests

* shorttests

* docker

* testspassed
This commit is contained in:
Michael McGuinness
2025-02-20 19:02:44 +00:00
parent 0ea544926b
commit 3d434eedb8
74 changed files with 2395 additions and 1335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/clean.sql.go
+11 -43
View File
@@ -36,39 +36,21 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle
}
const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1
SELECT id, documentId, bucket, key, version
FROM currentCleanEntries
WHERE documentId = $1
`
type GetDocumentCleanEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentCleanEntry
//
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) {
// SELECT id, documentId, bucket, key, version
// FROM currentCleanEntries
// WHERE documentId = $1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*Currentcleanentry, error) {
row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid)
var i GetDocumentCleanEntryRow
var i Currentcleanentry
err := row.Scan(
&i.ID,
&i.Documentid,
&i.Bucket,
&i.Key,
@@ -79,28 +61,14 @@ func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.U
const isDocumentClean = `-- name: IsDocumentClean :one
SELECT EXISTS(
SELECT 1
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
)
`
// IsDocumentClean
//
// SELECT EXISTS(
// SELECT 1
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// SELECT 1 FROM currentCleanEntries WHERE documentId = $1
// )
func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentClean, documentid)
internal/database/repository/clean_test.go
+7 -6
View File
@@ -9,6 +9,7 @@ import (
"queryorchestration/internal/test"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -65,10 +66,10 @@ func TestClean(t *testing.T) {
clean, err := queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentCleanEntryRow{
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
assert.Equal(t, id, clean.Documentid)
assert.Equal(t, bucket, clean.Bucket)
assert.Equal(t, key, clean.Key)
assert.Equal(t, int32(1), clean.Version)
assert.NotEqual(t, pgtype.UUID{}, clean.ID)
assert.True(t, clean.ID.Valid)
}
internal/database/repository/client.sql.go
+6 -18
View File
@@ -43,15 +43,9 @@ func (q *Queries) CreateClient(ctx context.Context, name string) (pgtype.UUID, e
}
const getClient = `-- name: GetClient :one
SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
SELECT c.id, c.name, cs.canSync
FROM clients as c
LEFT JOIN (
SELECT clientId, canSync
FROM clientCanSync
WHERE clientId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.clientId = c.id
JOIN currentClientCanSync as cs on cs.clientId = c.id
WHERE c.id = $1
`
@@ -63,18 +57,12 @@ type GetClientRow struct {
// GetClient
//
// SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
// SELECT c.id, c.name, cs.canSync
// FROM clients as c
// LEFT JOIN (
// SELECT clientId, canSync
// FROM clientCanSync
// WHERE clientId = $1
// ORDER BY id DESC
// LIMIT 1
// ) as cs on cs.clientId = c.id
// JOIN currentClientCanSync as cs on cs.clientId = c.id
// WHERE c.id = $1
func (q *Queries) GetClient(ctx context.Context, clientid pgtype.UUID) (*GetClientRow, error) {
row := q.db.QueryRow(ctx, getClient, clientid)
func (q *Queries) GetClient(ctx context.Context, id pgtype.UUID) (*GetClientRow, error) {
row := q.db.QueryRow(ctx, getClient, id)
var i GetClientRow
err := row.Scan(&i.ID, &i.Name, &i.Cansync)
return &i, err
internal/database/repository/collector.sql.go
+35 -52
View File
@@ -12,11 +12,11 @@ import (
)
const addCollectorQuery = `-- name: AddCollectorQuery :exec
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
`
type AddCollectorQueryParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
@@ -24,10 +24,10 @@ type AddCollectorQueryParams struct {
// AddCollectorQuery
//
// INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
// INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4)
func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryParams) error {
_, err := q.db.Exec(ctx, addCollectorQuery,
arg.Collectorid,
arg.Jobid,
arg.Name,
arg.Queryid,
arg.Addedversion,
@@ -36,45 +36,30 @@ func (q *Queries) AddCollectorQuery(ctx context.Context, arg *AddCollectorQueryP
}
const addLatestCollectorVersion = `-- name: AddLatestCollectorVersion :one
INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id
INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id
`
// AddLatestCollectorVersion
//
// INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id
func (q *Queries) AddLatestCollectorVersion(ctx context.Context, collectorid pgtype.UUID) (int32, error) {
row := q.db.QueryRow(ctx, addLatestCollectorVersion, collectorid)
// INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id
func (q *Queries) AddLatestCollectorVersion(ctx context.Context, jobid pgtype.UUID) (int32, error) {
row := q.db.QueryRow(ctx, addLatestCollectorVersion, jobid)
var id int32
err := row.Scan(&id)
return id, err
}
const createCollector = `-- name: CreateCollector :one
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
`
// CreateCollector
//
// INSERT INTO collectors (jobId) VALUES ($1) RETURNING id
func (q *Queries) CreateCollector(ctx context.Context, jobid pgtype.UUID) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, createCollector, jobid)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const getCollector = `-- name: GetCollector :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
// GetCollector
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE id = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, id)
// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollector(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollector, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
@@ -86,17 +71,16 @@ func (q *Queries) GetCollector(ctx context.Context, id pgtype.UUID) (*Fullactive
}
const getCollectorByJobID = `-- name: GetCollectorByJobID :one
SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
`
// GetCollectorByJobID
//
// SELECT id, jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
// SELECT jobid, mincleanversion, mintextversion, activeversion, latestversion, fields FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1
func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*Fullactivecollector, error) {
row := q.db.QueryRow(ctx, getCollectorByJobID, jobid)
var i Fullactivecollector
err := row.Scan(
&i.ID,
&i.Jobid,
&i.Mincleanversion,
&i.Mintextversion,
@@ -108,14 +92,14 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*
}
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1
`
// ListCollectorQueries
//
// SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
// SELECT jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE jobId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, jobid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, jobid)
if err != nil {
return nil, err
}
@@ -124,7 +108,6 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
for rows.Next() {
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Jobid,
&i.Queryid,
&i.Type,
@@ -142,72 +125,72 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
}
const removeCollectorQuery = `-- name: RemoveCollectorQuery :exec
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null
`
type RemoveCollectorQueryParams struct {
Removedversion *int32 `db:"removedversion"`
Queryid pgtype.UUID `db:"queryid"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
}
// RemoveCollectorQuery
//
// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null
// UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null
func (q *Queries) RemoveCollectorQuery(ctx context.Context, arg *RemoveCollectorQueryParams) error {
_, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Collectorid)
_, err := q.db.Exec(ctx, removeCollectorQuery, arg.Removedversion, arg.Queryid, arg.Jobid)
return err
}
const setActiveCollectorVersion = `-- name: SetActiveCollectorVersion :exec
INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2)
INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2)
`
type SetActiveCollectorVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Versionid int32 `db:"versionid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
}
// SetActiveCollectorVersion
//
// INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2)
// INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2)
func (q *Queries) SetActiveCollectorVersion(ctx context.Context, arg *SetActiveCollectorVersionParams) error {
_, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Collectorid, arg.Versionid)
_, err := q.db.Exec(ctx, setActiveCollectorVersion, arg.Jobid, arg.Versionid)
return err
}
const setCollectorCleanVersion = `-- name: SetCollectorCleanVersion :exec
INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
`
type SetCollectorCleanVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
}
// SetCollectorCleanVersion
//
// INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
// INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
func (q *Queries) SetCollectorCleanVersion(ctx context.Context, arg *SetCollectorCleanVersionParams) error {
_, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Collectorid, arg.Addedversion, arg.Versionid)
_, err := q.db.Exec(ctx, setCollectorCleanVersion, arg.Jobid, arg.Addedversion, arg.Versionid)
return err
}
const setCollectorTextVersion = `-- name: SetCollectorTextVersion :exec
INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
`
type SetCollectorTextVersionParams struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Addedversion int32 `db:"addedversion"`
Versionid int32 `db:"versionid"`
}
// SetCollectorTextVersion
//
// INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3)
// INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3)
func (q *Queries) SetCollectorTextVersion(ctx context.Context, arg *SetCollectorTextVersionParams) error {
_, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Collectorid, arg.Addedversion, arg.Versionid)
_, err := q.db.Exec(ctx, setCollectorTextVersion, arg.Jobid, arg.Addedversion, arg.Versionid)
return err
}
internal/database/repository/collector_test.go
+27 -35
View File
@@ -38,11 +38,16 @@ func TestCollector(t *testing.T) {
assert.NoError(t, err)
version, err := queries.AddLatestQueryVersion(ctx, jsonId)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonId,
Versionid: version,
})
assert.NoError(t, err)
assert.Equal(t, int32(1), version)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonId,
Requiredqueryid: contextId,
Addedversion: 1,
Addedversion: version,
})
assert.NoError(t, err)
@@ -53,13 +58,9 @@ func TestCollector(t *testing.T) {
minCleanVersion := int32(2)
minTextVersion := int32(4)
collId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
coll, err := queries.GetCollector(ctx, collId)
coll, err := queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -70,7 +71,6 @@ func TestCollector(t *testing.T) {
coll, err = queries.GetCollectorByJobID(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -78,14 +78,13 @@ func TestCollector(t *testing.T) {
Latestversion: 0,
}, coll)
version, err = queries.AddLatestCollectorVersion(ctx, collId)
version, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, int32(1), version)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -94,15 +93,14 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 1,
Collectorid: collId,
Versionid: 1,
Jobid: jobId,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: 0,
@@ -111,7 +109,7 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Addedversion: 1,
Name: "example_key",
@@ -119,16 +117,15 @@ func TestCollector(t *testing.T) {
assert.NoError(t, err)
err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: collId,
Jobid: jobId,
Addedversion: 1,
Versionid: minTextVersion,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: 0,
Mintextversion: minTextVersion,
@@ -138,16 +135,15 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: collId,
Jobid: jobId,
Addedversion: 1,
Versionid: minCleanVersion,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion,
@@ -156,20 +152,18 @@ func TestCollector(t *testing.T) {
Fields: []byte(fmt.Sprintf("{\"example_key\": \"%s\"}", database.MustToUUID(jsonId).String())),
}, coll)
qs, err := queries.ListCollectorQueries(ctx, collId)
qs, err := queries.ListCollectorQueries(ctx, jobId)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Queryversion: 0,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
Requiredids: []pgtype.UUID{contextId},
},
{
Collectorid: collId,
Jobid: jobId,
Queryid: contextId,
Queryversion: 0,
@@ -178,34 +172,33 @@ func TestCollector(t *testing.T) {
},
}, qs)
version, err = queries.AddLatestCollectorVersion(ctx, collId)
version, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
assert.Equal(t, int32(2), version)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Removedversion: &version,
})
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 2,
Collectorid: collId,
Versionid: 2,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.SetCollectorTextVersion(ctx, &repository.SetCollectorTextVersionParams{
Collectorid: collId,
Jobid: jobId,
Versionid: minTextVersion + 1,
Addedversion: version,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion,
Mintextversion: minTextVersion + 1,
@@ -215,16 +208,15 @@ func TestCollector(t *testing.T) {
}, coll)
err = queries.SetCollectorCleanVersion(ctx, &repository.SetCollectorCleanVersionParams{
Collectorid: collId,
Jobid: jobId,
Versionid: minCleanVersion + 1,
Addedversion: version,
})
assert.NoError(t, err)
coll, err = queries.GetCollector(ctx, collId)
coll, err = queries.GetCollector(ctx, jobId)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivecollector{
ID: collId,
Jobid: jobId,
Mincleanversion: minCleanVersion + 1,
Mintextversion: minTextVersion + 1,
internal/database/repository/job.sql.go
+166 -18
View File
@@ -43,15 +43,9 @@ func (q *Queries) CreateJob(ctx context.Context, clientid pgtype.UUID) (pgtype.U
}
const getJob = `-- name: GetJob :one
SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
SELECT j.id, j.clientId, cs.canSync
FROM jobs as j
LEFT JOIN (
SELECT jobId, canSync
FROM jobCanSync
WHERE jobId = $1
ORDER BY id DESC
LIMIT 1
) as cs on cs.jobId = j.id
JOIN currentJobCanSync as cs on cs.jobId = j.id
WHERE j.id = $1
`
@@ -63,23 +57,177 @@ type GetJobRow struct {
// GetJob
//
// SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
// SELECT j.id, j.clientId, cs.canSync
// FROM jobs as j
// LEFT JOIN (
// SELECT jobId, canSync
// FROM jobCanSync
// WHERE jobId = $1
// ORDER BY id DESC
// LIMIT 1
// ) as cs on cs.jobId = j.id
// JOIN currentJobCanSync as cs on cs.jobId = j.id
// WHERE j.id = $1
func (q *Queries) GetJob(ctx context.Context, jobid pgtype.UUID) (*GetJobRow, error) {
row := q.db.QueryRow(ctx, getJob, jobid)
func (q *Queries) GetJob(ctx context.Context, id pgtype.UUID) (*GetJobRow, error) {
row := q.db.QueryRow(ctx, getJob, id)
var i GetJobRow
err := row.Scan(&i.ID, &i.Clientid, &i.Cansync)
return &i, err
}
const isJobSynced = `-- name: IsJobSynced :one
WITH
docs AS (
-- Get all documents for this job
SELECT id, jobId FROM documents WHERE jobId = $1
),
doc_text_entries AS (
-- Documents with their current text entries
SELECT
d.id AS document_id,
cte.id AS text_entry_id
FROM
docs d
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
),
required_results AS (
-- All required document-query-version combinations
SELECT
d.id AS document_id,
cqdt.queryId,
cqdt.queryVersion,
dte.text_entry_id
FROM
docs d
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
JOIN doc_text_entries dte ON dte.document_id = d.id
and dte.text_entry_id IS NOT NULL
),
existing_results AS (
-- Valid results that exist
SELECT
rr.document_id AS document_id,
r.queryId,
r.id AS result_id
FROM
results r
JOIN required_results rr ON
r.queryId = rr.queryId AND
r.queryVersion = rr.queryVersion AND
r.textEntryId = rr.text_entry_id
),
missing_results AS (
-- Find missing results
SELECT rr.queryId
FROM required_results rr
LEFT JOIN existing_results er on er.queryId = rr.queryId
WHERE er.result_id is null
),
dependency_check AS (
-- Check for missing dependencies
SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
FROM existing_results er
JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
WHERE rd.resultId is null
)
SELECT (
-- No documents means job is synced
NOT EXISTS (SELECT 1 FROM docs)
OR
-- Documents with no text entries
(NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
and
-- Documents with missing results
NOT EXISTS (SELECT 1 FROM missing_results)
and
-- Documents with missing dependencies
NOT EXISTS (SELECT 1 FROM dependency_check))
)::bool as is_synced
`
// IsJobSynced
//
// WITH
// docs AS (
// -- Get all documents for this job
// SELECT id, jobId FROM documents WHERE jobId = $1
// ),
// doc_text_entries AS (
// -- Documents with their current text entries
// SELECT
// d.id AS document_id,
// cte.id AS text_entry_id
// FROM
// docs d
// LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
// ),
// required_results AS (
// -- All required document-query-version combinations
// SELECT
// d.id AS document_id,
// cqdt.queryId,
// cqdt.queryVersion,
// dte.text_entry_id
// FROM
// docs d
// JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
// JOIN doc_text_entries dte ON dte.document_id = d.id
// and dte.text_entry_id IS NOT NULL
// ),
// existing_results AS (
// -- Valid results that exist
// SELECT
// rr.document_id AS document_id,
// r.queryId,
// r.id AS result_id
// FROM
// results r
// JOIN required_results rr ON
// r.queryId = rr.queryId AND
// r.queryVersion = rr.queryVersion AND
// r.textEntryId = rr.text_entry_id
// ),
// missing_results AS (
// -- Find missing results
// SELECT rr.queryId
// FROM required_results rr
// LEFT JOIN existing_results er on er.queryId = rr.queryId
// WHERE er.result_id is null
// ),
// dependency_check AS (
// -- Check for missing dependencies
// SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
// FROM existing_results er
// JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
// LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
// WHERE rd.resultId is null
// )
// SELECT (
// -- No documents means job is synced
// NOT EXISTS (SELECT 1 FROM docs)
//
// OR
//
// -- Documents with no text entries
// (NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
//
// and
//
// -- Documents with missing results
// NOT EXISTS (SELECT 1 FROM missing_results)
//
// and
//
// -- Documents with missing dependencies
// NOT EXISTS (SELECT 1 FROM dependency_check))
// )::bool as is_synced
func (q *Queries) IsJobSynced(ctx context.Context, dollar_1 pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isJobSynced, dollar_1)
var is_synced bool
err := row.Scan(&is_synced)
return is_synced, err
}
const listJobDocumentIDsBatch = `-- name: ListJobDocumentIDsBatch :many
SELECT id, totalCount FROM listJobDocumentIDs($1, $2, $3)
`
internal/database/repository/job_test.go
+425
View File
@@ -164,3 +164,428 @@ func TestListJobDocumentIDs(t *testing.T) {
},
}, ids)
}
func TestJobSync(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
queries := cfg.GetDBQueries()
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 1,
})
assert.NoError(t, err)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
clientId, err := queries.CreateClient(ctx, "example_client")
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 1,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
isSynced, err := queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
cleanentry, err = queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
textentry, err = queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
_, err = queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
collversion := int32(2)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: 2,
Jobid: jobId,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.RemoveCollectorQuery(ctx, &repository.RemoveCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Removedversion: &collversion,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 2,
Name: "second_key",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: contextQueryID,
Addedversion: 2,
Name: "example_key",
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
superQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, superQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: superQueryID,
Versionid: 1,
})
assert.NoError(t, err)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: contextQueryID,
Requiredqueryid: superQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
superresultid, err := queries.AddResult(ctx, &repository.AddResultParams{
Queryid: superQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
depresultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 2,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
resultid, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Value: "example_value",
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: resultid,
Requiredresultid: depresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, isSynced)
err = queries.AddResultDependency(ctx, &repository.AddResultDependencyParams{
Resultid: depresultid,
Requiredresultid: superresultid,
})
assert.NoError(t, err)
isSynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, isSynced)
}
internal/database/repository/models.go
+90 -31
View File
@@ -73,30 +73,25 @@ type Clientcansync struct {
Cansync bool `db:"cansync"`
}
type Collector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
}
type Collectoractiveversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Versionid int32 `db:"versionid"`
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
}
type Collectorcurrentactiveversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Activeversion int32 `db:"activeversion"`
}
type Collectorlatestversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Latestversion int32 `db:"latestversion"`
}
type Collectormincleanversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
@@ -104,7 +99,7 @@ type Collectormincleanversion struct {
type Collectormintextversion struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Versionid int32 `db:"versionid"`
Addedversion int32 `db:"addedversion"`
Removedversion *int32 `db:"removedversion"`
@@ -112,7 +107,7 @@ type Collectormintextversion struct {
type Collectorquery struct {
ID pgtype.UUID `db:"id"`
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Name string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
@@ -120,7 +115,6 @@ type Collectorquery struct {
}
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
@@ -129,9 +123,57 @@ type Collectorquerydependencytree struct {
}
type Collectorversion struct {
Collectorid pgtype.UUID `db:"collectorid"`
ID int32 `db:"id"`
Addedat pgtype.Timestamp `db:"addedat"`
Jobid pgtype.UUID `db:"jobid"`
ID int32 `db:"id"`
Addedat pgtype.Timestamp `db:"addedat"`
}
type Currentcleanentry struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
type Currentclientcansync struct {
Clientid pgtype.UUID `db:"clientid"`
Cansync bool `db:"cansync"`
}
type Currentcollectormincleanversion struct {
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
}
type Currentcollectormintextversion struct {
Jobid pgtype.UUID `db:"jobid"`
Mintextversion int32 `db:"mintextversion"`
}
type Currentcollectorqueriesjsonagg struct {
Jobid pgtype.UUID `db:"jobid"`
Fields []byte `db:"fields"`
}
type Currentcollectorquery struct {
Jobid pgtype.UUID `db:"jobid"`
Name *string `db:"name"`
Queryid pgtype.UUID `db:"queryid"`
}
type Currentjobcansync struct {
Jobid pgtype.UUID `db:"jobid"`
Cansync bool `db:"cansync"`
}
type Currenttextentry struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
}
type Document struct {
@@ -156,15 +198,14 @@ type Documententry struct {
}
type Documenttextextraction struct {
ID pgtype.UUID `db:"id"`
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
ID pgtype.UUID `db:"id"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
}
type Fullactivecollector struct {
ID pgtype.UUID `db:"id"`
Jobid pgtype.UUID `db:"jobid"`
Mincleanversion int32 `db:"mincleanversion"`
Mintextversion int32 `db:"mintextversion"`
@@ -218,13 +259,28 @@ type Queryconfig struct {
}
type Querycurrentactiveversion struct {
Queryid pgtype.UUID `db:"queryid"`
Activeversion int32 `db:"activeversion"`
}
type Querycurrentconfig struct {
Queryid pgtype.UUID `db:"queryid"`
ID int32 `db:"id"`
Config []byte `db:"config"`
}
type Querycurrentrequiredid struct {
Queryid pgtype.UUID `db:"queryid"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
}
type Querycurrentrequiredidsagg struct {
Queryid pgtype.UUID `db:"queryid"`
Requiredids []pgtype.UUID `db:"requiredids"`
}
type Querylatestversion struct {
Queryid pgtype.UUID `db:"queryid"`
ID int32 `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Latestversion int32 `db:"latestversion"`
}
type Queryversion struct {
@@ -243,10 +299,13 @@ type Requiredquery struct {
type Result struct {
ID pgtype.UUID `db:"id"`
Textentryid pgtype.UUID `db:"textentryid"`
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
type Resultdependency struct {
Resultid pgtype.UUID `db:"resultid"`
Requiredresultid pgtype.UUID `db:"requiredresultid"`
}
internal/database/repository/query.sql.go
+56 -56
View File
@@ -92,6 +92,20 @@ func (q *Queries) CreateQuery(ctx context.Context, type_ Querytype) (pgtype.UUID
return id, err
}
const getActiveQueryConfig = `-- name: GetActiveQueryConfig :one
SELECT config FROM queryCurrentConfigs where queryId = $1
`
// GetActiveQueryConfig
//
// SELECT config FROM queryCurrentConfigs where queryId = $1
func (q *Queries) GetActiveQueryConfig(ctx context.Context, queryid pgtype.UUID) ([]byte, error) {
row := q.db.QueryRow(ctx, getActiveQueryConfig, queryid)
var config []byte
err := row.Scan(&config)
return config, err
}
const getQuery = `-- name: GetQuery :one
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries WHERE id = $1
`
@@ -113,30 +127,6 @@ func (q *Queries) GetQuery(ctx context.Context, id pgtype.UUID) (*Fullactivequer
return &i, err
}
const getQueryConfig = `-- name: GetQueryConfig :one
SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2
`
type GetQueryConfigParams struct {
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
}
type GetQueryConfigRow struct {
ID pgtype.UUID `db:"id"`
Config []byte `db:"config"`
}
// GetQueryConfig
//
// SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2
func (q *Queries) GetQueryConfig(ctx context.Context, arg *GetQueryConfigParams) (*GetQueryConfigRow, error) {
row := q.db.QueryRow(ctx, getQueryConfig, arg.Queryid, arg.Addedversion)
var i GetQueryConfigRow
err := row.Scan(&i.ID, &i.Config)
return &i, err
}
const getQueryWithVersion = `-- name: GetQueryWithVersion :one
WITH query as (
SELECT id, type FROM queries WHERE id = $1
@@ -149,22 +139,23 @@ SELECT c.queryId, c.config
),
requiredIds as (
SELECT r.queryId,
coalesce(
ARRAY_AGG(DISTINCT r.requiredQueryId)
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
as requiredIds
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM query AS q
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion($2, r.addedVersion, r.removedVersion)
GROUP BY r.queryId
)
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
coalesce(
r.requiredIds,
array[]::uuid[]
)::uuid[] as requiredIds
SELECT DISTINCT q.id, q.type,
av.activeVersion,
lv.latestVersion, c.config,
r.requiredIds
FROM query AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.id
LEFT JOIN config AS c ON q.id = c.queryId
LEFT JOIN requiredIds AS r ON q.id = r.queryId
`
@@ -196,22 +187,23 @@ type GetQueryWithVersionRow struct {
// ),
// requiredIds as (
// SELECT r.queryId,
// coalesce(
// ARRAY_AGG(DISTINCT r.requiredQueryId)
// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
// as requiredIds
// FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
// array[]::uuid[]
// )::uuid[] as requiredIds
// FROM query AS q
// LEFT JOIN requiredQueries AS r ON q.id = r.queryId
// and isInVersion($2, r.addedVersion, r.removedVersion)
// GROUP BY r.queryId
// )
// SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
// coalesce(
// r.requiredIds,
// array[]::uuid[]
// )::uuid[] as requiredIds
// SELECT DISTINCT q.id, q.type,
// av.activeVersion,
// lv.latestVersion, c.config,
// r.requiredIds
// FROM query AS q
// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
// JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// JOIN queryLatestVersions as lv on lv.queryId = q.id
// LEFT JOIN config AS c ON q.id = c.queryId
// LEFT JOIN requiredIds AS r ON q.id = r.queryId
func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVersionParams) (*GetQueryWithVersionRow, error) {
@@ -230,22 +222,28 @@ func (q *Queries) GetQueryWithVersion(ctx context.Context, arg *GetQueryWithVers
const isQueryInDependencyTree = `-- name: IsQueryInDependencyTree :one
SELECT EXISTS (
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
SELECT 1 FROM queryActiveDependencies
WHERE id = any($1)
and requiredQueryId = $2
or $2 = any($1)
)
`
type IsQueryInDependencyTreeParams struct {
ID []pgtype.UUID `db:"id"`
Requiredqueryid pgtype.UUID `db:"requiredqueryid"`
Requiredqueryids []pgtype.UUID `db:"requiredqueryids"`
Queryid pgtype.UUID `db:"queryid"`
}
// IsQueryInDependencyTree
//
// SELECT EXISTS (
// SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
// SELECT 1 FROM queryActiveDependencies
// WHERE id = any($1)
// and requiredQueryId = $2
// or $2 = any($1)
// )
func (q *Queries) IsQueryInDependencyTree(ctx context.Context, arg *IsQueryInDependencyTreeParams) (bool, error) {
row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.ID, arg.Requiredqueryid)
row := q.db.QueryRow(ctx, isQueryInDependencyTree, arg.Requiredqueryids, arg.Queryid)
var exists bool
err := row.Scan(&exists)
return exists, err
@@ -324,14 +322,15 @@ WITH doc AS (
SELECT id, jobId FROM documents where id = $2
)
SELECT dt.queryId
FROM collectorQueryDependencyTree as dt
JOIN doc as d on d.jobId = dt.jobId
where $1 = any(dt.requiredIds)
FROM doc as d
JOIN collectorQueryDependencyTree as dt
on d.jobId = dt.jobId
and $1 = any(dt.requiredIds)
`
type ListQueryDirectDependentsByDocumentIDParams struct {
Requiredids pgtype.UUID `db:"requiredids"`
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
}
// ListQueryDirectDependentsByDocumentID
@@ -340,11 +339,12 @@ type ListQueryDirectDependentsByDocumentIDParams struct {
// SELECT id, jobId FROM documents where id = $2
// )
// SELECT dt.queryId
// FROM collectorQueryDependencyTree as dt
// JOIN doc as d on d.jobId = dt.jobId
// where $1 = any(dt.requiredIds)
// FROM doc as d
// JOIN collectorQueryDependencyTree as dt
// on d.jobId = dt.jobId
// and $1 = any(dt.requiredIds)
func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Requiredids, arg.ID)
rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Queryid, arg.Documentid)
if err != nil {
return nil, err
}
internal/database/repository/query_test.go
+95 -82
View File
@@ -152,13 +152,6 @@ func TestQueries(t *testing.T) {
})
assert.NoError(t, err)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: []byte(`{"second":"key"}`),
Addedversion: 2,
})
assert.NoError(t, err)
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: jsonConfig,
@@ -166,29 +159,40 @@ func TestQueries(t *testing.T) {
})
assert.NoError(t, err)
jsonQueryConfig, err := queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
Queryid: jsonQueryID,
Addedversion: jsonQuery.Activeversion,
})
jsonQueryConfig, err := queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig.Config)
assert.Equal(t, jsonConfig, jsonQueryConfig)
jsonQueryConfig, err = queries.GetQueryConfig(ctx, &repository.GetQueryConfigParams{
err = queries.SetQueryConfig(ctx, &repository.SetQueryConfigParams{
Queryid: jsonQueryID,
Config: []byte(`{"second":"key"}`),
Addedversion: 2,
})
assert.NoError(t, err)
assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig.Config)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, jsonConfig, jsonQueryConfig)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 2,
})
assert.NoError(t, err)
jsonQueryConfig, err = queries.GetActiveQueryConfig(ctx, jsonQueryID)
assert.NoError(t, err)
assert.Equal(t, []byte(`{"second": "key"}`), jsonQueryConfig)
jsonQuery, err = queries.GetQuery(ctx, jsonQueryID)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.Fullactivequery{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Activeversion: 2,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
Config: jsonQueryConfig,
Requiredids: []pgtype.UUID{},
}, jsonQuery)
v := int32(1)
@@ -200,7 +204,7 @@ func TestQueries(t *testing.T) {
assert.EqualExportedValues(t, &repository.GetQueryWithVersionRow{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Activeversion: 2,
Latestversion: 2,
Config: jsonConfig,
Requiredids: []pgtype.UUID{contextQueryID},
@@ -248,13 +252,11 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
jobID, err := queries.CreateJob(ctx, clientID)
assert.NoError(t, err)
collID, err := queries.CreateCollector(ctx, jobID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collID)
version, err := queries.AddLatestCollectorVersion(ctx, jobID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collID,
Versionid: 1,
Jobid: jobID,
Versionid: version,
})
assert.NoError(t, err)
docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
@@ -265,10 +267,17 @@ func TestQueryDependencyTree(t *testing.T) {
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, contextQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: contextQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
@@ -277,10 +286,15 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, jsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: jsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
@@ -293,20 +307,20 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Jobid: jobID,
Name: "example_name'",
Queryid: jsonQueryID,
Addedversion: 1,
@@ -314,14 +328,14 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
@@ -330,16 +344,21 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
_, err = queries.AddLatestQueryVersion(ctx, secondJsonQueryID)
assert.NoError(t, err)
err = queries.AddActiveQueryVersion(ctx, &repository.AddActiveQueryVersionParams{
Queryid: secondJsonQueryID,
Versionid: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
@@ -352,26 +371,26 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Jobid: jobID,
Name: "example_name'",
Queryid: secondJsonQueryID,
Addedversion: 1,
@@ -379,55 +398,55 @@ func TestQueryDependencyTree(t *testing.T) {
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
Documentid: docID,
Queryid: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
Documentid: docID,
Queryid: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
Documentid: docID,
Queryid: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{contextQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{secondJsonQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{jsonQueryID},
Queryid: jsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: secondJsonQueryID,
ID: []pgtype.UUID{jsonQueryID, contextQueryID},
Queryid: secondJsonQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, contextQueryID},
})
assert.NoError(t, err)
assert.False(t, isdependent)
isdependent, err = queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: contextQueryID,
ID: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
Queryid: contextQueryID,
Requiredqueryids: []pgtype.UUID{jsonQueryID, secondJsonQueryID},
})
assert.NoError(t, err)
assert.True(t, isdependent)
@@ -521,17 +540,15 @@ func TestListQueryJobs(t *testing.T) {
assert.NoError(t, err)
jobOneID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
collOneID, err := queries.CreateCollector(ctx, jobOneID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collOneID)
versionOne, err := queries.AddLatestCollectorVersion(ctx, jobOneID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collOneID,
Versionid: 1,
Jobid: jobOneID,
Versionid: versionOne,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collOneID,
Jobid: jobOneID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -544,17 +561,15 @@ func TestListQueryJobs(t *testing.T) {
jobTwoID, err := queries.CreateJob(ctx, clientOneID)
assert.NoError(t, err)
collTwoID, err := queries.CreateCollector(ctx, jobTwoID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collTwoID)
versionTwo, err := queries.AddLatestCollectorVersion(ctx, jobTwoID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collTwoID,
Versionid: 1,
Jobid: jobTwoID,
Versionid: versionTwo,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collTwoID,
Jobid: jobTwoID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -569,17 +584,15 @@ func TestListQueryJobs(t *testing.T) {
assert.NoError(t, err)
jobThreeID, err := queries.CreateJob(ctx, clientTwoID)
assert.NoError(t, err)
collThreeID, err := queries.CreateCollector(ctx, jobThreeID)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collThreeID)
versionThree, err := queries.AddLatestCollectorVersion(ctx, jobThreeID)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collThreeID,
Versionid: 1,
Jobid: jobThreeID,
Versionid: versionThree,
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collThreeID,
Jobid: jobThreeID,
Queryid: contextID,
Addedversion: 1,
Name: "example_key",
@@ -601,7 +614,7 @@ func TestListQueryJobs(t *testing.T) {
})
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collOneID,
Jobid: jobOneID,
Queryid: jsonID,
Addedversion: 1,
Name: "example_key",
internal/database/repository/result.sql.go
+133 -120
View File
@@ -11,34 +11,91 @@ import (
"github.com/jackc/pgx/v5/pgtype"
)
const addResult = `-- name: AddResult :one
INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
`
type AddResultParams struct {
Queryid pgtype.UUID `db:"queryid"`
Value string `db:"value"`
Textentryid pgtype.UUID `db:"textentryid"`
Queryversion int32 `db:"queryversion"`
}
// AddResult
//
// INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id
func (q *Queries) AddResult(ctx context.Context, arg *AddResultParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, addResult,
arg.Queryid,
arg.Value,
arg.Textentryid,
arg.Queryversion,
)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
}
const addResultDependency = `-- name: AddResultDependency :exec
INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
`
type AddResultDependencyParams struct {
Resultid pgtype.UUID `db:"resultid"`
Requiredresultid pgtype.UUID `db:"requiredresultid"`
}
// AddResultDependency
//
// INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2)
func (q *Queries) AddResultDependency(ctx context.Context, arg *AddResultDependencyParams) error {
_, err := q.db.Exec(ctx, addResultDependency, arg.Resultid, arg.Requiredresultid)
return err
}
const getResultValueWithVersion = `-- name: GetResultValueWithVersion :one
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
WITH doc as (
SELECT id, jobId
FROM documents
WHERE id = $3
)
SELECT r.id, r.value
FROM doc as d
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = $1
and r.queryVersion = $2
and r.textEntryId = cte.id
`
type GetResultValueWithVersionParams struct {
Queryid pgtype.UUID `db:"queryid"`
Queryversion int32 `db:"queryversion"`
Queryversion *int32 `db:"queryversion"`
Documentid pgtype.UUID `db:"documentid"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
}
type GetResultValueWithVersionRow struct {
ID pgtype.UUID `db:"id"`
Value string `db:"value"`
Value *string `db:"value"`
}
// GetResultValueWithVersion
//
// SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5
// WITH doc as (
// SELECT id, jobId
// FROM documents
// WHERE id = $3
// )
// SELECT r.id, r.value
// FROM doc as d
// JOIN currentTextEntries as cte on cte.documentId = d.id
// LEFT JOIN results as r
// on r.queryId = $1
// and r.queryVersion = $2
// and r.textEntryId = cte.id
func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultValueWithVersionParams) (*GetResultValueWithVersionRow, error) {
row := q.db.QueryRow(ctx, getResultValueWithVersion,
arg.Queryid,
arg.Queryversion,
arg.Documentid,
arg.Cleanversion,
arg.Textversion,
)
row := q.db.QueryRow(ctx, getResultValueWithVersion, arg.Queryid, arg.Queryversion, arg.Documentid)
var i GetResultValueWithVersionRow
err := row.Scan(&i.ID, &i.Value)
return &i, err
@@ -46,131 +103,110 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV
const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
WITH reqQueries as (
SELECT q.id as queryId, av.id as activeVersion, q.type
SELECT av.queryId, av.activeVersion, q.type
FROM requiredQueries as rq
JOIN queries as q on q.id = rq.requiredQueryId
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
WHERE rq.queryId = $2
and isInVersion($3, rq.addedVersion, rq.removedVersion)
JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
JOIN queries as q on q.id = av.queryId
WHERE rq.queryId = $1
and isInVersion($2, rq.addedVersion, rq.removedVersion)
),
docs as (
SELECT id, jobId
FROM documents
WHERE id = $1
WHERE id = $3
),
codeVersions as (
SELECT
SELECT
d.id as documentId,
coalesce(ccv.versionId, 1) as minCleanVersion,
coalesce(ctv.versionId, 1) as minTextVersion
mcv.minCleanVersion,
mtv.minTextVersion
FROM docs as d
LEFT JOIN collectors as c on c.jobId = d.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
LIMIT 1
JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
),
latestVersions AS (
SELECT
SELECT
r.id,
rq.queryId,
rq.type,
r.queryVersion,
r.cleanVersion,
r.textVersion,
r.textEntryId,
r.value,
ROW_NUMBER() OVER (
PARTITION BY rq.queryId
ORDER BY r.cleanVersion DESC, r.textVersion DESC
PARTITION BY r.queryId
ORDER BY r.id DESC
) as rowNumber
FROM reqQueries as rq
JOIN currentTextEntries as cte on cte.documentId = $3
LEFT JOIN results as r ON rq.queryId = r.queryId
and r.documentId = $1
and r.queryVersion = rq.activeVersion
JOIN codeVersions as ccv on ccv.documentId = r.documentId
and r.cleanVersion >= ccv.minCleanVersion
and r.textVersion >= ccv.minTextVersion
and cte.id = r.textEntryId
)
SELECT DISTINCT lv.queryId, lv.type, r.value
FROM latestVersions as lv
JOIN results as r ON r.queryId = lv.queryId
and r.documentId = $1
and r.queryVersion = lv.queryVersion
and r.cleanVersion = lv.cleanVersion
and r.textVersion = lv.textVersion
and lv.rowNumber = 1
SELECT DISTINCT id, queryId, type, value
FROM latestVersions
WHERE rowNumber = 1
`
type ListQueryRequirementValuesParams struct {
Documentid pgtype.UUID `db:"documentid"`
Queryid pgtype.UUID `db:"queryid"`
Version int32 `db:"version"`
Version *int32 `db:"version"`
Documentid pgtype.UUID `db:"documentid"`
}
type ListQueryRequirementValuesRow struct {
ID pgtype.UUID `db:"id"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Value string `db:"value"`
Value *string `db:"value"`
}
// ListQueryRequirementValues
//
// WITH reqQueries as (
// SELECT q.id as queryId, av.id as activeVersion, q.type
// SELECT av.queryId, av.activeVersion, q.type
// FROM requiredQueries as rq
// JOIN queries as q on q.id = rq.requiredQueryId
// LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
// WHERE rq.queryId = $2
// and isInVersion($3, rq.addedVersion, rq.removedVersion)
// JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
// JOIN queries as q on q.id = av.queryId
// WHERE rq.queryId = $1
// and isInVersion($2, rq.addedVersion, rq.removedVersion)
// ),
// docs as (
// SELECT id, jobId
// FROM documents
// WHERE id = $1
// WHERE id = $3
// ),
// codeVersions as (
// SELECT
// d.id as documentId,
// coalesce(ccv.versionId, 1) as minCleanVersion,
// coalesce(ctv.versionId, 1) as minTextVersion
// mcv.minCleanVersion,
// mtv.minTextVersion
// FROM docs as d
// LEFT JOIN collectors as c on c.jobId = d.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
// and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
// LIMIT 1
// JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
// JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
// ),
// latestVersions AS (
// SELECT
// r.id,
// rq.queryId,
// rq.type,
// r.queryVersion,
// r.cleanVersion,
// r.textVersion,
// r.textEntryId,
// r.value,
// ROW_NUMBER() OVER (
// PARTITION BY rq.queryId
// ORDER BY r.cleanVersion DESC, r.textVersion DESC
// PARTITION BY r.queryId
// ORDER BY r.id DESC
// ) as rowNumber
// FROM reqQueries as rq
// JOIN currentTextEntries as cte on cte.documentId = $3
// LEFT JOIN results as r ON rq.queryId = r.queryId
// and r.documentId = $1
// and r.queryVersion = rq.activeVersion
// JOIN codeVersions as ccv on ccv.documentId = r.documentId
// and r.cleanVersion >= ccv.minCleanVersion
// and r.textVersion >= ccv.minTextVersion
// and cte.id = r.textEntryId
// )
// SELECT DISTINCT lv.queryId, lv.type, r.value
// FROM latestVersions as lv
// JOIN results as r ON r.queryId = lv.queryId
// and r.documentId = $1
// and r.queryVersion = lv.queryVersion
// and r.cleanVersion = lv.cleanVersion
// and r.textVersion = lv.textVersion
// and lv.rowNumber = 1
// SELECT DISTINCT id, queryId, type, value
// FROM latestVersions
// WHERE rowNumber = 1
func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version)
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Queryid, arg.Version, arg.Documentid)
if err != nil {
return nil, err
}
@@ -178,7 +214,12 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
items := []*ListQueryRequirementValuesRow{}
for rows.Next() {
var i ListQueryRequirementValuesRow
if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil {
if err := rows.Scan(
&i.ID,
&i.Queryid,
&i.Type,
&i.Value,
); err != nil {
return nil, err
}
items = append(items, &i)
@@ -194,14 +235,14 @@ WITH docs as (
SELECT id, jobId from documents where id = $1
),
unsyncedQueries AS (
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
SELECT DISTINCT dt.queryId, dt.requiredIds
from docs as d
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
JOIN fullActiveCollectors as c on c.id = dt.collectorId
LEFT JOIN results as r on r.queryId = dt.queryId
and r.documentId = d.id
JOIN currentTextEntries as cte on cte.documentId = d.id
LEFT JOIN results as r
on r.queryId = dt.queryId
and r.queryVersion = dt.queryVersion
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
and cte.id = r.textEntryId
where r.value is null
)
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
@@ -216,14 +257,14 @@ SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
// SELECT id, jobId from documents where id = $1
// ),
// unsyncedQueries AS (
// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
// SELECT DISTINCT dt.queryId, dt.requiredIds
// from docs as d
// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
// JOIN fullActiveCollectors as c on c.id = dt.collectorId
// LEFT JOIN results as r on r.queryId = dt.queryId
// and r.documentId = d.id
// JOIN currentTextEntries as cte on cte.documentId = d.id
// LEFT JOIN results as r
// on r.queryId = dt.queryId
// and r.queryVersion = dt.queryVersion
// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
// and cte.id = r.textEntryId
// where r.value is null
// )
// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
@@ -249,31 +290,3 @@ func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1
}
return items, nil
}
const setResult = `-- name: SetResult :exec
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
`
type SetResultParams struct {
Queryid pgtype.UUID `db:"queryid"`
Documentid pgtype.UUID `db:"documentid"`
Value string `db:"value"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Queryversion int32 `db:"queryversion"`
}
// SetResult
//
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error {
_, err := q.db.Exec(ctx, setResult,
arg.Queryid,
arg.Documentid,
arg.Value,
arg.Cleanversion,
arg.Textversion,
arg.Queryversion,
)
return err
}
internal/database/repository/result_test.go
+215 -92
View File
@@ -44,38 +44,98 @@ func TestResults(t *testing.T) {
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
issynced, err := queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash",
})
assert.NoError(t, err)
jsonQuery, err := queries.GetQuery(ctx, jsonQueryID)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
cleanVersion := int32(1)
textVersion := int32(1)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Jobid: jobId,
Queryid: jsonQueryID,
Addedversion: 1,
Name: "example_key",
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.False(t, issynced)
jsonResultValue := "example_value"
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: jsonResultValue,
Cleanversion: cleanVersion,
Textversion: textVersion,
Queryversion: jsonQuery.Activeversion,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
issynced, err = queries.IsJobSynced(ctx, jobId)
assert.NoError(t, err)
assert.True(t, issynced)
qv := int32(1)
res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: jsonQueryID,
Queryversion: jsonQuery.Activeversion,
Queryversion: &qv,
Documentid: documentID,
Cleanversion: cleanVersion,
Textversion: textVersion,
})
assert.NoError(t, err)
assert.EqualExportedValues(t, jsonResultValue, res.Value)
assert.NotNil(t, res.Value)
assert.Equal(t, jsonResultValue, *res.Value)
}
func TestResultValues(t *testing.T) {
@@ -147,58 +207,79 @@ func TestResultValues(t *testing.T) {
qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
result := repository.SetResultParams{
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
result := repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_1",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion,
}
err = queries.SetResult(ctx, &result)
_, err = queries.AddResult(ctx, &result)
assert.NoError(t, err)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_1", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_2",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion - 1,
})
assert.NoError(t, err)
@@ -206,24 +287,20 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_1", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value_3",
Cleanversion: 2,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: contextQuery.Activeversion,
})
assert.NoError(t, err)
@@ -231,24 +308,20 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_3",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_3", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "json_value_1",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: jsonVersion,
})
assert.NoError(t, err)
@@ -256,17 +329,15 @@ func TestResultValues(t *testing.T) {
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
Version: &jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_3",
},
}, qResults)
assert.Equal(t, contextQueryID, qResults[0].Queryid)
assert.Equal(t, repository.QuerytypeContextFull, qResults[0].Type)
assert.Equal(t, "context_value_3", *qResults[0].Value)
assert.NotEqual(t, pgtype.UUID{}, qResults[0].ID)
assert.True(t, qResults[0].ID.Valid)
}
func TestUnsyncedNoDepsQueries(t *testing.T) {
@@ -290,13 +361,11 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.NoError(t, err)
jobId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
collectorId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
_, err = queries.AddLatestCollectorVersion(ctx, collectorId)
version, err := queries.AddLatestCollectorVersion(ctx, jobId)
assert.NoError(t, err)
err = queries.SetActiveCollectorVersion(ctx, &repository.SetActiveCollectorVersionParams{
Collectorid: collectorId,
Versionid: 1,
Versionid: version,
Jobid: jobId,
})
assert.NoError(t, err)
documentID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
@@ -336,7 +405,7 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.NoError(t, err)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collectorId,
Jobid: jobId,
Name: "example_name",
Queryid: jsonQueryID,
Addedversion: 1,
@@ -345,19 +414,52 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, documentID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
textentry, err := queries.GetDocumentTextEntry(ctx, documentID)
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -368,15 +470,12 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: textentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -386,15 +485,41 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
assert.Len(t, qs, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: documentTwoID,
Version: 1,
Bucket: "hi",
Key: "hello",
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
cleantwoentry, err := queries.GetDocumentCleanEntry(ctx, documentTwoID)
assert.NoError(t, err)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: "hi",
Key: "hello",
Cleanentryid: cleantwoentry.ID,
})
assert.NoError(t, err)
texttwoentry, err := queries.GetDocumentTextEntry(ctx, documentTwoID)
assert.NoError(t, err)
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: contextQueryID,
Documentid: documentTwoID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: texttwoentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
@@ -407,12 +532,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
err = queries.SetResult(ctx, &repository.SetResultParams{
_, err = queries.AddResult(ctx, &repository.AddResultParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Textentryid: texttwoentry.ID,
Queryversion: 1,
})
assert.NoError(t, err)
internal/database/repository/text.sql.go
+19 -50
View File
@@ -12,95 +12,64 @@ import (
)
const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4)
`
type AddDocumentTextEntryParams struct {
Documentid pgtype.UUID `db:"documentid"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Cleanentryid pgtype.UUID `db:"cleanentryid"`
}
// AddDocumentTextEntry
//
// INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4)
// INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4)
func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
_, err := q.db.Exec(ctx, addDocumentTextEntry,
arg.Documentid,
arg.Version,
arg.Bucket,
arg.Key,
arg.Cleanentryid,
)
return err
}
const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
ORDER BY d.id DESC LIMIT 1
SELECT id, documentId, bucket, key, version, cleanEntryId
FROM currentTextEntries
WHERE documentId = $1
`
type GetDocumentTextEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentTextEntry
//
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) {
// SELECT id, documentId, bucket, key, version, cleanEntryId
// FROM currentTextEntries
// WHERE documentId = $1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*Currenttextentry, error) {
row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid)
var i GetDocumentTextEntryRow
var i Currenttextentry
err := row.Scan(
&i.ID,
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
&i.Cleanentryid,
)
return &i, err
}
const isDocumentTextExtracted = `-- name: IsDocumentTextExtracted :one
SELECT EXISTS(
SELECT 1
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
SELECT 1 FROM currentTextEntries WHERE documentId = $1
)
`
// IsDocumentTextExtracted
//
// SELECT EXISTS(
// SELECT 1
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
// LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
// and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
// WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
// SELECT 1 FROM currentTextEntries WHERE documentId = $1
// )
func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid)
internal/database/repository/text_test.go
+26 -17
View File
@@ -9,6 +9,7 @@ import (
"queryorchestration/internal/test"
"testing"
"github.com/jackc/pgx/v5/pgtype"
"github.com/stretchr/testify/assert"
)
@@ -21,11 +22,11 @@ func TestTextExtraction(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../../.."))
_, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
_, textup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
})
defer cleanup()
defer textup()
queries := cfg.GetDBQueries()
@@ -42,16 +43,9 @@ func TestTextExtraction(t *testing.T) {
assert.NoError(t, err)
assert.NotEmpty(t, id)
isextract, err := queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
_, err = queries.GetDocumentTextEntry(ctx, id)
assert.Error(t, err)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
Documentid: id,
Version: 1,
Bucket: bucket,
@@ -59,16 +53,31 @@ func TestTextExtraction(t *testing.T) {
})
assert.NoError(t, err)
cleanentry, err := queries.GetDocumentCleanEntry(ctx, id)
assert.NoError(t, err)
isextract, err := queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Version: 1,
Bucket: bucket,
Key: key,
Cleanentryid: cleanentry.ID,
})
assert.NoError(t, err)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.True(t, isextract)
clean, err := queries.GetDocumentTextEntry(ctx, id)
text, err := queries.GetDocumentTextEntry(ctx, id)
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetDocumentTextEntryRow{
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
assert.Equal(t, id, text.Documentid)
assert.Equal(t, bucket, text.Bucket)
assert.Equal(t, key, text.Key)
assert.Equal(t, int32(1), text.Version)
assert.NotEqual(t, pgtype.UUID{}, text.ID)
assert.True(t, text.ID.Valid)
}