jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/repository/clean.sql.go
+21 -15
View File
@@ -36,12 +36,12 @@ func (q *Queries) AddDocumentCleanEntry(ctx context.Context, arg *AddDocumentCle
}
const getDocumentCleanEntry = `-- name: GetDocumentCleanEntry :one
SELECT dc.documentId, dc.bucket, dc.key
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
ORDER BY d.id DESC LIMIT 1
`
@@ -49,21 +49,27 @@ type GetDocumentCleanEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentCleanEntry
//
// SELECT dc.documentId, dc.bucket, dc.key
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentCleanEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentCleanEntryRow, error) {
row := q.db.QueryRow(ctx, getDocumentCleanEntry, documentid)
var i GetDocumentCleanEntryRow
err := row.Scan(&i.Documentid, &i.Bucket, &i.Key)
err := row.Scan(
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
)
return &i, err
}
@@ -72,9 +78,9 @@ SELECT EXISTS(
SELECT 1
FROM documentCleans AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
)
`
@@ -84,9 +90,9 @@ SELECT EXISTS(
// SELECT 1
// FROM documentCleans AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
// )
func (q *Queries) IsDocumentClean(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentClean, documentid)
internal/database/repository/clean_test.go
+4 -22
View File
@@ -42,20 +42,13 @@ func TestClean(t *testing.T) {
assert.NoError(t, err)
assert.NotEmpty(t, id)
collId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{
Collectorid: collId,
Addedversion: 1,
Mincleanversion: 2,
Mintextversion: 1,
})
assert.NoError(t, err)
isclean, err := queries.IsDocumentClean(ctx, id)
assert.NoError(t, err)
assert.False(t, isclean)
_, err = queries.GetDocumentCleanEntry(ctx, id)
assert.Error(t, err)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentCleanEntry(ctx, &repository.AddDocumentCleanEntryParams{
@@ -66,18 +59,6 @@ func TestClean(t *testing.T) {
})
assert.NoError(t, err)
isclean, err = queries.IsDocumentClean(ctx, id)
assert.NoError(t, err)
assert.False(t, isclean)
err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{
Collectorid: collId,
Addedversion: 1,
Mincleanversion: 1,
Mintextversion: 1,
})
assert.NoError(t, err)
isclean, err = queries.IsDocumentClean(ctx, id)
assert.NoError(t, err)
assert.True(t, isclean)
@@ -88,5 +69,6 @@ func TestClean(t *testing.T) {
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
}
internal/database/repository/collector.sql.go
+3 -2
View File
@@ -118,12 +118,12 @@ func (q *Queries) GetCollectorByJobID(ctx context.Context, jobid pgtype.UUID) (*
}
const listCollectorQueries = `-- name: ListCollectorQueries :many
SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
`
// ListCollectorQueries
//
// SELECT collectorid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
// SELECT collectorid, jobid, queryid, type, queryversion, requiredids FROM collectorQueryDependencyTree WHERE collectorId = $1
func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.UUID) ([]*Collectorquerydependencytree, error) {
rows, err := q.db.Query(ctx, listCollectorQueries, collectorid)
if err != nil {
@@ -135,6 +135,7 @@ func (q *Queries) ListCollectorQueries(ctx context.Context, collectorid pgtype.U
var i Collectorquerydependencytree
if err := rows.Scan(
&i.Collectorid,
&i.Jobid,
&i.Queryid,
&i.Type,
&i.Queryversion,
internal/database/repository/collector_test.go
+2
View File
@@ -109,6 +109,7 @@ func TestCollector(t *testing.T) {
assert.ElementsMatch(t, []*repository.Collectorquerydependencytree{
{
Collectorid: collId,
Jobid: jobId,
Queryid: jsonId,
Queryversion: 1,
Type: repository.QuerytypeJsonExtractor,
@@ -116,6 +117,7 @@ func TestCollector(t *testing.T) {
},
{
Collectorid: collId,
Jobid: jobId,
Queryid: contextId,
Queryversion: 1,
Type: repository.QuerytypeContextFull,
internal/database/repository/document_test.go
+13
View File
@@ -41,6 +41,19 @@ func TestDocument(t *testing.T) {
})
assert.NoError(t, err)
assert.NotEmpty(t, id)
documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash_two",
})
assert.NoError(t, err)
assert.NotEmpty(t, documentTwoID)
jobTwoId, err := queries.CreateJob(ctx, clientId)
assert.NoError(t, err)
_, err = queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobTwoId,
Hash: "example_hash",
})
assert.NoError(t, err)
doc, err := queries.GetDocument(ctx, id)
assert.NoError(t, err)
internal/database/repository/models.go
+1
View File
@@ -101,6 +101,7 @@ type Collectorquery struct {
type Collectorquerydependencytree struct {
Collectorid pgtype.UUID `db:"collectorid"`
Jobid pgtype.UUID `db:"jobid"`
Queryid pgtype.UUID `db:"queryid"`
Type Querytype `db:"type"`
Queryversion int32 `db:"queryversion"`
internal/database/repository/query.sql.go
+44
View File
@@ -268,6 +268,50 @@ func (q *Queries) ListQueriesById(ctx context.Context, id []pgtype.UUID) ([]*Ful
return items, nil
}
const listQueryDirectDependentsByDocumentID = `-- name: ListQueryDirectDependentsByDocumentID :many
WITH doc AS (
SELECT id, jobId FROM documents where id = $2
)
SELECT dt.queryId
FROM collectorQueryDependencyTree as dt
JOIN doc as d on d.jobId = dt.jobId
where $1 = any(dt.requiredIds)
`
type ListQueryDirectDependentsByDocumentIDParams struct {
Requiredids pgtype.UUID `db:"requiredids"`
ID pgtype.UUID `db:"id"`
}
// ListQueryDirectDependentsByDocumentID
//
// WITH doc AS (
// SELECT id, jobId FROM documents where id = $2
// )
// SELECT dt.queryId
// FROM collectorQueryDependencyTree as dt
// JOIN doc as d on d.jobId = dt.jobId
// where $1 = any(dt.requiredIds)
func (q *Queries) ListQueryDirectDependentsByDocumentID(ctx context.Context, arg *ListQueryDirectDependentsByDocumentIDParams) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, listQueryDirectDependentsByDocumentID, arg.Requiredids, arg.ID)
if err != nil {
return nil, err
}
defer rows.Close()
items := []pgtype.UUID{}
for rows.Next() {
var queryid pgtype.UUID
if err := rows.Scan(&queryid); err != nil {
return nil, err
}
items = append(items, queryid)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const removeQueryConfig = `-- name: RemoveQueryConfig :exec
UPDATE queryConfigs SET removedVersion = $1 WHERE queryId = $2 and removedVersion is null
`
internal/database/repository/query_test.go
+121
View File
@@ -183,11 +183,38 @@ func TestQueryDependencyTree(t *testing.T) {
queries := cfg.GetDBQueries()
clientID, err := queries.CreateClient(ctx, "example_client")
assert.NoError(t, err)
jobID, err := queries.CreateJob(ctx, clientID)
assert.NoError(t, err)
collID, err := queries.CreateCollector(ctx, jobID)
assert.NoError(t, err)
docID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobID,
Hash: "sample",
})
assert.NoError(t, err)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
dependents, err := queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
jsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
@@ -195,8 +222,56 @@ func TestQueryDependencyTree(t *testing.T) {
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Name: "example_name'",
Queryid: jsonQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
secondJsonQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeJsonExtractor))
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: secondJsonQueryID,
Requiredqueryid: jsonQueryID,
@@ -204,6 +279,52 @@ func TestQueryDependencyTree(t *testing.T) {
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
err = queries.AddCollectorQuery(ctx, &repository.AddCollectorQueryParams{
Collectorid: collID,
Name: "example_name'",
Queryid: secondJsonQueryID,
Addedversion: 1,
})
assert.NoError(t, err)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: secondJsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: jsonQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{secondJsonQueryID}, dependents)
dependents, err = queries.ListQueryDirectDependentsByDocumentID(ctx, &repository.ListQueryDirectDependentsByDocumentIDParams{
ID: docID,
Requiredids: contextQueryID,
})
assert.NoError(t, err)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, dependents)
isdependent, err := queries.IsQueryInDependencyTree(ctx, &repository.IsQueryInDependencyTreeParams{
Requiredqueryid: jsonQueryID,
ID: []pgtype.UUID{contextQueryID},
internal/database/repository/result.sql.go
+142 -105
View File
@@ -45,76 +45,122 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV
}
const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
WITH latest_versions AS (
SELECT
r.queryId,
MAX(r.cleanVersion) as max_clean_version,
MAX(r.textVersion) as max_text_version
FROM results r
WHERE r.documentId = $3
AND r.queryVersion = $2
GROUP BY r.queryId
)
SELECT rq.requiredQueryId as queryId, r.value, q.type
WITH reqQueries as (
SELECT q.id as queryId, q.activeVersion, q.type
FROM requiredQueries as rq
JOIN results as r on r.queryId = rq.requiredQueryId
JOIN queries as q on q.id = rq.requiredQueryId
JOIN latest_versions lv ON lv.queryId = r.queryId
WHERE rq.queryId = $1 and r.documentId = $3
and $2 >= rq.addedVersion
and $2 < COALESCE(rq.removedVersion, $2 + 1)
and r.queryVersion = $2
and r.cleanVersion >= $4 and r.textVersion >= $5
AND r.cleanVersion = lv.max_clean_version
AND r.textVersion = lv.max_text_version
WHERE rq.queryId = $2
and $3 >= rq.addedVersion
and (rq.removedVersion is null or $3 < rq.removedVersion)
),
codeVersions as (
SELECT
d.id as documentId,
coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
coalesce(ccv.minTextVersion, 1) as minTextVersion
FROM documents as d
LEFT JOIN collectors as c on c.jobId = d.jobId
LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
and c.activeVersion >= ccv.addedVersion
and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
WHERE d.id = $1
LIMIT 1
),
latestVersions AS (
SELECT
rq.queryId,
rq.type,
r.queryVersion,
r.cleanVersion,
r.textVersion,
ROW_NUMBER() OVER (
PARTITION BY rq.queryId
ORDER BY r.cleanVersion DESC, r.textVersion DESC
) as rowNumber
FROM reqQueries as rq
LEFT JOIN results as r ON rq.queryId = r.queryId
and r.documentId = $1
JOIN codeVersions as ccv on ccv.documentId = r.documentId
WHERE r.documentId = $1
and r.queryVersion = rq.activeVersion
and r.cleanVersion >= ccv.minCleanVersion
and r.textVersion >= ccv.minTextVersion
)
SELECT lv.queryId, lv.type, r.value
FROM latestVersions as lv
JOIN results as r ON r.queryId = lv.queryId
and r.documentId = $1
and r.queryVersion = lv.queryVersion
and r.cleanVersion = lv.cleanVersion
and r.textVersion = lv.textVersion
and lv.rowNumber = 1
`
type ListQueryRequirementValuesParams struct {
Queryid pgtype.UUID `db:"queryid"`
Addedversion int32 `db:"addedversion"`
Documentid pgtype.UUID `db:"documentid"`
Cleanversion int32 `db:"cleanversion"`
Textversion int32 `db:"textversion"`
Documentid pgtype.UUID `db:"documentid"`
Queryid pgtype.UUID `db:"queryid"`
Version int32 `db:"version"`
}
type ListQueryRequirementValuesRow struct {
Queryid pgtype.UUID `db:"queryid"`
Value string `db:"value"`
Type Querytype `db:"type"`
Value string `db:"value"`
}
// ListQueryRequirementValues
//
// WITH latest_versions AS (
// SELECT
// r.queryId,
// MAX(r.cleanVersion) as max_clean_version,
// MAX(r.textVersion) as max_text_version
// FROM results r
// WHERE r.documentId = $3
// AND r.queryVersion = $2
// GROUP BY r.queryId
// )
// SELECT rq.requiredQueryId as queryId, r.value, q.type
// WITH reqQueries as (
// SELECT q.id as queryId, q.activeVersion, q.type
// FROM requiredQueries as rq
// JOIN results as r on r.queryId = rq.requiredQueryId
// JOIN queries as q on q.id = rq.requiredQueryId
// JOIN latest_versions lv ON lv.queryId = r.queryId
// WHERE rq.queryId = $1 and r.documentId = $3
// and $2 >= rq.addedVersion
// and $2 < COALESCE(rq.removedVersion, $2 + 1)
// and r.queryVersion = $2
// and r.cleanVersion >= $4 and r.textVersion >= $5
// AND r.cleanVersion = lv.max_clean_version
// AND r.textVersion = lv.max_text_version
// WHERE rq.queryId = $2
// and $3 >= rq.addedVersion
// and (rq.removedVersion is null or $3 < rq.removedVersion)
// ),
// codeVersions as (
// SELECT
// d.id as documentId,
// coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
// coalesce(ccv.minTextVersion, 1) as minTextVersion
// FROM documents as d
// LEFT JOIN collectors as c on c.jobId = d.jobId
// LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
// and c.activeVersion >= ccv.addedVersion
// and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
// WHERE d.id = $1
// LIMIT 1
// ),
// latestVersions AS (
// SELECT
// rq.queryId,
// rq.type,
// r.queryVersion,
// r.cleanVersion,
// r.textVersion,
// ROW_NUMBER() OVER (
// PARTITION BY rq.queryId
// ORDER BY r.cleanVersion DESC, r.textVersion DESC
// ) as rowNumber
// FROM reqQueries as rq
// LEFT JOIN results as r ON rq.queryId = r.queryId
// and r.documentId = $1
// JOIN codeVersions as ccv on ccv.documentId = r.documentId
// WHERE r.documentId = $1
// and r.queryVersion = rq.activeVersion
// and r.cleanVersion >= ccv.minCleanVersion
// and r.textVersion >= ccv.minTextVersion
// )
// SELECT lv.queryId, lv.type, r.value
// FROM latestVersions as lv
// JOIN results as r ON r.queryId = lv.queryId
// and r.documentId = $1
// and r.queryVersion = lv.queryVersion
// and r.cleanVersion = lv.cleanVersion
// and r.textVersion = lv.textVersion
// and lv.rowNumber = 1
func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
rows, err := q.db.Query(ctx, listQueryRequirementValues,
arg.Queryid,
arg.Addedversion,
arg.Documentid,
arg.Cleanversion,
arg.Textversion,
)
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version)
if err != nil {
return nil, err
}
@@ -122,7 +168,7 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
items := []*ListQueryRequirementValuesRow{}
for rows.Next() {
var i ListQueryRequirementValuesRow
if err := rows.Scan(&i.Queryid, &i.Value, &i.Type); err != nil {
if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil {
return nil, err
}
items = append(items, &i)
@@ -133,67 +179,60 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
return items, nil
}
const listUnsyncedQueriesByDocId = `-- name: ListUnsyncedQueriesByDocId :many
WITH RECURSIVE unsyncedQueries AS (
SELECT dt.queryId, dt.requiredIds
from documents as d
JOIN fullActiveCollectors as c on d.jobId = c.jobId
JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDocId :many
WITH docs as (
SELECT id, jobId from documents where id = $1
),
unsyncedQueries AS (
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
from docs as d
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
JOIN fullActiveCollectors as c on c.id = dt.collectorId
LEFT JOIN results as r on r.queryId = dt.queryId
and r.documentId = d.id
and r.queryVersion = dt.queryVersion
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
where d.id = $1 and r.value is null
UNION
SELECT DISTINCT dt.queryId, dt.requiredIds
FROM unsyncedQueries as u
JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
where r.value is null
)
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
WHERE id in (SELECT queryId FROM unsyncedQueries)
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
WHERE NOT EXISTS (
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
)
`
// ListUnsyncedQueriesByDocId
// ListUnsyncedNoDepsQueriesByDocId
//
// WITH RECURSIVE unsyncedQueries AS (
// SELECT dt.queryId, dt.requiredIds
// from documents as d
// JOIN fullActiveCollectors as c on d.jobId = c.jobId
// JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
// WITH docs as (
// SELECT id, jobId from documents where id = $1
// ),
// unsyncedQueries AS (
// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
// from docs as d
// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
// JOIN fullActiveCollectors as c on c.id = dt.collectorId
// LEFT JOIN results as r on r.queryId = dt.queryId
// and r.documentId = d.id
// and r.queryVersion = dt.queryVersion
// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
// where d.id = $1 and r.value is null
//
// UNION
//
// SELECT DISTINCT dt.queryId, dt.requiredIds
// FROM unsyncedQueries as u
// JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
// where r.value is null
// )
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
// WHERE id in (SELECT queryId FROM unsyncedQueries)
func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID) ([]*Fullactivequery, error) {
rows, err := q.db.Query(ctx, listUnsyncedQueriesByDocId, id)
// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
// WHERE NOT EXISTS (
// SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
// )
func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1 pgtype.UUID) ([]pgtype.UUID, error) {
rows, err := q.db.Query(ctx, listUnsyncedNoDepsQueriesByDocId, dollar_1)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*Fullactivequery{}
items := []pgtype.UUID{}
for rows.Next() {
var i Fullactivequery
if err := rows.Scan(
&i.ID,
&i.Type,
&i.Activeversion,
&i.Latestversion,
&i.Config,
&i.Requiredids,
); err != nil {
var queryid pgtype.UUID
if err := rows.Scan(&queryid); err != nil {
return nil, err
}
items = append(items, &i)
items = append(items, queryid)
}
if err := rows.Err(); err != nil {
return nil, err
@@ -201,8 +240,8 @@ func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID
return items, nil
}
const setResult = `-- name: SetResult :one
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
const setResult = `-- name: SetResult :exec
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
`
type SetResultParams struct {
@@ -216,9 +255,9 @@ type SetResultParams struct {
// SetResult
//
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.UUID, error) {
row := q.db.QueryRow(ctx, setResult,
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error {
_, err := q.db.Exec(ctx, setResult,
arg.Queryid,
arg.Documentid,
arg.Value,
@@ -226,7 +265,5 @@ func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.U
arg.Textversion,
arg.Queryversion,
)
var id pgtype.UUID
err := row.Scan(&id)
return id, err
return err
}
internal/database/repository/result_test.go
+137 -119
View File
@@ -2,6 +2,7 @@ package repository_test
import (
"context"
"log"
"os"
"path"
"queryorchestration/internal/database/repository"
@@ -50,7 +51,7 @@ func TestResults(t *testing.T) {
textVersion := int32(1)
jsonResultValue := "example_value"
jsonResultID, err := queries.SetResult(ctx, &repository.SetResultParams{
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: jsonResultValue,
@@ -59,7 +60,6 @@ func TestResults(t *testing.T) {
Queryversion: jsonQuery.Activeversion,
})
assert.NoError(t, err)
assert.True(t, jsonResultID.Valid)
res, err := queries.GetResultValueWithVersion(ctx, &repository.GetResultValueWithVersionParams{
Queryid: jsonQueryID,
@@ -69,10 +69,7 @@ func TestResults(t *testing.T) {
Textversion: textVersion,
})
assert.NoError(t, err)
assert.EqualExportedValues(t, &repository.GetResultValueWithVersionRow{
ID: jsonResultID,
Value: jsonResultValue,
}, res)
assert.EqualExportedValues(t, jsonResultValue, res.Value)
}
func TestResultValues(t *testing.T) {
@@ -104,79 +101,155 @@ func TestResultValues(t *testing.T) {
Hash: "example_hash",
})
assert.NoError(t, err)
documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash_two",
})
assert.NoError(t, err)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
jsonVersion := int32(1)
err = queries.AddRequiredQuery(ctx, &repository.AddRequiredQueryParams{
Queryid: jsonQueryID,
Requiredqueryid: contextQueryID,
Addedversion: 1,
Addedversion: jsonVersion,
})
assert.NoError(t, err)
contextQuery, err := queries.GetQuery(ctx, contextQueryID)
assert.NoError(t, err)
qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
result := repository.SetResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
Value: "context_value_1",
Cleanversion: 1,
Textversion: 2,
Queryversion: contextQuery.Activeversion,
}
_, err = queries.SetResult(ctx, &result)
err = queries.SetResult(ctx, &result)
assert.NoError(t, err)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentTwoID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 0)
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
Value: "context_value_2",
Cleanversion: 1,
Textversion: 2,
Queryversion: contextQuery.Activeversion - 1,
})
assert.NoError(t, err)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_1",
},
}, qResults)
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
Value: "context_value_3",
Cleanversion: 2,
Textversion: 2,
Queryversion: contextQuery.Activeversion,
})
assert.NoError(t, err)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "context_value",
Cleanversion: 1,
Textversion: 2,
Queryversion: contextQuery.Activeversion,
})
assert.NoError(t, err)
qResults, err := queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Addedversion: 1,
Cleanversion: result.Cleanversion,
Textversion: result.Textversion,
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
})
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: result.Value,
Value: "context_value_3",
},
}, qResults)
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "json_value_1",
Cleanversion: 1,
Textversion: 2,
Queryversion: jsonVersion,
})
assert.NoError(t, err)
qResults, err = queries.ListQueryRequirementValues(ctx, &repository.ListQueryRequirementValuesParams{
Queryid: jsonQueryID,
Documentid: documentID,
Version: jsonVersion,
})
for _, r := range qResults {
log.Print(r)
}
assert.NoError(t, err)
assert.Len(t, qResults, 1)
assert.EqualExportedValues(t, []*repository.ListQueryRequirementValuesRow{
{
Queryid: contextQueryID,
Type: repository.QuerytypeContextFull,
Value: "context_value_3",
},
}, qResults)
}
func TestUnsyncedQueries(t *testing.T) {
func TestUnsyncedNoDepsQueries(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
@@ -204,6 +277,11 @@ func TestUnsyncedQueries(t *testing.T) {
Hash: "example_hash",
})
assert.NoError(t, err)
documentTwoID, err := queries.CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: jobId,
Hash: "example_hash_two",
})
assert.NoError(t, err)
contextQueryID, err := queries.CreateQuery(ctx, repository.Querytype(repository.QuerytypeContextFull))
assert.NoError(t, err)
@@ -224,29 +302,16 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err := queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err := queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
},
{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{},
},
}, qs)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
@@ -256,19 +321,14 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
},
}, qs)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentTwoID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
err = queries.UpdateQuery(ctx, &repository.UpdateQueryParams{
Latestversion: 2,
@@ -277,29 +337,12 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 1,
Latestversion: 1,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
},
{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{},
},
}, qs)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: contextQueryID,
Documentid: documentID,
Value: "context_value",
@@ -308,7 +351,7 @@ func TestUnsyncedQueries(t *testing.T) {
Queryversion: 2,
})
assert.NoError(t, err)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "context_value",
@@ -318,7 +361,7 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
@@ -329,21 +372,12 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
},
}, qs)
assert.ElementsMatch(t, []pgtype.UUID{jsonQueryID}, qs)
_, err = queries.SetResult(ctx, &repository.SetResultParams{
err = queries.SetResult(ctx, &repository.SetResultParams{
Queryid: jsonQueryID,
Documentid: documentID,
Value: "context_value",
@@ -353,7 +387,7 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 0)
@@ -364,25 +398,9 @@ func TestUnsyncedQueries(t *testing.T) {
})
assert.NoError(t, err)
qs, err = queries.ListUnsyncedQueriesByDocId(ctx, documentID)
qs, err = queries.ListUnsyncedNoDepsQueriesByDocId(ctx, documentID)
assert.NoError(t, err)
assert.Len(t, qs, 2)
assert.ElementsMatch(t, []*repository.Fullactivequery{
{
ID: jsonQueryID,
Type: repository.QuerytypeJsonExtractor,
Activeversion: 2,
Latestversion: 2,
Config: nil,
Requiredids: []pgtype.UUID{contextQueryID},
},
{
ID: contextQueryID,
Type: repository.QuerytypeContextFull,
Activeversion: 3,
Latestversion: 3,
Config: nil,
Requiredids: []pgtype.UUID{},
},
}, qs)
assert.Len(t, qs, 1)
assert.ElementsMatch(t, []pgtype.UUID{contextQueryID}, qs)
}
internal/database/repository/text.sql.go
+21 -15
View File
@@ -36,12 +36,12 @@ func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentText
}
const getDocumentTextEntry = `-- name: GetDocumentTextEntry :one
SELECT dc.documentId, dc.bucket, dc.key
SELECT dc.documentId, dc.bucket, dc.key, dc.version
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
ORDER BY d.id DESC LIMIT 1
`
@@ -49,21 +49,27 @@ type GetDocumentTextEntryRow struct {
Documentid pgtype.UUID `db:"documentid"`
Bucket string `db:"bucket"`
Key string `db:"key"`
Version int32 `db:"version"`
}
// GetDocumentTextEntry
//
// SELECT dc.documentId, dc.bucket, dc.key
// SELECT dc.documentId, dc.bucket, dc.key, dc.version
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
// ORDER BY d.id DESC LIMIT 1
func (q *Queries) GetDocumentTextEntry(ctx context.Context, documentid pgtype.UUID) (*GetDocumentTextEntryRow, error) {
row := q.db.QueryRow(ctx, getDocumentTextEntry, documentid)
var i GetDocumentTextEntryRow
err := row.Scan(&i.Documentid, &i.Bucket, &i.Key)
err := row.Scan(
&i.Documentid,
&i.Bucket,
&i.Key,
&i.Version,
)
return &i, err
}
@@ -72,9 +78,9 @@ SELECT EXISTS(
SELECT 1
FROM documentTextExtractions AS dc
JOIN documents AS d ON d.id = dc.documentId
JOIN collectors as c ON d.jobId = c.jobId
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
LEFT JOIN collectors as c ON d.jobId = c.jobId
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
)
`
@@ -84,9 +90,9 @@ SELECT EXISTS(
// SELECT 1
// FROM documentTextExtractions AS dc
// JOIN documents AS d ON d.id = dc.documentId
// JOIN collectors as c ON d.jobId = c.jobId
// JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
// LEFT JOIN collectors as c ON d.jobId = c.jobId
// LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
// WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
// )
func (q *Queries) IsDocumentTextExtracted(ctx context.Context, documentid pgtype.UUID) (bool, error) {
row := q.db.QueryRow(ctx, isDocumentTextExtracted, documentid)
internal/database/repository/text_test.go
+4 -22
View File
@@ -42,20 +42,13 @@ func TestTextExtraction(t *testing.T) {
assert.NoError(t, err)
assert.NotEmpty(t, id)
collId, err := queries.CreateCollector(ctx, jobId)
assert.NoError(t, err)
err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{
Collectorid: collId,
Addedversion: 1,
Mincleanversion: 1,
Mintextversion: 2,
})
assert.NoError(t, err)
isextract, err := queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
_, err = queries.GetDocumentTextEntry(ctx, id)
assert.Error(t, err)
bucket := "example_bucket"
key := "example_key"
err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
@@ -66,18 +59,6 @@ func TestTextExtraction(t *testing.T) {
})
assert.NoError(t, err)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.False(t, isextract)
err = queries.AddCollectorCodeVersion(ctx, &repository.AddCollectorCodeVersionParams{
Collectorid: collId,
Addedversion: 1,
Mincleanversion: 1,
Mintextversion: 1,
})
assert.NoError(t, err)
isextract, err = queries.IsDocumentTextExtracted(ctx, id)
assert.NoError(t, err)
assert.True(t, isextract)
@@ -88,5 +69,6 @@ func TestTextExtraction(t *testing.T) {
Documentid: id,
Bucket: bucket,
Key: key,
Version: 1,
}, clean)
}