Merged in feature/splitqueryrunning (pull request #57)
Split Query Running + Debugging Full Flow * completedquerysyncrunner * spliitinglogic * synccomplete * informdependents * only push same collector * deps * livetesting * foundissue * some issues resolved * activeupdate * collectorupdatefixes * fix dbquesries * tests * tests * pollingdebug
This commit is contained in:
Michael McGuinness
@@ -45,76 +45,122 @@ func (q *Queries) GetResultValueWithVersion(ctx context.Context, arg *GetResultV
|
||||
}
|
||||
|
||||
const listQueryRequirementValues = `-- name: ListQueryRequirementValues :many
|
||||
WITH latest_versions AS (
|
||||
SELECT
|
||||
r.queryId,
|
||||
MAX(r.cleanVersion) as max_clean_version,
|
||||
MAX(r.textVersion) as max_text_version
|
||||
FROM results r
|
||||
WHERE r.documentId = $3
|
||||
AND r.queryVersion = $2
|
||||
GROUP BY r.queryId
|
||||
)
|
||||
SELECT rq.requiredQueryId as queryId, r.value, q.type
|
||||
WITH reqQueries as (
|
||||
SELECT q.id as queryId, q.activeVersion, q.type
|
||||
FROM requiredQueries as rq
|
||||
JOIN results as r on r.queryId = rq.requiredQueryId
|
||||
JOIN queries as q on q.id = rq.requiredQueryId
|
||||
JOIN latest_versions lv ON lv.queryId = r.queryId
|
||||
WHERE rq.queryId = $1 and r.documentId = $3
|
||||
and $2 >= rq.addedVersion
|
||||
and $2 < COALESCE(rq.removedVersion, $2 + 1)
|
||||
and r.queryVersion = $2
|
||||
and r.cleanVersion >= $4 and r.textVersion >= $5
|
||||
AND r.cleanVersion = lv.max_clean_version
|
||||
AND r.textVersion = lv.max_text_version
|
||||
WHERE rq.queryId = $2
|
||||
and $3 >= rq.addedVersion
|
||||
and (rq.removedVersion is null or $3 < rq.removedVersion)
|
||||
),
|
||||
codeVersions as (
|
||||
SELECT
|
||||
d.id as documentId,
|
||||
coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
|
||||
coalesce(ccv.minTextVersion, 1) as minTextVersion
|
||||
FROM documents as d
|
||||
LEFT JOIN collectors as c on c.jobId = d.jobId
|
||||
LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
|
||||
and c.activeVersion >= ccv.addedVersion
|
||||
and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
|
||||
WHERE d.id = $1
|
||||
LIMIT 1
|
||||
),
|
||||
latestVersions AS (
|
||||
SELECT
|
||||
rq.queryId,
|
||||
rq.type,
|
||||
r.queryVersion,
|
||||
r.cleanVersion,
|
||||
r.textVersion,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY rq.queryId
|
||||
ORDER BY r.cleanVersion DESC, r.textVersion DESC
|
||||
) as rowNumber
|
||||
FROM reqQueries as rq
|
||||
LEFT JOIN results as r ON rq.queryId = r.queryId
|
||||
and r.documentId = $1
|
||||
JOIN codeVersions as ccv on ccv.documentId = r.documentId
|
||||
WHERE r.documentId = $1
|
||||
and r.queryVersion = rq.activeVersion
|
||||
and r.cleanVersion >= ccv.minCleanVersion
|
||||
and r.textVersion >= ccv.minTextVersion
|
||||
)
|
||||
SELECT lv.queryId, lv.type, r.value
|
||||
FROM latestVersions as lv
|
||||
JOIN results as r ON r.queryId = lv.queryId
|
||||
and r.documentId = $1
|
||||
and r.queryVersion = lv.queryVersion
|
||||
and r.cleanVersion = lv.cleanVersion
|
||||
and r.textVersion = lv.textVersion
|
||||
and lv.rowNumber = 1
|
||||
`
|
||||
|
||||
type ListQueryRequirementValuesParams struct {
|
||||
Queryid pgtype.UUID `db:"queryid"`
|
||||
Addedversion int32 `db:"addedversion"`
|
||||
Documentid pgtype.UUID `db:"documentid"`
|
||||
Cleanversion int32 `db:"cleanversion"`
|
||||
Textversion int32 `db:"textversion"`
|
||||
Documentid pgtype.UUID `db:"documentid"`
|
||||
Queryid pgtype.UUID `db:"queryid"`
|
||||
Version int32 `db:"version"`
|
||||
}
|
||||
|
||||
type ListQueryRequirementValuesRow struct {
|
||||
Queryid pgtype.UUID `db:"queryid"`
|
||||
Value string `db:"value"`
|
||||
Type Querytype `db:"type"`
|
||||
Value string `db:"value"`
|
||||
}
|
||||
|
||||
// ListQueryRequirementValues
|
||||
//
|
||||
// WITH latest_versions AS (
|
||||
// SELECT
|
||||
// r.queryId,
|
||||
// MAX(r.cleanVersion) as max_clean_version,
|
||||
// MAX(r.textVersion) as max_text_version
|
||||
// FROM results r
|
||||
// WHERE r.documentId = $3
|
||||
// AND r.queryVersion = $2
|
||||
// GROUP BY r.queryId
|
||||
// )
|
||||
// SELECT rq.requiredQueryId as queryId, r.value, q.type
|
||||
// WITH reqQueries as (
|
||||
// SELECT q.id as queryId, q.activeVersion, q.type
|
||||
// FROM requiredQueries as rq
|
||||
// JOIN results as r on r.queryId = rq.requiredQueryId
|
||||
// JOIN queries as q on q.id = rq.requiredQueryId
|
||||
// JOIN latest_versions lv ON lv.queryId = r.queryId
|
||||
// WHERE rq.queryId = $1 and r.documentId = $3
|
||||
// and $2 >= rq.addedVersion
|
||||
// and $2 < COALESCE(rq.removedVersion, $2 + 1)
|
||||
// and r.queryVersion = $2
|
||||
// and r.cleanVersion >= $4 and r.textVersion >= $5
|
||||
// AND r.cleanVersion = lv.max_clean_version
|
||||
// AND r.textVersion = lv.max_text_version
|
||||
// WHERE rq.queryId = $2
|
||||
// and $3 >= rq.addedVersion
|
||||
// and (rq.removedVersion is null or $3 < rq.removedVersion)
|
||||
// ),
|
||||
// codeVersions as (
|
||||
// SELECT
|
||||
// d.id as documentId,
|
||||
// coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
|
||||
// coalesce(ccv.minTextVersion, 1) as minTextVersion
|
||||
// FROM documents as d
|
||||
// LEFT JOIN collectors as c on c.jobId = d.jobId
|
||||
// LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
|
||||
// and c.activeVersion >= ccv.addedVersion
|
||||
// and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
|
||||
// WHERE d.id = $1
|
||||
// LIMIT 1
|
||||
// ),
|
||||
// latestVersions AS (
|
||||
// SELECT
|
||||
// rq.queryId,
|
||||
// rq.type,
|
||||
// r.queryVersion,
|
||||
// r.cleanVersion,
|
||||
// r.textVersion,
|
||||
// ROW_NUMBER() OVER (
|
||||
// PARTITION BY rq.queryId
|
||||
// ORDER BY r.cleanVersion DESC, r.textVersion DESC
|
||||
// ) as rowNumber
|
||||
// FROM reqQueries as rq
|
||||
// LEFT JOIN results as r ON rq.queryId = r.queryId
|
||||
// and r.documentId = $1
|
||||
// JOIN codeVersions as ccv on ccv.documentId = r.documentId
|
||||
// WHERE r.documentId = $1
|
||||
// and r.queryVersion = rq.activeVersion
|
||||
// and r.cleanVersion >= ccv.minCleanVersion
|
||||
// and r.textVersion >= ccv.minTextVersion
|
||||
// )
|
||||
// SELECT lv.queryId, lv.type, r.value
|
||||
// FROM latestVersions as lv
|
||||
// JOIN results as r ON r.queryId = lv.queryId
|
||||
// and r.documentId = $1
|
||||
// and r.queryVersion = lv.queryVersion
|
||||
// and r.cleanVersion = lv.cleanVersion
|
||||
// and r.textVersion = lv.textVersion
|
||||
// and lv.rowNumber = 1
|
||||
func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQueryRequirementValuesParams) ([]*ListQueryRequirementValuesRow, error) {
|
||||
rows, err := q.db.Query(ctx, listQueryRequirementValues,
|
||||
arg.Queryid,
|
||||
arg.Addedversion,
|
||||
arg.Documentid,
|
||||
arg.Cleanversion,
|
||||
arg.Textversion,
|
||||
)
|
||||
rows, err := q.db.Query(ctx, listQueryRequirementValues, arg.Documentid, arg.Queryid, arg.Version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -122,7 +168,7 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
|
||||
items := []*ListQueryRequirementValuesRow{}
|
||||
for rows.Next() {
|
||||
var i ListQueryRequirementValuesRow
|
||||
if err := rows.Scan(&i.Queryid, &i.Value, &i.Type); err != nil {
|
||||
if err := rows.Scan(&i.Queryid, &i.Type, &i.Value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, &i)
|
||||
@@ -133,67 +179,60 @@ func (q *Queries) ListQueryRequirementValues(ctx context.Context, arg *ListQuery
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listUnsyncedQueriesByDocId = `-- name: ListUnsyncedQueriesByDocId :many
|
||||
WITH RECURSIVE unsyncedQueries AS (
|
||||
SELECT dt.queryId, dt.requiredIds
|
||||
from documents as d
|
||||
JOIN fullActiveCollectors as c on d.jobId = c.jobId
|
||||
JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
|
||||
const listUnsyncedNoDepsQueriesByDocId = `-- name: ListUnsyncedNoDepsQueriesByDocId :many
|
||||
WITH docs as (
|
||||
SELECT id, jobId from documents where id = $1
|
||||
),
|
||||
unsyncedQueries AS (
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
|
||||
from docs as d
|
||||
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
|
||||
JOIN fullActiveCollectors as c on c.id = dt.collectorId
|
||||
LEFT JOIN results as r on r.queryId = dt.queryId
|
||||
and r.documentId = d.id
|
||||
and r.queryVersion = dt.queryVersion
|
||||
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
|
||||
where d.id = $1 and r.value is null
|
||||
|
||||
UNION
|
||||
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds
|
||||
FROM unsyncedQueries as u
|
||||
JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
|
||||
where r.value is null
|
||||
)
|
||||
SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
|
||||
WHERE id in (SELECT queryId FROM unsyncedQueries)
|
||||
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
|
||||
)
|
||||
`
|
||||
|
||||
// ListUnsyncedQueriesByDocId
|
||||
// ListUnsyncedNoDepsQueriesByDocId
|
||||
//
|
||||
// WITH RECURSIVE unsyncedQueries AS (
|
||||
// SELECT dt.queryId, dt.requiredIds
|
||||
// from documents as d
|
||||
// JOIN fullActiveCollectors as c on d.jobId = c.jobId
|
||||
// JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
|
||||
// WITH docs as (
|
||||
// SELECT id, jobId from documents where id = $1
|
||||
// ),
|
||||
// unsyncedQueries AS (
|
||||
// SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
|
||||
// from docs as d
|
||||
// JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
|
||||
// JOIN fullActiveCollectors as c on c.id = dt.collectorId
|
||||
// LEFT JOIN results as r on r.queryId = dt.queryId
|
||||
// and r.documentId = d.id
|
||||
// and r.queryVersion = dt.queryVersion
|
||||
// and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
|
||||
// where d.id = $1 and r.value is null
|
||||
//
|
||||
// UNION
|
||||
//
|
||||
// SELECT DISTINCT dt.queryId, dt.requiredIds
|
||||
// FROM unsyncedQueries as u
|
||||
// JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
|
||||
// where r.value is null
|
||||
// )
|
||||
// SELECT id, type, activeversion, latestversion, config, requiredids FROM fullActiveQueries
|
||||
// WHERE id in (SELECT queryId FROM unsyncedQueries)
|
||||
func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID) ([]*Fullactivequery, error) {
|
||||
rows, err := q.db.Query(ctx, listUnsyncedQueriesByDocId, id)
|
||||
// SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
|
||||
// WHERE NOT EXISTS (
|
||||
// SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
|
||||
// )
|
||||
func (q *Queries) ListUnsyncedNoDepsQueriesByDocId(ctx context.Context, dollar_1 pgtype.UUID) ([]pgtype.UUID, error) {
|
||||
rows, err := q.db.Query(ctx, listUnsyncedNoDepsQueriesByDocId, dollar_1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []*Fullactivequery{}
|
||||
items := []pgtype.UUID{}
|
||||
for rows.Next() {
|
||||
var i Fullactivequery
|
||||
if err := rows.Scan(
|
||||
&i.ID,
|
||||
&i.Type,
|
||||
&i.Activeversion,
|
||||
&i.Latestversion,
|
||||
&i.Config,
|
||||
&i.Requiredids,
|
||||
); err != nil {
|
||||
var queryid pgtype.UUID
|
||||
if err := rows.Scan(&queryid); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, &i)
|
||||
items = append(items, queryid)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
@@ -201,8 +240,8 @@ func (q *Queries) ListUnsyncedQueriesByDocId(ctx context.Context, id pgtype.UUID
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const setResult = `-- name: SetResult :one
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
|
||||
const setResult = `-- name: SetResult :exec
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
`
|
||||
|
||||
type SetResultParams struct {
|
||||
@@ -216,9 +255,9 @@ type SetResultParams struct {
|
||||
|
||||
// SetResult
|
||||
//
|
||||
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id
|
||||
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.UUID, error) {
|
||||
row := q.db.QueryRow(ctx, setResult,
|
||||
// INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6)
|
||||
func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) error {
|
||||
_, err := q.db.Exec(ctx, setResult,
|
||||
arg.Queryid,
|
||||
arg.Documentid,
|
||||
arg.Value,
|
||||
@@ -226,7 +265,5 @@ func (q *Queries) SetResult(ctx context.Context, arg *SetResultParams) (pgtype.U
|
||||
arg.Textversion,
|
||||
arg.Queryversion,
|
||||
)
|
||||
var id pgtype.UUID
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user