Merged in feature/splitqueryrunning (pull request #57)
Split Query Running + Debugging Full Flow * completedquerysyncrunner * spliitinglogic * synccomplete * informdependents * only push same collector * deps * livetesting * foundissue * some issues resolved * activeupdate * collectorupdatefixes * fix dbquesries * tests * tests * pollingdebug
This commit is contained in:
Michael McGuinness
@@ -3,19 +3,19 @@ SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
JOIN collectors as c ON d.jobId = c.jobId
|
||||
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
|
||||
);
|
||||
|
||||
-- name: AddDocumentCleanEntry :exec
|
||||
INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentCleanEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
JOIN collectors as c ON d.jobId = c.jobId
|
||||
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= cv.minCleanVersion
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minCleanVersion, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
@@ -48,4 +48,13 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist
|
||||
-- name: IsQueryInDependencyTree :one
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
|
||||
);
|
||||
);
|
||||
|
||||
-- name: ListQueryDirectDependentsByDocumentID :many
|
||||
WITH doc AS (
|
||||
SELECT id, jobId FROM documents where id = $2
|
||||
)
|
||||
SELECT dt.queryId
|
||||
FROM collectorQueryDependencyTree as dt
|
||||
JOIN doc as d on d.jobId = dt.jobId
|
||||
where $1 = any(dt.requiredIds);
|
||||
+65
-37
@@ -1,49 +1,77 @@
|
||||
-- name: ListQueryRequirementValues :many
|
||||
WITH latest_versions AS (
|
||||
SELECT
|
||||
r.queryId,
|
||||
MAX(r.cleanVersion) as max_clean_version,
|
||||
MAX(r.textVersion) as max_text_version
|
||||
FROM results r
|
||||
WHERE r.documentId = $3
|
||||
AND r.queryVersion = $2
|
||||
GROUP BY r.queryId
|
||||
)
|
||||
SELECT rq.requiredQueryId as queryId, r.value, q.type
|
||||
WITH reqQueries as (
|
||||
SELECT q.id as queryId, q.activeVersion, q.type
|
||||
FROM requiredQueries as rq
|
||||
JOIN results as r on r.queryId = rq.requiredQueryId
|
||||
JOIN queries as q on q.id = rq.requiredQueryId
|
||||
JOIN latest_versions lv ON lv.queryId = r.queryId
|
||||
WHERE rq.queryId = $1 and r.documentId = $3
|
||||
and $2 >= rq.addedVersion
|
||||
and $2 < COALESCE(rq.removedVersion, $2 + 1)
|
||||
and r.queryVersion = $2
|
||||
and r.cleanVersion >= $4 and r.textVersion >= $5
|
||||
AND r.cleanVersion = lv.max_clean_version
|
||||
AND r.textVersion = lv.max_text_version;
|
||||
WHERE rq.queryId = @queryId
|
||||
and @version >= rq.addedVersion
|
||||
and (rq.removedVersion is null or @version < rq.removedVersion)
|
||||
),
|
||||
codeVersions as (
|
||||
SELECT
|
||||
d.id as documentId,
|
||||
coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
|
||||
coalesce(ccv.minTextVersion, 1) as minTextVersion
|
||||
FROM documents as d
|
||||
LEFT JOIN collectors as c on c.jobId = d.jobId
|
||||
LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
|
||||
and c.activeVersion >= ccv.addedVersion
|
||||
and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
|
||||
WHERE d.id = @documentId
|
||||
LIMIT 1
|
||||
),
|
||||
latestVersions AS (
|
||||
SELECT
|
||||
rq.queryId,
|
||||
rq.type,
|
||||
r.queryVersion,
|
||||
r.cleanVersion,
|
||||
r.textVersion,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY rq.queryId
|
||||
ORDER BY r.cleanVersion DESC, r.textVersion DESC
|
||||
) as rowNumber
|
||||
FROM reqQueries as rq
|
||||
LEFT JOIN results as r ON rq.queryId = r.queryId
|
||||
and r.documentId = @documentId
|
||||
JOIN codeVersions as ccv on ccv.documentId = r.documentId
|
||||
WHERE r.documentId = @documentId
|
||||
and r.queryVersion = rq.activeVersion
|
||||
and r.cleanVersion >= ccv.minCleanVersion
|
||||
and r.textVersion >= ccv.minTextVersion
|
||||
)
|
||||
SELECT lv.queryId, lv.type, r.value
|
||||
FROM latestVersions as lv
|
||||
JOIN results as r ON r.queryId = lv.queryId
|
||||
and r.documentId = @documentId
|
||||
and r.queryVersion = lv.queryVersion
|
||||
and r.cleanVersion = lv.cleanVersion
|
||||
and r.textVersion = lv.textVersion
|
||||
and lv.rowNumber = 1;
|
||||
|
||||
-- name: SetResult :one
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
|
||||
-- name: SetResult :exec
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6);
|
||||
|
||||
-- name: GetResultValueWithVersion :one
|
||||
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;
|
||||
|
||||
-- name: ListUnsyncedQueriesByDocId :many
|
||||
WITH RECURSIVE unsyncedQueries AS (
|
||||
SELECT dt.queryId, dt.requiredIds
|
||||
from documents as d
|
||||
JOIN fullActiveCollectors as c on d.jobId = c.jobId
|
||||
JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
|
||||
-- name: ListUnsyncedNoDepsQueriesByDocId :many
|
||||
WITH docs as (
|
||||
SELECT id, jobId from documents where id = $1
|
||||
),
|
||||
unsyncedQueries AS (
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
|
||||
from docs as d
|
||||
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
|
||||
JOIN fullActiveCollectors as c on c.id = dt.collectorId
|
||||
LEFT JOIN results as r on r.queryId = dt.queryId
|
||||
and r.documentId = d.id
|
||||
and r.queryVersion = dt.queryVersion
|
||||
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
|
||||
where d.id = $1 and r.value is null
|
||||
|
||||
UNION
|
||||
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds
|
||||
FROM unsyncedQueries as u
|
||||
JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
|
||||
where r.value is null
|
||||
)
|
||||
SELECT * FROM fullActiveQueries
|
||||
WHERE id in (SELECT queryId FROM unsyncedQueries);
|
||||
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
|
||||
);
|
||||
|
||||
@@ -3,19 +3,19 @@ SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
JOIN collectors as c ON d.jobId = c.jobId
|
||||
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
|
||||
);
|
||||
|
||||
-- name: AddDocumentTextEntry :exec
|
||||
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentTextEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
JOIN collectors as c ON d.jobId = c.jobId
|
||||
JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= cv.minTextVersion
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCodeVersions as cv ON c.id = cv.collectorId
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(cv.minTextVersion, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
Reference in New Issue
Block a user