Merged in feature/jobsynced (pull request #72)
Job Status Get and DB tidy up * initalquery * tests * shorttests * testing queries * job * solvedthequery * updatingdb * fixingtests * repotests * shorttests * docker * testspassed
This commit is contained in:
Michael McGuinness
@@ -1,25 +1,12 @@
|
||||
-- name: IsDocumentClean :one
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
|
||||
);
|
||||
|
||||
-- name: AddDocumentCleanEntry :exec
|
||||
INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentCleanEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
SELECT id, documentId, bucket, key, version
|
||||
FROM currentCleanEntries
|
||||
WHERE documentId = $1;
|
||||
@@ -2,15 +2,9 @@
|
||||
INSERT INTO clients (name) VALUES ($1) RETURNING id;
|
||||
|
||||
-- name: GetClient :one
|
||||
SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
|
||||
SELECT c.id, c.name, cs.canSync
|
||||
FROM clients as c
|
||||
LEFT JOIN (
|
||||
SELECT clientId, canSync
|
||||
FROM clientCanSync
|
||||
WHERE clientId = $1
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
) as cs on cs.clientId = c.id
|
||||
JOIN currentClientCanSync as cs on cs.clientId = c.id
|
||||
WHERE c.id = $1;
|
||||
|
||||
-- name: UpdateClient :exec
|
||||
|
||||
@@ -1,29 +1,26 @@
|
||||
-- name: ListCollectorQueries :many
|
||||
SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1;
|
||||
SELECT * FROM collectorQueryDependencyTree WHERE jobId = @jobId;
|
||||
|
||||
-- name: GetCollectorByJobID :one
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1;
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
|
||||
|
||||
-- name: GetCollector :one
|
||||
SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1;
|
||||
|
||||
-- name: CreateCollector :one
|
||||
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id;
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
|
||||
|
||||
-- name: AddLatestCollectorVersion :one
|
||||
INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id;
|
||||
INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id;
|
||||
|
||||
-- name: SetActiveCollectorVersion :exec
|
||||
INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2);
|
||||
INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2);
|
||||
|
||||
-- name: SetCollectorCleanVersion :exec
|
||||
INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
-- name: SetCollectorTextVersion :exec
|
||||
INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
-- name: AddCollectorQuery :exec
|
||||
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
|
||||
INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: RemoveCollectorQuery :exec
|
||||
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null;
|
||||
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null;
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
-- name: GetJob :one
|
||||
SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
|
||||
SELECT j.id, j.clientId, cs.canSync
|
||||
FROM jobs as j
|
||||
LEFT JOIN (
|
||||
SELECT jobId, canSync
|
||||
FROM jobCanSync
|
||||
WHERE jobId = $1
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
) as cs on cs.jobId = j.id
|
||||
JOIN currentJobCanSync as cs on cs.jobId = j.id
|
||||
WHERE j.id = $1;
|
||||
|
||||
-- name: CreateJob :one
|
||||
@@ -18,3 +12,79 @@ INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2);
|
||||
|
||||
-- name: ListJobDocumentIDsBatch :many
|
||||
SELECT id, totalCount FROM listJobDocumentIDs(@jobId, @batchSize, @pageOffset);
|
||||
|
||||
-- name: IsJobSynced :one
|
||||
WITH
|
||||
docs AS (
|
||||
-- Get all documents for this job
|
||||
SELECT id, jobId FROM documents WHERE jobId = $1
|
||||
),
|
||||
doc_text_entries AS (
|
||||
-- Documents with their current text entries
|
||||
SELECT
|
||||
d.id AS document_id,
|
||||
cte.id AS text_entry_id
|
||||
FROM
|
||||
docs d
|
||||
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
|
||||
),
|
||||
required_results AS (
|
||||
-- All required document-query-version combinations
|
||||
SELECT
|
||||
d.id AS document_id,
|
||||
cqdt.queryId,
|
||||
cqdt.queryVersion,
|
||||
dte.text_entry_id
|
||||
FROM
|
||||
docs d
|
||||
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
|
||||
JOIN doc_text_entries dte ON dte.document_id = d.id
|
||||
and dte.text_entry_id IS NOT NULL
|
||||
),
|
||||
existing_results AS (
|
||||
-- Valid results that exist
|
||||
SELECT
|
||||
rr.document_id AS document_id,
|
||||
r.queryId,
|
||||
r.id AS result_id
|
||||
FROM
|
||||
results r
|
||||
JOIN required_results rr ON
|
||||
r.queryId = rr.queryId AND
|
||||
r.queryVersion = rr.queryVersion AND
|
||||
r.textEntryId = rr.text_entry_id
|
||||
),
|
||||
missing_results AS (
|
||||
-- Find missing results
|
||||
SELECT rr.queryId
|
||||
FROM required_results rr
|
||||
LEFT JOIN existing_results er on er.queryId = rr.queryId
|
||||
WHERE er.result_id is null
|
||||
),
|
||||
dependency_check AS (
|
||||
-- Check for missing dependencies
|
||||
SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
|
||||
FROM existing_results er
|
||||
JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
|
||||
LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
|
||||
WHERE rd.resultId is null
|
||||
)
|
||||
SELECT (
|
||||
-- No documents means job is synced
|
||||
NOT EXISTS (SELECT 1 FROM docs)
|
||||
|
||||
OR
|
||||
|
||||
-- Documents with no text entries
|
||||
(NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
|
||||
|
||||
and
|
||||
|
||||
-- Documents with missing results
|
||||
NOT EXISTS (SELECT 1 FROM missing_results)
|
||||
|
||||
and
|
||||
|
||||
-- Documents with missing dependencies
|
||||
NOT EXISTS (SELECT 1 FROM dependency_check))
|
||||
)::bool as is_synced;
|
||||
|
||||
+22
-17
@@ -1,5 +1,5 @@
|
||||
-- name: GetQueryConfig :one
|
||||
SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2;
|
||||
-- name: GetActiveQueryConfig :one
|
||||
SELECT config FROM queryCurrentConfigs where queryId = $1;
|
||||
|
||||
-- name: GetQuery :one
|
||||
SELECT * FROM fullActiveQueries WHERE id = $1;
|
||||
@@ -16,22 +16,23 @@ SELECT c.queryId, c.config
|
||||
),
|
||||
requiredIds as (
|
||||
SELECT r.queryId,
|
||||
coalesce(
|
||||
ARRAY_AGG(DISTINCT r.requiredQueryId)
|
||||
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
|
||||
as requiredIds
|
||||
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
|
||||
array[]::uuid[]
|
||||
)::uuid[] as requiredIds
|
||||
FROM query AS q
|
||||
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
|
||||
and isInVersion(@version, r.addedVersion, r.removedVersion)
|
||||
GROUP BY r.queryId
|
||||
)
|
||||
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
|
||||
coalesce(
|
||||
r.requiredIds,
|
||||
array[]::uuid[]
|
||||
)::uuid[] as requiredIds
|
||||
SELECT DISTINCT q.id, q.type,
|
||||
av.activeVersion,
|
||||
lv.latestVersion, c.config,
|
||||
r.requiredIds
|
||||
FROM query AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
LEFT JOIN config AS c ON q.id = c.queryId
|
||||
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
|
||||
|
||||
@@ -66,17 +67,21 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist
|
||||
|
||||
-- name: IsQueryInDependencyTree :one
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
|
||||
SELECT 1 FROM queryActiveDependencies
|
||||
WHERE id = any(@requiredQueryIds)
|
||||
and requiredQueryId = @queryId
|
||||
or @queryId = any(@requiredQueryIds)
|
||||
);
|
||||
|
||||
-- name: ListQueryDirectDependentsByDocumentID :many
|
||||
WITH doc AS (
|
||||
SELECT id, jobId FROM documents where id = $2
|
||||
SELECT id, jobId FROM documents where id = @documentId
|
||||
)
|
||||
SELECT dt.queryId
|
||||
FROM collectorQueryDependencyTree as dt
|
||||
JOIN doc as d on d.jobId = dt.jobId
|
||||
where $1 = any(dt.requiredIds);
|
||||
FROM doc as d
|
||||
JOIN collectorQueryDependencyTree as dt
|
||||
on d.jobId = dt.jobId
|
||||
and @queryId = any(dt.requiredIds);
|
||||
|
||||
-- name: ListQueryJobIDs :many
|
||||
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
|
||||
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
|
||||
|
||||
+41
-39
@@ -1,9 +1,9 @@
|
||||
-- name: ListQueryRequirementValues :many
|
||||
WITH reqQueries as (
|
||||
SELECT q.id as queryId, av.id as activeVersion, q.type
|
||||
SELECT av.queryId, av.activeVersion, q.type
|
||||
FROM requiredQueries as rq
|
||||
JOIN queries as q on q.id = rq.requiredQueryId
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
|
||||
JOIN queries as q on q.id = av.queryId
|
||||
WHERE rq.queryId = @queryId
|
||||
and isInVersion(@version, rq.addedVersion, rq.removedVersion)
|
||||
),
|
||||
@@ -13,70 +13,72 @@ docs as (
|
||||
WHERE id = @documentId
|
||||
),
|
||||
codeVersions as (
|
||||
SELECT
|
||||
SELECT
|
||||
d.id as documentId,
|
||||
coalesce(ccv.versionId, 1) as minCleanVersion,
|
||||
coalesce(ctv.versionId, 1) as minTextVersion
|
||||
mcv.minCleanVersion,
|
||||
mtv.minTextVersion
|
||||
FROM docs as d
|
||||
LEFT JOIN collectors as c on c.jobId = d.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
|
||||
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
|
||||
LIMIT 1
|
||||
JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
|
||||
JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
|
||||
),
|
||||
latestVersions AS (
|
||||
SELECT
|
||||
SELECT
|
||||
r.id,
|
||||
rq.queryId,
|
||||
rq.type,
|
||||
r.queryVersion,
|
||||
r.cleanVersion,
|
||||
r.textVersion,
|
||||
r.textEntryId,
|
||||
r.value,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY rq.queryId
|
||||
ORDER BY r.cleanVersion DESC, r.textVersion DESC
|
||||
PARTITION BY r.queryId
|
||||
ORDER BY r.id DESC
|
||||
) as rowNumber
|
||||
FROM reqQueries as rq
|
||||
JOIN currentTextEntries as cte on cte.documentId = @documentId
|
||||
LEFT JOIN results as r ON rq.queryId = r.queryId
|
||||
and r.documentId = @documentId
|
||||
and r.queryVersion = rq.activeVersion
|
||||
JOIN codeVersions as ccv on ccv.documentId = r.documentId
|
||||
and r.cleanVersion >= ccv.minCleanVersion
|
||||
and r.textVersion >= ccv.minTextVersion
|
||||
and cte.id = r.textEntryId
|
||||
)
|
||||
SELECT DISTINCT lv.queryId, lv.type, r.value
|
||||
FROM latestVersions as lv
|
||||
JOIN results as r ON r.queryId = lv.queryId
|
||||
and r.documentId = @documentId
|
||||
and r.queryVersion = lv.queryVersion
|
||||
and r.cleanVersion = lv.cleanVersion
|
||||
and r.textVersion = lv.textVersion
|
||||
and lv.rowNumber = 1;
|
||||
SELECT DISTINCT id, queryId, type, value
|
||||
FROM latestVersions
|
||||
WHERE rowNumber = 1;
|
||||
|
||||
-- name: SetResult :exec
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6);
|
||||
-- name: AddResult :one
|
||||
INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id;
|
||||
|
||||
-- name: AddResultDependency :exec
|
||||
INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2);
|
||||
|
||||
-- name: GetResultValueWithVersion :one
|
||||
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;
|
||||
WITH doc as (
|
||||
SELECT id, jobId
|
||||
FROM documents
|
||||
WHERE id = @documentId
|
||||
)
|
||||
SELECT r.id, r.value
|
||||
FROM doc as d
|
||||
JOIN currentTextEntries as cte on cte.documentId = d.id
|
||||
LEFT JOIN results as r
|
||||
on r.queryId = @queryId
|
||||
and r.queryVersion = @queryVersion
|
||||
and r.textEntryId = cte.id;
|
||||
|
||||
-- name: ListUnsyncedNoDepsQueriesByDocId :many
|
||||
WITH docs as (
|
||||
SELECT id, jobId from documents where id = $1
|
||||
),
|
||||
unsyncedQueries AS (
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds
|
||||
from docs as d
|
||||
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
|
||||
JOIN fullActiveCollectors as c on c.id = dt.collectorId
|
||||
LEFT JOIN results as r on r.queryId = dt.queryId
|
||||
and r.documentId = d.id
|
||||
JOIN currentTextEntries as cte on cte.documentId = d.id
|
||||
LEFT JOIN results as r
|
||||
on r.queryId = dt.queryId
|
||||
and r.queryVersion = dt.queryVersion
|
||||
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
|
||||
and cte.id = r.textEntryId
|
||||
where r.value is null
|
||||
)
|
||||
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
|
||||
);
|
||||
|
||||
@@ -1,25 +1,12 @@
|
||||
-- name: IsDocumentTextExtracted :one
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
SELECT 1 FROM currentTextEntries WHERE documentId = @documentId
|
||||
);
|
||||
|
||||
-- name: AddDocumentTextEntry :exec
|
||||
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentTextEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
SELECT id, documentId, bucket, key, version, cleanEntryId
|
||||
FROM currentTextEntries
|
||||
WHERE documentId = @documentId;
|
||||
|
||||
Reference in New Issue
Block a user