Merged in feature/jobsynced (pull request #72)
Job Status Get and DB tidy up * initalquery * tests * shorttests * testing queries * job * solvedthequery * updatingdb * fixingtests * repotests * shorttests * docker * testspassed
This commit is contained in:
Michael McGuinness
@@ -1,16 +1,9 @@
|
||||
CREATE TABLE collectors (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
jobId uuid not null,
|
||||
foreign key (jobId) references jobs(id),
|
||||
unique (jobId)
|
||||
);
|
||||
|
||||
CREATE TABLE collectorVersions (
|
||||
collectorId uuid not null,
|
||||
jobId uuid not null,
|
||||
id int not null,
|
||||
addedAt timestamp not null default current_timestamp,
|
||||
primary key (id, collectorId),
|
||||
foreign key (collectorId) references collectors(id)
|
||||
foreign key (jobId) references jobs(id),
|
||||
primary key (id, jobId)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
|
||||
@@ -19,8 +12,8 @@ BEGIN
|
||||
SELECT COALESCE(MAX(id), 0) + 1
|
||||
INTO NEW.id
|
||||
FROM collectorVersions
|
||||
WHERE collectorId = NEW.collectorId;
|
||||
|
||||
WHERE jobId = NEW.jobId;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -32,20 +25,20 @@ EXECUTE FUNCTION setCollectorVersionNumber();
|
||||
|
||||
CREATE TABLE collectorActiveVersions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
collectorId uuid not null,
|
||||
jobId uuid not null,
|
||||
versionId int not null,
|
||||
foreign key (collectorId, versionId) references collectorVersions(collectorId, id)
|
||||
foreign key (jobId, versionId) references collectorVersions(jobId, id)
|
||||
);
|
||||
|
||||
CREATE TABLE collectorMinCleanVersions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
collectorId uuid not null,
|
||||
jobId uuid not null,
|
||||
versionId int not null,
|
||||
addedVersion int not null,
|
||||
removedVersion int,
|
||||
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
|
||||
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
|
||||
foreign key (collectorId) references collectors(id)
|
||||
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
|
||||
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
|
||||
foreign key (jobId) references jobs(id)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
|
||||
@@ -53,8 +46,8 @@ RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE collectorMinCleanVersions
|
||||
SET removedVersion = NEW.addedVersion
|
||||
WHERE collectorId = NEW.collectorId and removedVersion is null;
|
||||
|
||||
WHERE jobId = NEW.jobId and removedVersion is null;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -66,13 +59,13 @@ EXECUTE FUNCTION removeMinCleanVersion();
|
||||
|
||||
CREATE TABLE collectorMinTextVersions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
collectorId uuid not null,
|
||||
jobId uuid not null,
|
||||
versionId int not null,
|
||||
addedVersion int not null,
|
||||
removedVersion int,
|
||||
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
|
||||
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
|
||||
foreign key (collectorId) references collectors(id)
|
||||
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
|
||||
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
|
||||
foreign key (jobId) references jobs(id)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION removeMinTextVersion()
|
||||
@@ -80,8 +73,8 @@ RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE collectorMinTextVersions
|
||||
SET removedVersion = NEW.addedVersion
|
||||
WHERE collectorId = NEW.collectorId and removedVersion is null;
|
||||
|
||||
WHERE jobId = NEW.jobId and removedVersion is null;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -93,14 +86,14 @@ EXECUTE FUNCTION removeMinTextVersion();
|
||||
|
||||
CREATE TABLE collectorQueries (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
collectorId uuid not null,
|
||||
jobId uuid not null,
|
||||
name varchar(255) not null,
|
||||
queryId uuid not null,
|
||||
addedVersion int not null,
|
||||
removedVersion int,
|
||||
foreign key (queryId) references queries(id),
|
||||
foreign key (collectorId) references collectors(id),
|
||||
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
|
||||
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
|
||||
unique (collectorId, name, removedVersion)
|
||||
);
|
||||
foreign key (jobId) references jobs(id),
|
||||
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
|
||||
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
|
||||
unique (jobId, name, removedVersion)
|
||||
);
|
||||
|
||||
@@ -25,9 +25,9 @@ CREATE TABLE documentCleans (
|
||||
|
||||
CREATE TABLE documentTextExtractions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
documentId uuid not null,
|
||||
cleanEntryId uuid not null,
|
||||
version int not null,
|
||||
bucket text not null,
|
||||
key text not null,
|
||||
foreign key (documentId) references documents(id)
|
||||
foreign key (cleanEntryId) references documentCleans(id)
|
||||
);
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
CREATE TABLE results (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
textEntryId uuid not null,
|
||||
queryId uuid not null,
|
||||
documentId uuid not null,
|
||||
value TEXT not null,
|
||||
cleanVersion int not null,
|
||||
textVersion int not null,
|
||||
queryVersion int not null,
|
||||
foreign key (queryId) references queries(id),
|
||||
foreign key (documentId) references documents(id),
|
||||
unique (queryId, documentId, cleanVersion, textVersion, queryVersion),
|
||||
foreign key (textEntryId) references documentTextExtractions(id),
|
||||
foreign key (queryId, queryVersion) references queryVersions(queryId, id)
|
||||
);
|
||||
);
|
||||
|
||||
CREATE TABLE resultDependencies (
|
||||
resultId uuid not null,
|
||||
requiredResultId uuid not null,
|
||||
foreign key (resultId) references results(id),
|
||||
foreign key (requiredResultId) references results(id),
|
||||
CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
|
||||
);
|
||||
|
||||
@@ -1,59 +1,71 @@
|
||||
CREATE VIEW queryCurrentActiveVersions as
|
||||
SELECT DISTINCT
|
||||
av.queryId,
|
||||
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.queryId ORDER BY av.id DESC))::int as id
|
||||
FROM queryActiveVersions as av;
|
||||
q.id as queryId,
|
||||
coalesce(
|
||||
(FIRST_VALUE(av.versionId) OVER (PARTITION BY q.id ORDER BY av.id DESC)),
|
||||
0
|
||||
)::int as activeVersion
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryActiveVersions as av on av.queryId = q.id;
|
||||
|
||||
CREATE VIEW queryLatestVersions as
|
||||
SELECT queryId, max(id)::int as id
|
||||
FROM queryVersions
|
||||
GROUP BY queryId;
|
||||
SELECT
|
||||
q.id as queryId,
|
||||
coalesce(max(v.id), 0)::int as latestVersion
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryVersions as v on v.queryId = q.id
|
||||
GROUP BY q.id;
|
||||
|
||||
CREATE VIEW fullActiveQueries AS
|
||||
with config as (
|
||||
SELECT c.queryId, c.config
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN queryConfigs AS c ON q.id = c.queryId
|
||||
and isInVersion(av.id, c.addedVersion, c.removedVersion)
|
||||
),
|
||||
requiredIds as (
|
||||
SELECT r.queryId,
|
||||
ARRAY_AGG(DISTINCT r.requiredQueryId)
|
||||
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
|
||||
as requiredIds
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
|
||||
and isInVersion(av.id, r.addedVersion, r.removedVersion)
|
||||
GROUP BY r.queryId
|
||||
)
|
||||
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
|
||||
CREATE VIEW queryCurrentConfigs as
|
||||
SELECT av.queryId, c.config
|
||||
FROM queryCurrentActiveVersions as av
|
||||
LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
|
||||
and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
|
||||
|
||||
CREATE VIEW queryCurrentRequiredIds as
|
||||
SELECT DISTINCT av.queryId, r.requiredQueryId
|
||||
FROM queryCurrentActiveVersions as av
|
||||
LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
|
||||
and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
|
||||
|
||||
CREATE VIEW queryCurrentRequiredIdsAGG as
|
||||
SELECT queryId,
|
||||
coalesce(
|
||||
r.requiredIds,
|
||||
ARRAY_AGG(DISTINCT requiredQueryId)
|
||||
FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
|
||||
array[]::uuid[]
|
||||
)::uuid[] as requiredIds
|
||||
FROM queryCurrentRequiredIds
|
||||
GROUP BY queryId;
|
||||
|
||||
CREATE VIEW fullActiveQueries AS
|
||||
SELECT DISTINCT q.id, q.type, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
LEFT JOIN config AS c ON q.id = c.queryId
|
||||
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
|
||||
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
JOIN queryCurrentConfigs AS c ON q.id = c.queryId
|
||||
JOIN queryCurrentRequiredIdsAGG AS r ON q.id = r.queryId;
|
||||
|
||||
CREATE VIEW queryActiveDependencies AS
|
||||
with recursive queryActiveDependencies AS (
|
||||
SELECT q.id, r.requiredQueryId
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
|
||||
and isInVersion(av.id, r.addedVersion, r.removedVersion)
|
||||
WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
|
||||
SELECT
|
||||
queryId,
|
||||
requiredQueryId,
|
||||
ARRAY[queryId, requiredQueryId]::uuid[] AS path,
|
||||
false AS cycle
|
||||
FROM queryCurrentRequiredIds
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT qd.id, r.requiredQueryId
|
||||
FROM queries AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
|
||||
and isInVersion(av.id, r.addedVersion, r.removedVersion)
|
||||
JOIN queryActiveDependencies as qd on q.id = qd.requiredQueryId
|
||||
SELECT
|
||||
q.queryId,
|
||||
qd.requiredQueryId,
|
||||
path || qd.requiredQueryId,
|
||||
qd.requiredQueryId = ANY(path) AS cycle
|
||||
FROM queryCurrentRequiredIds as q
|
||||
JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
|
||||
WHERE NOT qd.cycle -- Stop if we detect a cycle
|
||||
)
|
||||
SELECT DISTINCT id, requiredQueryId FROM queryActiveDependencies;
|
||||
SELECT DISTINCT queryId as id, requiredQueryId
|
||||
FROM queryActiveDependencies
|
||||
WHERE NOT cycle;
|
||||
|
||||
@@ -1,47 +1,77 @@
|
||||
CREATE VIEW collectorCurrentActiveVersions as
|
||||
SELECT DISTINCT
|
||||
av.collectorId,
|
||||
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.collectorId ORDER BY av.id DESC))::int as id
|
||||
FROM collectorActiveVersions as av;
|
||||
j.id as jobId,
|
||||
coalesce(
|
||||
(FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)),
|
||||
0
|
||||
)::int as activeVersion
|
||||
FROM jobs as j
|
||||
LEFT JOIN collectorActiveVersions as v on v.jobId = j.id;
|
||||
|
||||
CREATE VIEW collectorLatestVersions as
|
||||
SELECT collectorId, max(id)::int as id
|
||||
FROM collectorVersions
|
||||
GROUP BY collectorId;
|
||||
SELECT
|
||||
j.id as jobId,
|
||||
coalesce(max(v.id), 0)::int as latestVersion
|
||||
FROM jobs as j
|
||||
LEFT JOIN collectorVersions as v on v.jobId = j.id
|
||||
GROUP BY j.id;
|
||||
|
||||
CREATE VIEW currentCollectorMinTextVersions as
|
||||
SELECT DISTINCT
|
||||
av.jobId,
|
||||
coalesce(ctv.versionId, 0) as minTextVersion
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
LEFT JOIN collectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
|
||||
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
|
||||
|
||||
CREATE VIEW currentCollectorMinCleanVersions as
|
||||
SELECT DISTINCT
|
||||
av.jobId,
|
||||
coalesce(ctv.versionId, 0) as minCleanVersion
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
LEFT JOIN collectorMinCleanVersions AS ctv ON av.jobId = ctv.jobId
|
||||
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
|
||||
|
||||
CREATE VIEW currentCollectorQueries as
|
||||
SELECT DISTINCT
|
||||
av.jobId,
|
||||
q.name,
|
||||
q.queryId
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
LEFT JOIN collectorQueries AS q ON av.jobId = q.jobId
|
||||
and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
|
||||
|
||||
CREATE VIEW currentCollectorQueriesJSONAGG as
|
||||
SELECT DISTINCT
|
||||
jobId,
|
||||
jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
|
||||
FROM currentCollectorQueries
|
||||
GROUP BY jobId;
|
||||
|
||||
CREATE VIEW fullActiveCollectors AS
|
||||
SELECT DISTINCT c.id, c.jobId, coalesce(ccv.versionId, 0) as minCleanVersion, coalesce(ctv.versionId, 0) as minTextVersion, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion,
|
||||
jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
|
||||
FROM collectors AS c
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorLatestVersions as lv on lv.collectorId = c.id
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
|
||||
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
|
||||
LEFT JOIN collectorQueries AS q ON c.id = q.collectorId
|
||||
and isInVersion(av.id, q.addedVersion, q.removedVersion)
|
||||
GROUP BY c.id, c.jobId, ccv.versionId, ctv.versionId, av.id, lv.id;
|
||||
SELECT DISTINCT av.jobId,
|
||||
ccv.minCleanVersion, ctv.minTextVersion,
|
||||
av.activeVersion, lv.latestVersion,
|
||||
q.fields
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
JOIN collectorLatestVersions as lv on lv.jobId = av.jobId
|
||||
JOIN currentCollectorMinCleanVersions AS ccv ON av.jobId = ccv.jobId
|
||||
JOIN currentCollectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
|
||||
JOIN currentCollectorQueriesJSONAGG AS q ON av.jobId = q.jobId;
|
||||
|
||||
CREATE VIEW collectorQueryDependencyTree AS
|
||||
WITH RECURSIVE
|
||||
coll as (
|
||||
SELECT DISTINCT c.id, c.jobId, q.queryId
|
||||
FROM collectors as c
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorQueries as q ON c.id = q.collectorId
|
||||
and isInVersion(av.id, q.addedVersion, q.removedVersion)
|
||||
),
|
||||
collectorQueryDependencyTree AS (
|
||||
SELECT cq.id as collectorId, cq.jobId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion
|
||||
FROM coll as cq
|
||||
JOIN fullActiveQueries as aqc on cq.queryId = aqc.id
|
||||
WITH RECURSIVE collectorQueryDependencyTree AS (
|
||||
SELECT cq.jobId, cq.queryId, ri.requiredIds
|
||||
FROM currentCollectorQueries as cq
|
||||
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT acq.collectorId, acq.jobId, q.id as queryId, q.type, q.requiredIds, q.activeVersion
|
||||
FROM fullActiveQueries as q
|
||||
JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
|
||||
SELECT acq.jobId, q.queryId, q.requiredIds
|
||||
FROM queryCurrentRequiredIdsAGG as q
|
||||
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
|
||||
)
|
||||
SELECT DISTINCT collectorId, jobId, queryId, type, activeVersion as queryVersion, requiredIds
|
||||
FROM collectorQueryDependencyTree;
|
||||
SELECT DISTINCT ct.jobId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds
|
||||
FROM collectorQueryDependencyTree as ct
|
||||
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
|
||||
JOIN queries as q on q.id = ct.queryId;
|
||||
|
||||
@@ -8,7 +8,7 @@ DECLARE
|
||||
totalCount BIGINT := 0;
|
||||
BEGIN
|
||||
SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE jobId = _jobId;
|
||||
|
||||
|
||||
RETURN QUERY
|
||||
SELECT d.id, totalCount
|
||||
FROM documents as d
|
||||
@@ -17,4 +17,83 @@ BEGIN
|
||||
LIMIT _batchSize
|
||||
OFFSET _offset;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE VIEW currentCleanEntries as
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
dc.id,
|
||||
dc.documentId,
|
||||
dc.bucket,
|
||||
dc.key,
|
||||
dc.version,
|
||||
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC) as row_num
|
||||
FROM documents d
|
||||
JOIN currentCollectorMinCleanVersions ccv ON d.jobId = ccv.jobId
|
||||
JOIN documentCleans dc ON dc.documentId = d.id
|
||||
AND dc.version >= ccv.minCleanVersion
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
documentId,
|
||||
bucket,
|
||||
key,
|
||||
version
|
||||
FROM RankedExtractions
|
||||
WHERE row_num = 1;
|
||||
|
||||
CREATE VIEW currentTextEntries as
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
dte.id,
|
||||
cc.documentId,
|
||||
dte.bucket,
|
||||
dte.key,
|
||||
dte.version,
|
||||
dte.cleanEntryId,
|
||||
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num
|
||||
FROM documents d
|
||||
JOIN currentCleanEntries cc on cc.documentId = d.id
|
||||
JOIN currentCollectorMinTextVersions ctv ON ctv.jobId = d.jobId
|
||||
JOIN documentTextExtractions dte ON dte.cleanEntryId = cc.id
|
||||
AND dte.version >= ctv.minTextVersion
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
documentId,
|
||||
bucket,
|
||||
key,
|
||||
version,
|
||||
cleanEntryId
|
||||
FROM RankedExtractions
|
||||
WHERE row_num = 1;
|
||||
|
||||
CREATE VIEW currentClientCanSync as
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
ccs.clientId,
|
||||
ccs.canSync,
|
||||
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.id DESC) as row_num
|
||||
FROM clients c
|
||||
JOIN clientCanSync ccs on c.id = ccs.clientId
|
||||
)
|
||||
SELECT
|
||||
c.id as clientId,
|
||||
coalesce(r.canSync, false) as canSync
|
||||
FROM clients c
|
||||
LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1;
|
||||
|
||||
CREATE VIEW currentJobCanSync as
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
ccs.jobId,
|
||||
ccs.canSync,
|
||||
ROW_NUMBER() OVER (PARTITION BY ccs.jobId ORDER BY ccs.id DESC) as row_num
|
||||
FROM jobs c
|
||||
JOIN jobCanSync ccs on c.id = ccs.jobId
|
||||
)
|
||||
SELECT
|
||||
c.id as jobId,
|
||||
coalesce(r.canSync, false) as canSync
|
||||
FROM jobs c
|
||||
LEFT JOIN RankedExtractions r on r.jobId = c.id and r.row_num = 1;
|
||||
|
||||
@@ -1,25 +1,12 @@
|
||||
-- name: IsDocumentClean :one
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
SELECT 1 FROM currentCleanEntries WHERE documentId = $1
|
||||
);
|
||||
|
||||
-- name: AddDocumentCleanEntry :exec
|
||||
INSERT INTO documentCleans (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentCleanEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentCleans AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
SELECT id, documentId, bucket, key, version
|
||||
FROM currentCleanEntries
|
||||
WHERE documentId = $1;
|
||||
@@ -2,15 +2,9 @@
|
||||
INSERT INTO clients (name) VALUES ($1) RETURNING id;
|
||||
|
||||
-- name: GetClient :one
|
||||
SELECT c.id, c.name, coalesce(cs.canSync, false) as canSync
|
||||
SELECT c.id, c.name, cs.canSync
|
||||
FROM clients as c
|
||||
LEFT JOIN (
|
||||
SELECT clientId, canSync
|
||||
FROM clientCanSync
|
||||
WHERE clientId = $1
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
) as cs on cs.clientId = c.id
|
||||
JOIN currentClientCanSync as cs on cs.clientId = c.id
|
||||
WHERE c.id = $1;
|
||||
|
||||
-- name: UpdateClient :exec
|
||||
|
||||
@@ -1,29 +1,26 @@
|
||||
-- name: ListCollectorQueries :many
|
||||
SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1;
|
||||
SELECT * FROM collectorQueryDependencyTree WHERE jobId = @jobId;
|
||||
|
||||
-- name: GetCollectorByJobID :one
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1;
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
|
||||
|
||||
-- name: GetCollector :one
|
||||
SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1;
|
||||
|
||||
-- name: CreateCollector :one
|
||||
INSERT INTO collectors (jobId) VALUES ($1) RETURNING id;
|
||||
SELECT * FROM fullActiveCollectors WHERE jobId = @jobId LIMIT 1;
|
||||
|
||||
-- name: AddLatestCollectorVersion :one
|
||||
INSERT INTO collectorVersions (collectorId) VALUES ($1) RETURNING id;
|
||||
INSERT INTO collectorVersions (jobId) VALUES ($1) RETURNING id;
|
||||
|
||||
-- name: SetActiveCollectorVersion :exec
|
||||
INSERT INTO collectorActiveVersions (collectorId, versionId) VALUES ($1, $2);
|
||||
INSERT INTO collectorActiveVersions (jobId, versionId) VALUES ($1, $2);
|
||||
|
||||
-- name: SetCollectorCleanVersion :exec
|
||||
INSERT INTO collectorMinCleanVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
INSERT INTO collectorMinCleanVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
-- name: SetCollectorTextVersion :exec
|
||||
INSERT INTO collectorMinTextVersions (collectorId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
INSERT INTO collectorMinTextVersions (jobId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
-- name: AddCollectorQuery :exec
|
||||
INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
|
||||
INSERT INTO collectorQueries (jobId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: RemoveCollectorQuery :exec
|
||||
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and collectorId = $3 and removedVersion is null;
|
||||
UPDATE collectorQueries SET removedVersion = $1 WHERE queryId = $2 and jobId = $3 and removedVersion is null;
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
-- name: GetJob :one
|
||||
SELECT j.id, j.clientId, coalesce(cs.canSync, false) as canSync
|
||||
SELECT j.id, j.clientId, cs.canSync
|
||||
FROM jobs as j
|
||||
LEFT JOIN (
|
||||
SELECT jobId, canSync
|
||||
FROM jobCanSync
|
||||
WHERE jobId = $1
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
) as cs on cs.jobId = j.id
|
||||
JOIN currentJobCanSync as cs on cs.jobId = j.id
|
||||
WHERE j.id = $1;
|
||||
|
||||
-- name: CreateJob :one
|
||||
@@ -18,3 +12,79 @@ INSERT INTO jobCanSync (canSync, jobId) VALUES ($1, $2);
|
||||
|
||||
-- name: ListJobDocumentIDsBatch :many
|
||||
SELECT id, totalCount FROM listJobDocumentIDs(@jobId, @batchSize, @pageOffset);
|
||||
|
||||
-- name: IsJobSynced :one
|
||||
WITH
|
||||
docs AS (
|
||||
-- Get all documents for this job
|
||||
SELECT id, jobId FROM documents WHERE jobId = $1
|
||||
),
|
||||
doc_text_entries AS (
|
||||
-- Documents with their current text entries
|
||||
SELECT
|
||||
d.id AS document_id,
|
||||
cte.id AS text_entry_id
|
||||
FROM
|
||||
docs d
|
||||
LEFT JOIN currentTextEntries cte ON cte.documentId = d.id
|
||||
),
|
||||
required_results AS (
|
||||
-- All required document-query-version combinations
|
||||
SELECT
|
||||
d.id AS document_id,
|
||||
cqdt.queryId,
|
||||
cqdt.queryVersion,
|
||||
dte.text_entry_id
|
||||
FROM
|
||||
docs d
|
||||
JOIN collectorQueryDependencyTree cqdt ON cqdt.jobId = d.jobId
|
||||
JOIN doc_text_entries dte ON dte.document_id = d.id
|
||||
and dte.text_entry_id IS NOT NULL
|
||||
),
|
||||
existing_results AS (
|
||||
-- Valid results that exist
|
||||
SELECT
|
||||
rr.document_id AS document_id,
|
||||
r.queryId,
|
||||
r.id AS result_id
|
||||
FROM
|
||||
results r
|
||||
JOIN required_results rr ON
|
||||
r.queryId = rr.queryId AND
|
||||
r.queryVersion = rr.queryVersion AND
|
||||
r.textEntryId = rr.text_entry_id
|
||||
),
|
||||
missing_results AS (
|
||||
-- Find missing results
|
||||
SELECT rr.queryId
|
||||
FROM required_results rr
|
||||
LEFT JOIN existing_results er on er.queryId = rr.queryId
|
||||
WHERE er.result_id is null
|
||||
),
|
||||
dependency_check AS (
|
||||
-- Check for missing dependencies
|
||||
SELECT DISTINCT er.queryId, er.result_id, qcri.queryId, rd.resultId
|
||||
FROM existing_results er
|
||||
JOIN queryCurrentRequiredIds qcri on qcri.requiredQueryId = er.queryId
|
||||
LEFT JOIN resultDependencies rd on rd.requiredResultId = er.result_id
|
||||
WHERE rd.resultId is null
|
||||
)
|
||||
SELECT (
|
||||
-- No documents means job is synced
|
||||
NOT EXISTS (SELECT 1 FROM docs)
|
||||
|
||||
OR
|
||||
|
||||
-- Documents with no text entries
|
||||
(NOT EXISTS (SELECT 1 FROM doc_text_entries WHERE text_entry_id IS NULL)
|
||||
|
||||
and
|
||||
|
||||
-- Documents with missing results
|
||||
NOT EXISTS (SELECT 1 FROM missing_results)
|
||||
|
||||
and
|
||||
|
||||
-- Documents with missing dependencies
|
||||
NOT EXISTS (SELECT 1 FROM dependency_check))
|
||||
)::bool as is_synced;
|
||||
|
||||
+22
-17
@@ -1,5 +1,5 @@
|
||||
-- name: GetQueryConfig :one
|
||||
SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 and COALESCE(removedVersion, $2 - 1) < $2;
|
||||
-- name: GetActiveQueryConfig :one
|
||||
SELECT config FROM queryCurrentConfigs where queryId = $1;
|
||||
|
||||
-- name: GetQuery :one
|
||||
SELECT * FROM fullActiveQueries WHERE id = $1;
|
||||
@@ -16,22 +16,23 @@ SELECT c.queryId, c.config
|
||||
),
|
||||
requiredIds as (
|
||||
SELECT r.queryId,
|
||||
coalesce(
|
||||
ARRAY_AGG(DISTINCT r.requiredQueryId)
|
||||
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
|
||||
as requiredIds
|
||||
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
|
||||
array[]::uuid[]
|
||||
)::uuid[] as requiredIds
|
||||
FROM query AS q
|
||||
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
|
||||
and isInVersion(@version, r.addedVersion, r.removedVersion)
|
||||
GROUP BY r.queryId
|
||||
)
|
||||
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
|
||||
coalesce(
|
||||
r.requiredIds,
|
||||
array[]::uuid[]
|
||||
)::uuid[] as requiredIds
|
||||
SELECT DISTINCT q.id, q.type,
|
||||
av.activeVersion,
|
||||
lv.latestVersion, c.config,
|
||||
r.requiredIds
|
||||
FROM query AS q
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
JOIN queryLatestVersions as lv on lv.queryId = q.id
|
||||
LEFT JOIN config AS c ON q.id = c.queryId
|
||||
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
|
||||
|
||||
@@ -66,17 +67,21 @@ SELECT COUNT(*) = COUNT(DISTINCT id) AS all_exist
|
||||
|
||||
-- name: IsQueryInDependencyTree :one
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM queryActiveDependencies WHERE id = any($1) and requiredQueryId = $2 or $2 = any($1)
|
||||
SELECT 1 FROM queryActiveDependencies
|
||||
WHERE id = any(@requiredQueryIds)
|
||||
and requiredQueryId = @queryId
|
||||
or @queryId = any(@requiredQueryIds)
|
||||
);
|
||||
|
||||
-- name: ListQueryDirectDependentsByDocumentID :many
|
||||
WITH doc AS (
|
||||
SELECT id, jobId FROM documents where id = $2
|
||||
SELECT id, jobId FROM documents where id = @documentId
|
||||
)
|
||||
SELECT dt.queryId
|
||||
FROM collectorQueryDependencyTree as dt
|
||||
JOIN doc as d on d.jobId = dt.jobId
|
||||
where $1 = any(dt.requiredIds);
|
||||
FROM doc as d
|
||||
JOIN collectorQueryDependencyTree as dt
|
||||
on d.jobId = dt.jobId
|
||||
and @queryId = any(dt.requiredIds);
|
||||
|
||||
-- name: ListQueryJobIDs :many
|
||||
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
|
||||
SELECT jobId FROM collectorQueryDependencyTree WHERE queryId = $1;
|
||||
|
||||
+41
-39
@@ -1,9 +1,9 @@
|
||||
-- name: ListQueryRequirementValues :many
|
||||
WITH reqQueries as (
|
||||
SELECT q.id as queryId, av.id as activeVersion, q.type
|
||||
SELECT av.queryId, av.activeVersion, q.type
|
||||
FROM requiredQueries as rq
|
||||
JOIN queries as q on q.id = rq.requiredQueryId
|
||||
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
|
||||
JOIN queryCurrentActiveVersions as av on av.queryId = rq.requiredQueryId
|
||||
JOIN queries as q on q.id = av.queryId
|
||||
WHERE rq.queryId = @queryId
|
||||
and isInVersion(@version, rq.addedVersion, rq.removedVersion)
|
||||
),
|
||||
@@ -13,70 +13,72 @@ docs as (
|
||||
WHERE id = @documentId
|
||||
),
|
||||
codeVersions as (
|
||||
SELECT
|
||||
SELECT
|
||||
d.id as documentId,
|
||||
coalesce(ccv.versionId, 1) as minCleanVersion,
|
||||
coalesce(ctv.versionId, 1) as minTextVersion
|
||||
mcv.minCleanVersion,
|
||||
mtv.minTextVersion
|
||||
FROM docs as d
|
||||
LEFT JOIN collectors as c on c.jobId = d.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
|
||||
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
|
||||
LIMIT 1
|
||||
JOIN currentCollectorMinTextVersions as mtv on mtv.jobId = d.jobId
|
||||
JOIN currentCollectorMinCleanVersions as mcv on mcv.jobId = d.jobId
|
||||
),
|
||||
latestVersions AS (
|
||||
SELECT
|
||||
SELECT
|
||||
r.id,
|
||||
rq.queryId,
|
||||
rq.type,
|
||||
r.queryVersion,
|
||||
r.cleanVersion,
|
||||
r.textVersion,
|
||||
r.textEntryId,
|
||||
r.value,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY rq.queryId
|
||||
ORDER BY r.cleanVersion DESC, r.textVersion DESC
|
||||
PARTITION BY r.queryId
|
||||
ORDER BY r.id DESC
|
||||
) as rowNumber
|
||||
FROM reqQueries as rq
|
||||
JOIN currentTextEntries as cte on cte.documentId = @documentId
|
||||
LEFT JOIN results as r ON rq.queryId = r.queryId
|
||||
and r.documentId = @documentId
|
||||
and r.queryVersion = rq.activeVersion
|
||||
JOIN codeVersions as ccv on ccv.documentId = r.documentId
|
||||
and r.cleanVersion >= ccv.minCleanVersion
|
||||
and r.textVersion >= ccv.minTextVersion
|
||||
and cte.id = r.textEntryId
|
||||
)
|
||||
SELECT DISTINCT lv.queryId, lv.type, r.value
|
||||
FROM latestVersions as lv
|
||||
JOIN results as r ON r.queryId = lv.queryId
|
||||
and r.documentId = @documentId
|
||||
and r.queryVersion = lv.queryVersion
|
||||
and r.cleanVersion = lv.cleanVersion
|
||||
and r.textVersion = lv.textVersion
|
||||
and lv.rowNumber = 1;
|
||||
SELECT DISTINCT id, queryId, type, value
|
||||
FROM latestVersions
|
||||
WHERE rowNumber = 1;
|
||||
|
||||
-- name: SetResult :exec
|
||||
INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6);
|
||||
-- name: AddResult :one
|
||||
INSERT INTO results (queryId, value, textEntryId, queryVersion) VALUES ($1, $2, $3, $4) returning id;
|
||||
|
||||
-- name: AddResultDependency :exec
|
||||
INSERT INTO resultDependencies (resultId, requiredResultId) VALUES ($1, $2);
|
||||
|
||||
-- name: GetResultValueWithVersion :one
|
||||
SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;
|
||||
WITH doc as (
|
||||
SELECT id, jobId
|
||||
FROM documents
|
||||
WHERE id = @documentId
|
||||
)
|
||||
SELECT r.id, r.value
|
||||
FROM doc as d
|
||||
JOIN currentTextEntries as cte on cte.documentId = d.id
|
||||
LEFT JOIN results as r
|
||||
on r.queryId = @queryId
|
||||
and r.queryVersion = @queryVersion
|
||||
and r.textEntryId = cte.id;
|
||||
|
||||
-- name: ListUnsyncedNoDepsQueriesByDocId :many
|
||||
WITH docs as (
|
||||
SELECT id, jobId from documents where id = $1
|
||||
),
|
||||
unsyncedQueries AS (
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
|
||||
SELECT DISTINCT dt.queryId, dt.requiredIds
|
||||
from docs as d
|
||||
JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
|
||||
JOIN fullActiveCollectors as c on c.id = dt.collectorId
|
||||
LEFT JOIN results as r on r.queryId = dt.queryId
|
||||
and r.documentId = d.id
|
||||
JOIN currentTextEntries as cte on cte.documentId = d.id
|
||||
LEFT JOIN results as r
|
||||
on r.queryId = dt.queryId
|
||||
and r.queryVersion = dt.queryVersion
|
||||
and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
|
||||
and cte.id = r.textEntryId
|
||||
where r.value is null
|
||||
)
|
||||
SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
|
||||
);
|
||||
|
||||
@@ -1,25 +1,12 @@
|
||||
-- name: IsDocumentTextExtracted :one
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
SELECT 1 FROM currentTextEntries WHERE documentId = @documentId
|
||||
);
|
||||
|
||||
-- name: AddDocumentTextEntry :exec
|
||||
INSERT INTO documentTextExtractions (documentId, version, bucket, key) VALUES ($1, $2, $3, $4);
|
||||
INSERT INTO documentTextExtractions (version, bucket, key, cleanEntryId) VALUES ($1, $2, $3, $4);
|
||||
|
||||
-- name: GetDocumentTextEntry :one
|
||||
SELECT dc.documentId, dc.bucket, dc.key, dc.version
|
||||
FROM documentTextExtractions AS dc
|
||||
JOIN documents AS d ON d.id = dc.documentId
|
||||
LEFT JOIN collectors as c ON d.jobId = c.jobId
|
||||
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
|
||||
LEFT JOIN collectorMinTextVersions AS ccv ON c.id = ccv.collectorId
|
||||
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
|
||||
WHERE dc.documentId = $1 and dc.version >= coalesce(ccv.versionId, 1)
|
||||
ORDER BY d.id DESC LIMIT 1;
|
||||
SELECT id, documentId, bucket, key, version, cleanEntryId
|
||||
FROM currentTextEntries
|
||||
WHERE documentId = @documentId;
|
||||
|
||||
Reference in New Issue
Block a user