Merged in feature/jobsynced (pull request #72)

Job Status Get and DB tidy up

* initalquery

* tests

* shorttests

* testing queries

* job

* solvedthequery

* updatingdb

* fixingtests

* repotests

* shorttests

* docker

* testspassed
This commit is contained in:
Michael McGuinness
2025-02-20 19:02:44 +00:00
parent 0ea544926b
commit 3d434eedb8
74 changed files with 2395 additions and 1335 deletions
@@ -1,16 +1,9 @@
CREATE TABLE collectors (
id uuid primary key DEFAULT uuid_generate_v7(),
jobId uuid not null,
foreign key (jobId) references jobs(id),
unique (jobId)
);
CREATE TABLE collectorVersions (
collectorId uuid not null,
jobId uuid not null,
id int not null,
addedAt timestamp not null default current_timestamp,
primary key (id, collectorId),
foreign key (collectorId) references collectors(id)
foreign key (jobId) references jobs(id),
primary key (id, jobId)
);
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
@@ -19,8 +12,8 @@ BEGIN
SELECT COALESCE(MAX(id), 0) + 1
INTO NEW.id
FROM collectorVersions
WHERE collectorId = NEW.collectorId;
WHERE jobId = NEW.jobId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -32,20 +25,20 @@ EXECUTE FUNCTION setCollectorVersionNumber();
CREATE TABLE collectorActiveVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
foreign key (collectorId, versionId) references collectorVersions(collectorId, id)
foreign key (jobId, versionId) references collectorVersions(jobId, id)
);
CREATE TABLE collectorMinCleanVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
addedVersion int not null,
removedVersion int,
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId) references collectors(id)
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
foreign key (jobId) references jobs(id)
);
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
@@ -53,8 +46,8 @@ RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinCleanVersions
SET removedVersion = NEW.addedVersion
WHERE collectorId = NEW.collectorId and removedVersion is null;
WHERE jobId = NEW.jobId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -66,13 +59,13 @@ EXECUTE FUNCTION removeMinCleanVersion();
CREATE TABLE collectorMinTextVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
versionId int not null,
addedVersion int not null,
removedVersion int,
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId) references collectors(id)
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
foreign key (jobId) references jobs(id)
);
CREATE OR REPLACE FUNCTION removeMinTextVersion()
@@ -80,8 +73,8 @@ RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinTextVersions
SET removedVersion = NEW.addedVersion
WHERE collectorId = NEW.collectorId and removedVersion is null;
WHERE jobId = NEW.jobId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
@@ -93,14 +86,14 @@ EXECUTE FUNCTION removeMinTextVersion();
CREATE TABLE collectorQueries (
id uuid primary key DEFAULT uuid_generate_v7(),
collectorId uuid not null,
jobId uuid not null,
name varchar(255) not null,
queryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(id),
foreign key (collectorId) references collectors(id),
foreign key (collectorId, addedVersion) references collectorVersions(collectorId, id),
foreign key (collectorId, removedVersion) references collectorVersions(collectorId, id),
unique (collectorId, name, removedVersion)
);
foreign key (jobId) references jobs(id),
foreign key (jobId, addedVersion) references collectorVersions(jobId, id),
foreign key (jobId, removedVersion) references collectorVersions(jobId, id),
unique (jobId, name, removedVersion)
);
@@ -25,9 +25,9 @@ CREATE TABLE documentCleans (
CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
documentId uuid not null,
cleanEntryId uuid not null,
version int not null,
bucket text not null,
key text not null,
foreign key (documentId) references documents(id)
foreign key (cleanEntryId) references documentCleans(id)
);
@@ -1,13 +1,18 @@
CREATE TABLE results (
id uuid primary key DEFAULT uuid_generate_v7(),
textEntryId uuid not null,
queryId uuid not null,
documentId uuid not null,
value TEXT not null,
cleanVersion int not null,
textVersion int not null,
queryVersion int not null,
foreign key (queryId) references queries(id),
foreign key (documentId) references documents(id),
unique (queryId, documentId, cleanVersion, textVersion, queryVersion),
foreign key (textEntryId) references documentTextExtractions(id),
foreign key (queryId, queryVersion) references queryVersions(queryId, id)
);
);
CREATE TABLE resultDependencies (
resultId uuid not null,
requiredResultId uuid not null,
foreign key (resultId) references results(id),
foreign key (requiredResultId) references results(id),
CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
);
@@ -1,59 +1,71 @@
CREATE VIEW queryCurrentActiveVersions as
SELECT DISTINCT
av.queryId,
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.queryId ORDER BY av.id DESC))::int as id
FROM queryActiveVersions as av;
q.id as queryId,
coalesce(
(FIRST_VALUE(av.versionId) OVER (PARTITION BY q.id ORDER BY av.id DESC)),
0
)::int as activeVersion
FROM queries AS q
LEFT JOIN queryActiveVersions as av on av.queryId = q.id;
CREATE VIEW queryLatestVersions as
SELECT queryId, max(id)::int as id
FROM queryVersions
GROUP BY queryId;
SELECT
q.id as queryId,
coalesce(max(v.id), 0)::int as latestVersion
FROM queries AS q
LEFT JOIN queryVersions as v on v.queryId = q.id
GROUP BY q.id;
CREATE VIEW fullActiveQueries AS
with config as (
SELECT c.queryId, c.config
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryConfigs AS c ON q.id = c.queryId
and isInVersion(av.id, c.addedVersion, c.removedVersion)
),
requiredIds as (
SELECT r.queryId,
ARRAY_AGG(DISTINCT r.requiredQueryId)
FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[]
as requiredIds
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
GROUP BY r.queryId
)
SELECT DISTINCT q.id, q.type, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion, c.config,
CREATE VIEW queryCurrentConfigs as
SELECT av.queryId, c.config
FROM queryCurrentActiveVersions as av
LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
CREATE VIEW queryCurrentRequiredIds as
SELECT DISTINCT av.queryId, r.requiredQueryId
FROM queryCurrentActiveVersions as av
LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
CREATE VIEW queryCurrentRequiredIdsAGG as
SELECT queryId,
coalesce(
r.requiredIds,
ARRAY_AGG(DISTINCT requiredQueryId)
FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM queryCurrentRequiredIds
GROUP BY queryId;
CREATE VIEW fullActiveQueries AS
SELECT DISTINCT q.id, q.type, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN queryLatestVersions as lv on lv.queryId = q.id
LEFT JOIN config AS c ON q.id = c.queryId
LEFT JOIN requiredIds AS r ON q.id = r.queryId;
JOIN queryCurrentActiveVersions as av on q.id = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.id
JOIN queryCurrentConfigs AS c ON q.id = c.queryId
JOIN queryCurrentRequiredIdsAGG AS r ON q.id = r.queryId;
CREATE VIEW queryActiveDependencies AS
with recursive queryActiveDependencies AS (
SELECT q.id, r.requiredQueryId
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
SELECT
queryId,
requiredQueryId,
ARRAY[queryId, requiredQueryId]::uuid[] AS path,
false AS cycle
FROM queryCurrentRequiredIds
UNION ALL
SELECT qd.id, r.requiredQueryId
FROM queries AS q
LEFT JOIN queryCurrentActiveVersions as av on q.id = av.queryId
LEFT JOIN requiredQueries AS r ON q.id = r.queryId
and isInVersion(av.id, r.addedVersion, r.removedVersion)
JOIN queryActiveDependencies as qd on q.id = qd.requiredQueryId
SELECT
q.queryId,
qd.requiredQueryId,
path || qd.requiredQueryId,
qd.requiredQueryId = ANY(path) AS cycle
FROM queryCurrentRequiredIds as q
JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
WHERE NOT qd.cycle -- Stop if we detect a cycle
)
SELECT DISTINCT id, requiredQueryId FROM queryActiveDependencies;
SELECT DISTINCT queryId as id, requiredQueryId
FROM queryActiveDependencies
WHERE NOT cycle;
@@ -1,47 +1,77 @@
CREATE VIEW collectorCurrentActiveVersions as
SELECT DISTINCT
av.collectorId,
(FIRST_VALUE(av.versionId) OVER (PARTITION BY av.collectorId ORDER BY av.id DESC))::int as id
FROM collectorActiveVersions as av;
j.id as jobId,
coalesce(
(FIRST_VALUE(v.versionId) OVER (PARTITION BY j.id ORDER BY v.id DESC)),
0
)::int as activeVersion
FROM jobs as j
LEFT JOIN collectorActiveVersions as v on v.jobId = j.id;
CREATE VIEW collectorLatestVersions as
SELECT collectorId, max(id)::int as id
FROM collectorVersions
GROUP BY collectorId;
SELECT
j.id as jobId,
coalesce(max(v.id), 0)::int as latestVersion
FROM jobs as j
LEFT JOIN collectorVersions as v on v.jobId = j.id
GROUP BY j.id;
CREATE VIEW currentCollectorMinTextVersions as
SELECT DISTINCT
av.jobId,
coalesce(ctv.versionId, 0) as minTextVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorMinCleanVersions as
SELECT DISTINCT
av.jobId,
coalesce(ctv.versionId, 0) as minCleanVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinCleanVersions AS ctv ON av.jobId = ctv.jobId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorQueries as
SELECT DISTINCT
av.jobId,
q.name,
q.queryId
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorQueries AS q ON av.jobId = q.jobId
and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
CREATE VIEW currentCollectorQueriesJSONAGG as
SELECT DISTINCT
jobId,
jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
FROM currentCollectorQueries
GROUP BY jobId;
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT c.id, c.jobId, coalesce(ccv.versionId, 0) as minCleanVersion, coalesce(ctv.versionId, 0) as minTextVersion, coalesce(av.id, 0) as activeVersion, coalesce(lv.id, 0) as latestVersion,
jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
FROM collectors AS c
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorLatestVersions as lv on lv.collectorId = c.id
LEFT JOIN collectorMinCleanVersions AS ccv ON c.id = ccv.collectorId
and isInVersion(av.id, ccv.addedVersion, ccv.removedVersion)
LEFT JOIN collectorMinTextVersions AS ctv ON c.id = ctv.collectorId
and isInVersion(av.id, ctv.addedVersion, ctv.removedVersion)
LEFT JOIN collectorQueries AS q ON c.id = q.collectorId
and isInVersion(av.id, q.addedVersion, q.removedVersion)
GROUP BY c.id, c.jobId, ccv.versionId, ctv.versionId, av.id, lv.id;
SELECT DISTINCT av.jobId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion,
q.fields
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.jobId = av.jobId
JOIN currentCollectorMinCleanVersions AS ccv ON av.jobId = ccv.jobId
JOIN currentCollectorMinTextVersions AS ctv ON av.jobId = ctv.jobId
JOIN currentCollectorQueriesJSONAGG AS q ON av.jobId = q.jobId;
CREATE VIEW collectorQueryDependencyTree AS
WITH RECURSIVE
coll as (
SELECT DISTINCT c.id, c.jobId, q.queryId
FROM collectors as c
LEFT JOIN collectorCurrentActiveVersions as av on c.id = av.collectorId
LEFT JOIN collectorQueries as q ON c.id = q.collectorId
and isInVersion(av.id, q.addedVersion, q.removedVersion)
),
collectorQueryDependencyTree AS (
SELECT cq.id as collectorId, cq.jobId, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion
FROM coll as cq
JOIN fullActiveQueries as aqc on cq.queryId = aqc.id
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.jobId, cq.queryId, ri.requiredIds
FROM currentCollectorQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.collectorId, acq.jobId, q.id as queryId, q.type, q.requiredIds, q.activeVersion
FROM fullActiveQueries as q
JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
SELECT acq.jobId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT collectorId, jobId, queryId, type, activeVersion as queryVersion, requiredIds
FROM collectorQueryDependencyTree;
SELECT DISTINCT ct.jobId, ct.queryId, q.type, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.id = ct.queryId;
@@ -8,7 +8,7 @@ DECLARE
totalCount BIGINT := 0;
BEGIN
SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE jobId = _jobId;
RETURN QUERY
SELECT d.id, totalCount
FROM documents as d
@@ -17,4 +17,83 @@ BEGIN
LIMIT _batchSize
OFFSET _offset;
END;
$$ LANGUAGE plpgsql;
$$ LANGUAGE plpgsql;
CREATE VIEW currentCleanEntries as
WITH RankedExtractions AS (
SELECT
dc.id,
dc.documentId,
dc.bucket,
dc.key,
dc.version,
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC) as row_num
FROM documents d
JOIN currentCollectorMinCleanVersions ccv ON d.jobId = ccv.jobId
JOIN documentCleans dc ON dc.documentId = d.id
AND dc.version >= ccv.minCleanVersion
)
SELECT
id,
documentId,
bucket,
key,
version
FROM RankedExtractions
WHERE row_num = 1;
CREATE VIEW currentTextEntries as
WITH RankedExtractions AS (
SELECT
dte.id,
cc.documentId,
dte.bucket,
dte.key,
dte.version,
dte.cleanEntryId,
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num
FROM documents d
JOIN currentCleanEntries cc on cc.documentId = d.id
JOIN currentCollectorMinTextVersions ctv ON ctv.jobId = d.jobId
JOIN documentTextExtractions dte ON dte.cleanEntryId = cc.id
AND dte.version >= ctv.minTextVersion
)
SELECT
id,
documentId,
bucket,
key,
version,
cleanEntryId
FROM RankedExtractions
WHERE row_num = 1;
CREATE VIEW currentClientCanSync as
WITH RankedExtractions AS (
SELECT
ccs.clientId,
ccs.canSync,
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.id DESC) as row_num
FROM clients c
JOIN clientCanSync ccs on c.id = ccs.clientId
)
SELECT
c.id as clientId,
coalesce(r.canSync, false) as canSync
FROM clients c
LEFT JOIN RankedExtractions r on r.clientId = c.id and r.row_num = 1;
CREATE VIEW currentJobCanSync as
WITH RankedExtractions AS (
SELECT
ccs.jobId,
ccs.canSync,
ROW_NUMBER() OVER (PARTITION BY ccs.jobId ORDER BY ccs.id DESC) as row_num
FROM jobs c
JOIN jobCanSync ccs on c.id = ccs.jobId
)
SELECT
c.id as jobId,
coalesce(r.canSync, false) as canSync
FROM jobs c
LEFT JOIN RankedExtractions r on r.jobId = c.id and r.row_num = 1;