Merged in feature/jobcollector (pull request #30)

Initial Job Collector (changes pending)

* movearroundtocleancollector

* internalgetfunctions

* completecollectorquery

* simplify

* fixtests

* addvendor

* noplaceholder

database/migrations/20241219151404_create_activequeryrequirements_view.down.sql

@@ -1 +0,0 @@
-DROP VIEW activeQueryRequirements;

database/migrations/20241219151404_create_activequeryrequirements_view.up.sql

@@ -1,6 +0,0 @@
-CREATE VIEW activeQueryRequirements AS
-SELECT q.id, q.type, q.activeVersion, rq.requiredQueryId
-FROM queries q
-JOIN requiredQueries rq ON q.id = rq.queryId
-    AND q.activeVersion >= rq.addedVersion
-    AND q.activeVersion < COALESCE(rq.removedVersion, q.activeVersion + 1);

database/migrations/20241219161630_create_fullactivequery_view.up.sql

@@ -2,9 +2,9 @@ CREATE VIEW fullActiveQueries AS
     SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
     FROM queries AS q
     LEFT JOIN queryConfigs AS c ON q.id = c.queryId
-        and c.addedVersion >= q.activeVersion
-        and COALESCE(c.removedVersion, q.activeVersion - 1) < q.activeVersion
+        and q.activeVersion >= c.addedVersion
+        and q.activeVersion < COALESCE(c.removedVersion, q.activeVersion + 1)
     LEFT JOIN requiredQueries AS r ON q.id = r.queryId
-        and r.addedVersion >= q.activeVersion
-        and COALESCE(r.removedVersion, q.activeVersion - 1) < q.activeVersion
+        and q.activeVersion >= r.addedVersion
+        and q.activeVersion < COALESCE(r.removedVersion, q.activeVersion + 1)
     GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config;

database/migrations/20241219161635_create_activecollectorqueries_view.down.sql

@@ -1 +0,0 @@
-DROP VIEW activeCollectorQueries;

database/migrations/20241219161635_create_activecollectorqueries_view.up.sql

@@ -1,6 +0,0 @@
-CREATE VIEW activeCollectorQueries AS
-SELECT c.id as collectorId, c.activeVersion, cq.queryId
-FROM collectors c
-JOIN collectorQueries cq ON c.id = cq.collectorId
-    AND c.activeVersion >= cq.addedVersion
-    AND c.activeVersion < COALESCE(cq.removedVersion, c.activeVersion + 1);

database/migrations/20241219161635_create_activecollectorswithrequiredids_view.down.sql

@@ -0,0 +1 @@
+DROP VIEW activeCollectorsWithRequiredIDs;

database/migrations/20241219161635_create_activecollectorswithrequiredids_view.up.sql

@@ -0,0 +1,7 @@
+CREATE VIEW activeCollectorsWithRequiredIDs AS
+    SELECT DISTINCT c.id, c.activeVersion, array_agg(q.queryId) as queryIds
+    FROM collectors as c
+    LEFT JOIN collectorQueries as q ON c.id = q.collectorId
+        AND c.activeVersion >= q.addedVersion
+        and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1)
+    GROUP BY c.id, c.activeVersion;

database/migrations/20241219161636_create_collectorquerydep_view.up.sql

@@ -1,13 +1,14 @@
-CREATE VIEW collectorQueryDependencyTree (collectorId, queryId, type, requiredQueryId, queryVersion) AS
+CREATE VIEW collectorQueryDependencyTree AS
 WITH RECURSIVE collectorQueryDependencyTree AS (
-    SELECT cq.collectorId, aqc.id, aqc.type, aqc.requiredQueryId, aqc.activeVersion as queryVersion
-    FROM activeQueryRequirements aqc
-    JOIN activeCollectorQueries cq ON cq.queryId = aqc.id
+    SELECT cq.id, aqc.id as queryId, aqc.type, aqc.requiredIds, aqc.activeVersion
+    FROM fullActiveQueries as aqc
+    JOIN activeCollectorsWithRequiredIDs as cq ON aqc.id = ANY(cq.queryIds)
 
     UNION ALL
 
-    SELECT acq.collectorId, q.id as queryId, q.type, q.requiredQueryId, q.activeVersion as queryVersion
-    FROM activeQueryRequirements q
-    JOIN collectorQueryDependencyTree acq on q.id = acq.requiredQueryId
+    SELECT acq.id, q.id as queryId, q.type, q.requiredIds, q.activeVersion
+    FROM fullActiveQueries as q
+    JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
 )
-SELECT DISTINCT * FROM collectorQueryDependencyTree;
+SELECT id as collectorId, queryId, type, activeVersion as queryVersion, requiredIds::uuid[]
+    FROM collectorQueryDependencyTree;

database/migrations/20250120165410_create_fullactivecollector_view.down.sql

@@ -0,0 +1 @@
+DROP VIEW fullActiveCollectors;

database/migrations/20250120165410_create_fullactivecollector_view.up.sql

@@ -0,0 +1,8 @@
+CREATE VIEW fullActiveCollectors AS
+    SELECT DISTINCT c.id, c.jobId, c.minCleanVersion, c.minTextVersion, c.activeVersion, c.latestVersion,
+        jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
+    FROM collectors AS c
+    LEFT JOIN collectorQueries AS q ON c.id = q.collectorId
+        AND c.activeVersion >= q.addedVersion
+        and c.activeVersion < COALESCE(q.removedVersion, c.activeVersion + 1)
+    GROUP BY c.id, c.jobId, c.minCleanVersion, c.minTextVersion;

database/queries/collector.sql

@@ -1,8 +1,14 @@
--- name: GetCollectorQueries :many
-SELECT collectorId, queryId, type, queryVersion, ARRAY_AGG(requiredQueryId) AS requiredIds
-    FROM collectorQueryDependencyTree
-    WHERE collectorId = $1
-    GROUP BY queryId, collectorId, type, queryVersion;
+-- name: ListCollectorQueries :many
+SELECT * FROM collectorQueryDependencyTree WHERE collectorId = $1;
 
--- name: GetCollectorFromJobID :one
-SELECT id, jobId, minCleanVersion, minTextVersion FROM collectors WHERE jobId = $1 LIMIT 1;
+-- name: GetCollectorByJobID :one
+SELECT * FROM fullActiveCollectors WHERE jobId = $1 LIMIT 1;
+
+-- name: GetCollector :one
+SELECT * FROM fullActiveCollectors WHERE id = $1 LIMIT 1;
+
+-- name: CreateCollector :one
+INSERT INTO collectors (jobId, minCleanVersion, minTextVersion) VALUES ($1, $2, $3) RETURNING id;
+
+-- name: AddCollectorQuery :exec
+INSERT INTO collectorQueries (collectorId, name, queryId, addedVersion) VALUES ($1, $2, $3, $4);

database/queries/query.sql

@@ -10,10 +10,10 @@ SELECT EXISTS (
 );
 
 -- name: GetQuery :one
-SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries WHERE id = $1;
+SELECT * FROM fullActiveQueries WHERE id = $1;
 
 -- name: ListQueries :many
-SELECT id, type, activeVersion, latestVersion, config, requiredIds FROM fullActiveQueries;
+SELECT * FROM fullActiveQueries;
 
 -- name: CreateQuery :one
 INSERT INTO queries (type) VALUES ($1) RETURNING id;