Merged in feature/testquery (pull request #39)

Test Query

* depstextandclean

* startedcleaningresult

* resulttidyup

* roundone

* cleaning

* unsyncedquery

* startedtestsandsimplification

* api

* querytests

* resultprocessortests

* unittests

* cleanup

database/migrations/00000000000100_query_views.up.sql

@@ -1,5 +1,12 @@
 CREATE VIEW fullActiveQueries AS
-    SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
+    SELECT DISTINCT 
+    q.id, q.type, q.activeVersion, q.latestVersion,
+    coalesce(c.config, null) as config, 
+    coalesce(
+        ARRAY_AGG(r.requiredQueryId)
+        FILTER (WHERE r.requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
+        array[]::uuid[]
+    )::uuid[] as requiredIds
     FROM queries AS q
     LEFT JOIN queryConfigs AS c ON q.id = c.queryId
         and q.activeVersion >= c.addedVersion

database/migrations/00000000000101_collector_views.up.sql

@@ -1,5 +1,5 @@
 CREATE VIEW fullActiveCollectors AS
-    SELECT DISTINCT c.id, c.jobId, cv.minCleanVersion, cv.minTextVersion, c.activeVersion, c.latestVersion,
+    SELECT DISTINCT c.id, c.jobId, coalesce(cv.minCleanVersion, 0) as minCleanVersion, coalesce(cv.minTextVersion, 0) as minTextVersion, c.activeVersion, c.latestVersion,
         jsonb_object_agg(q.name, q.queryId) FILTER (WHERE q.name is not null) AS fields
     FROM collectors AS c
     LEFT JOIN collectorCodeVersions AS cv ON c.id = cv.collectorId
@@ -29,5 +29,5 @@ CREATE VIEW collectorQueryDependencyTree AS
         FROM fullActiveQueries as q
         JOIN collectorQueryDependencyTree as acq on q.id = ANY(acq.requiredIds)
     )
-    SELECT collectorId, queryId, type, activeVersion as queryVersion, requiredIds::uuid[]
+    SELECT collectorId, queryId, type, activeVersion as queryVersion, requiredIds
         FROM collectorQueryDependencyTree;

database/queries/query.sql

@@ -4,9 +4,24 @@ SELECT id, config FROM queryConfigs where queryId = $1 and addedVersion >= $2 an
 -- name: GetQuery :one
 SELECT * FROM fullActiveQueries WHERE id = $1;
 
+-- name: GetQueryWithVersion :one
+SELECT DISTINCT q.id, q.type, q.activeVersion, q.latestVersion, coalesce(c.config, null) as config, ARRAY_AGG(DISTINCT r.requiredQueryId)::uuid[] as requiredIds
+    FROM queries AS q
+    LEFT JOIN queryConfigs AS c ON q.id = c.queryId
+        and $2 >= c.addedVersion
+        and $2 < COALESCE(c.removedVersion, $2 + 1)
+    LEFT JOIN requiredQueries AS r ON q.id = r.queryId
+        and $2 >= r.addedVersion
+        and $2 < COALESCE(r.removedVersion, $2 + 1)
+    WHERE q.id = $1
+    GROUP BY q.id, q.type, q.activeversion, q.latestversion, c.config;
+
 -- name: ListQueries :many
 SELECT * FROM fullActiveQueries;
 
+-- name: ListQueriesById :many
+SELECT * FROM fullActiveQueries WHERE id = any($1);
+
 -- name: CreateQuery :one
 INSERT INTO queries (type) VALUES ($1) RETURNING id;
 

database/queries/result.sql

@@ -1,9 +1,49 @@
--- name: ListResultsByDocumentID :many
-SELECT id, queryId, queryVersion FROM results
-    where documentId = $1 and cleanVersion >= $2 and textVersion >= $3;
-
--- name: ListResultValuesByID :many
-SELECT id, queryId, value FROM results where id = ANY($1);
+-- name: ListQueryRequirementValues :many
+WITH latest_versions AS (
+    SELECT 
+        r.queryId,
+        MAX(r.cleanVersion) as max_clean_version,
+        MAX(r.textVersion) as max_text_version
+    FROM results r
+    WHERE r.documentId = $3
+        AND r.queryVersion = $2
+    GROUP BY r.queryId
+)
+SELECT rq.requiredQueryId as queryId, r.value, q.type
+    FROM requiredQueries as rq
+    JOIN results as r on r.queryId = rq.requiredQueryId
+    JOIN queries as q on q.id = rq.requiredQueryId
+    JOIN latest_versions lv ON lv.queryId = r.queryId
+    WHERE rq.queryId = $1 and r.documentId = $3
+        and $2 >= rq.addedVersion
+        and $2 < COALESCE(rq.removedVersion, $2 + 1)
+        and r.queryVersion = $2
+        and r.cleanVersion >= $4 and r.textVersion >= $5
+        AND r.cleanVersion = lv.max_clean_version 
+        AND r.textVersion = lv.max_text_version;
 
 -- name: SetResult :one
-INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
+INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
+
+-- name: GetResultValueWithVersion :one
+SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;
+
+-- name: ListUnsyncedQueriesByDocId :many
+WITH RECURSIVE unsyncedQueries AS (
+    SELECT dt.queryId, dt.requiredIds
+        from documents as d
+        JOIN fullActiveCollectors as c on d.jobId = c.jobId
+        JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
+        LEFT JOIN results as r on r.queryId = dt.queryId
+            and r.queryVersion = dt.queryVersion
+            and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
+        where d.id = $1 and r.value is null
+
+    UNION
+
+    SELECT DISTINCT dt.queryId, dt.requiredIds
+        FROM unsyncedQueries as u
+        JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
+)
+SELECT * FROM fullActiveQueries 
+WHERE id in (SELECT queryId FROM unsyncedQueries);