Merged in feature/splitqueryrunning (pull request #57)

Split Query Running + Debugging Full Flow * completedquerysyncrunner * spliitinglogic * synccomplete * informdependents * only push same collector * deps * livetesting * foundissue * some issues resolved * activeupdate * collectorupdatefixes * fix dbquesries * tests * tests * pollingdebug
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
@@ -1,49 +1,77 @@
 -- name: ListQueryRequirementValues :many
-WITH latest_versions AS (
-    SELECT 
-        r.queryId,
-        MAX(r.cleanVersion) as max_clean_version,
-        MAX(r.textVersion) as max_text_version
-    FROM results r
-    WHERE r.documentId = $3
-        AND r.queryVersion = $2
-    GROUP BY r.queryId
-)
-SELECT rq.requiredQueryId as queryId, r.value, q.type
+WITH reqQueries as (
+    SELECT q.id as queryId, q.activeVersion, q.type
    FROM requiredQueries as rq
-    JOIN results as r on r.queryId = rq.requiredQueryId
    JOIN queries as q on q.id = rq.requiredQueryId
-    JOIN latest_versions lv ON lv.queryId = r.queryId
-    WHERE rq.queryId = $1 and r.documentId = $3
-        and $2 >= rq.addedVersion
-        and $2 < COALESCE(rq.removedVersion, $2 + 1)
-        and r.queryVersion = $2
-        and r.cleanVersion >= $4 and r.textVersion >= $5
-        AND r.cleanVersion = lv.max_clean_version 
-        AND r.textVersion = lv.max_text_version;
+    WHERE rq.queryId = @queryId
+        and @version >= rq.addedVersion
+        and (rq.removedVersion is null or @version < rq.removedVersion)
+),
+codeVersions as (
+    SELECT 
+        d.id as documentId,
+        coalesce(ccv.minCleanVersion, 1) as minCleanVersion,
+        coalesce(ccv.minTextVersion, 1) as minTextVersion
+    FROM documents as d
+    LEFT JOIN collectors as c on c.jobId = d.jobId
+    LEFT JOIN collectorCodeVersions as ccv on c.id = ccv.collectorId
+        and c.activeVersion >= ccv.addedVersion
+        and c.activeVersion < COALESCE(ccv.removedVersion, c.activeVersion)
+    WHERE d.id = @documentId
+    LIMIT 1
+),
+latestVersions AS (
+    SELECT 
+        rq.queryId,
+        rq.type,
+        r.queryVersion,
+        r.cleanVersion,
+        r.textVersion,
+        ROW_NUMBER() OVER (
+            PARTITION BY rq.queryId
+            ORDER BY r.cleanVersion DESC, r.textVersion DESC
+        ) as rowNumber
+    FROM reqQueries as rq
+    LEFT JOIN results as r ON rq.queryId = r.queryId
+        and r.documentId = @documentId
+    JOIN codeVersions as ccv on ccv.documentId = r.documentId
+    WHERE r.documentId = @documentId
+        and r.queryVersion = rq.activeVersion
+        and r.cleanVersion >= ccv.minCleanVersion
+        and r.textVersion >= ccv.minTextVersion
+)
+SELECT lv.queryId, lv.type, r.value
+FROM latestVersions as lv
+JOIN results as r ON r.queryId = lv.queryId
+    and r.documentId = @documentId
+    and r.queryVersion = lv.queryVersion
+    and r.cleanVersion = lv.cleanVersion
+    and r.textVersion = lv.textVersion
+    and lv.rowNumber = 1;

-- name: SetResult :one
-INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id;
+-- name: SetResult :exec
+INSERT INTO results (queryId, documentId, value, cleanVersion, textVersion, queryVersion) VALUES ($1, $2, $3, $4, $5, $6);

 -- name: GetResultValueWithVersion :one
 SELECT id, value FROM results WHERE queryId = $1 and queryVersion = $2 and documentId = $3 and cleanVersion >= $4 and textVersion >= $5;

-- name: ListUnsyncedQueriesByDocId :many
-WITH RECURSIVE unsyncedQueries AS (
-    SELECT dt.queryId, dt.requiredIds
-        from documents as d
-        JOIN fullActiveCollectors as c on d.jobId = c.jobId
-        JOIN collectorQueryDependencyTree as dt on c.id = dt.collectorId
+-- name: ListUnsyncedNoDepsQueriesByDocId :many
+WITH docs as (
+    SELECT id, jobId from documents where id = $1
+),
+unsyncedQueries AS (
+    SELECT DISTINCT dt.queryId, dt.requiredIds, r.value, d.jobID, d.id
+        from docs as d
+        JOIN collectorQueryDependencyTree as dt on d.jobId = dt.jobId
+        JOIN fullActiveCollectors as c on c.id = dt.collectorId
        LEFT JOIN results as r on r.queryId = dt.queryId
+            and r.documentId = d.id
            and r.queryVersion = dt.queryVersion
            and r.cleanVersion >= c.minCleanVersion and r.textVersion >= c.minTextVersion
-        where d.id = $1 and r.value is null
-
-    UNION
-
-    SELECT DISTINCT dt.queryId, dt.requiredIds
-        FROM unsyncedQueries as u
-        JOIN collectorQueryDependencyTree as dt ON u.queryId = any(dt.requiredIds)
+        where r.value is null
 )
-SELECT * FROM fullActiveQueries 
-WHERE id in (SELECT queryId FROM unsyncedQueries);
+SELECT DISTINCT queryId FROM unsyncedQueries as baseuq
+    WHERE NOT EXISTS (
+        SELECT 1 FROM unsyncedQueries as uq WHERE uq.queryId = any(baseuq.requiredIds)
+    );
+