query-orchestration/internal/database/migrations/00000000000102_document_views.up.sql

CREATE OR REPLACE FUNCTION listDocumentIDs(
    _clientId varchar(255),
    _batchSize INTEGER,
    _offset INTEGER
)
RETURNS TABLE (id uuid, totalCount BIGINT) AS $$
DECLARE
    totalCount BIGINT := 0;
BEGIN
    SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE clientId = _clientId;

    RETURN QUERY
        SELECT d.id, totalCount
        FROM documents as d
        WHERE d.clientId = _clientId
        ORDER BY d.id
        LIMIT _batchSize
        OFFSET _offset;
END;
$$ LANGUAGE plpgsql;

CREATE VIEW currentCleanEntries as
WITH RankedExtractions AS (
    SELECT
        dc.id,
        dc.documentId,
        dc.bucket,
        dc.key,
        dc.mimetype,
        dc.fail,
        dc.hash,
        dce.version,
        d.clientId,
        ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) as row_num
    FROM documents d
    JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId
    JOIN documentCleans dc ON dc.documentId = d.id
    JOIN documentCleanEntries dce ON dc.id = dce.cleanId
        AND dce.version >= ccv.minCleanVersion
)
SELECT
    re.id,
    re.documentId,
    re.bucket,
    re.key,
    re.version,
    re.hash,
    re.mimetype,
    re.fail,
    re.clientId
FROM RankedExtractions re
WHERE row_num = 1;

CREATE VIEW currentTextEntries as
WITH RankedExtractions AS (
    SELECT
        extract.id,
        cc.documentId,
        extract.bucket,
        extract.key,
        extract.hash,
        extract.cleanId,
        dtee.version as extractionVersion,
        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
    FROM documents d
    JOIN currentCleanEntries cc on cc.documentId = d.id
    JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
    JOIN documentTextExtractions extract ON extract.cleanId = cc.id
    JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
        AND dtee.version >= ctv.minTextVersion
)
SELECT
    id,
    documentId,
    bucket,
    key,
    hash,
    cleanId,
    extractionVersion
FROM RankedExtractions
WHERE row_num = 1;

CREATE VIEW currentClientCanSync as
WITH RankedExtractions AS (
    SELECT
        ccs.clientId,
        ccs.canSync,
        ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) as row_num
    FROM clients c
    JOIN clientCanSync ccs on c.clientId = ccs.clientId
)
SELECT
    c.clientId,
    coalesce(r.canSync, false) as canSync
FROM clients c
LEFT JOIN RankedExtractions r on r.clientId = c.clientId and r.row_num = 1;

CREATE VIEW fullClients as
SELECT c.clientId, c.name, cs.canSync
    FROM clients as c
    JOIN currentClientCanSync as cs on cs.clientId = c.clientId;

CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid)
RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$
DECLARE
 count_before INT;
 count_after INT;
 iterations INT := 0;
BEGIN
    CREATE TEMPORARY TABLE allResults AS
    WITH
        docs AS (
            SELECT id as documentId, clientId
            FROM documents
            WHERE id = doc_id
    ),
    tree as (
        select q.queryId, q.requiredIds, q.clientId, q.queryVersion
            from docs,
                LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
    ),
    query_dependency_tree AS (
        WITH RECURSIVE query_deps AS (
            SELECT
            q.queryId,
            unnest(q.requiredIds) as requiredId
            FROM tree AS q
            WHERE array_length(q.requiredIds, 1) > 0

            UNION ALL

            SELECT
            qd.queryId,
            unnest(q.requiredIds) as requiredId
            FROM query_deps qd
            JOIN tree q ON qd.requiredId = q.queryId
            WHERE array_length(q.requiredIds, 1) > 0
        )
        SELECT
        query_deps.queryId,
        array_agg(DISTINCT requiredId) AS all_required_ids
        FROM query_deps
        GROUP BY query_deps.queryId
    ),
    docResults AS (
    SELECT
        d.documentId,
        q.queryId,
        r.id AS resultId,
        r.value,
        q.requiredIds as directRequiredIds,
        COALESCE(qdt.all_required_ids, ARRAY[]::uuid[]) AS allRequiredIds
        FROM docs AS d
        JOIN tree AS q ON q.clientId = d.clientId
        LEFT JOIN query_dependency_tree qdt ON q.queryId = qdt.queryId
        LEFT JOIN currentTextEntries AS tte ON tte.documentId = d.documentId
        LEFT JOIN results AS r
            ON q.queryId = r.queryId
            AND r.queryVersion = q.queryVersion
            AND tte.id = r.textEntryId
        WHERE r.value is not null
    )
    SELECT * FROM docResults dr;

    CREATE TEMPORARY TABLE finalResults AS
    SELECT * from allResults where array_length(directRequiredIds, 1) is null;

    SELECT COUNT(*) INTO count_before FROM finalResults;
    RAISE NOTICE 'Starting with % rows', count_before;

    LOOP
        iterations := iterations + 1;

        INSERT INTO finalResults
            SELECT dr.*
            FROM allResults dr
            WHERE
                dr.directRequiredIds IS NOT NULL
                AND NOT EXISTS (
                    SELECT 1
                    FROM unnest(dr.directRequiredIds) AS req_id(queryId)
                    WHERE NOT EXISTS (
                        SELECT 1
                        FROM allResults req_dr
                        WHERE req_dr.queryId = req_id.queryId
                        AND req_dr.resultId IN (SELECT fr.resultId FROM finalResults fr)
                        AND exists (
                            select 1 from resultDependencies rrd
                            where dr.resultId = rrd.resultId
                            and req_dr.resultId = rrd.requiredResultId
                        )
                    )
                )
                and dr.resultId NOT IN (SELECT fr.resultId FROM finalResults fr);

        GET DIAGNOSTICS count_after = ROW_COUNT;
        RAISE NOTICE 'Iteration %: Additional Entries %',
        iterations, count_after;

        IF count_after = 0 THEN
            EXIT;
        END IF;
    END LOOP;

    RETURN QUERY SELECT tr.documentId, tr.queryId, tr.resultId, tr.value
        FROM finalResults tr;

    DROP TABLE finalResults;
    DROP TABLE allResults;
END;
$$ LANGUAGE plpgsql;