jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/restore-migrations-v2 (pull request #205)

Browse Source 
Restore original migrations (001-120) to fix schema_migrations version mismatch

* Restore original migrations (001-120) to fix schema_migrations version mismatch

The previous migration collapse broke deployed databases because:
- Dev database was at version 120 (the real remove_text_extraction migration)
- Main had collapsed migrations (001-012) + stub files (119_stub, 120_stub)
- The migrate tool couldn't find version 119/120 with matching content

This commit restores the original 27 migrations (versions 001-120) from
before the collapse, ensuring backward compatibility with existing databases.
This commit is contained in:
Jay Brown
2026-01-15 23:18:53 +00:00
parent 3639af8b84
commit c10fa98d0a
69 changed files with 1271 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/migrations/00000000000001_extensions.down.sql
+1 -7
View File
@@ -1,7 +1 @@
-- Migration 001: Extensions and Core Functions (DOWN)
-- Drops in reverse order of creation
DROP DOMAIN IF EXISTS unsignedsmallint;
DROP FUNCTION IF EXISTS isInVersion;
DROP FUNCTION IF EXISTS uuid_generate_v7;
DROP EXTENSION IF EXISTS "pgcrypto";
DROP EXTENSION "pgcrypto";
internal/database/migrations/00000000000001_extensions.up.sql
+9 -15
View File
@@ -1,13 +1,9 @@
-- Migration 001: Extensions and Core Functions
-- Creates PostgreSQL extensions and utility functions used throughout the schema
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
-- UUID v7 generator function (time-ordered UUIDs)
CREATE OR REPLACE FUNCTION uuid_generate_v7()
RETURNS uuid
AS $$
SELECT encode(
create or replace function uuid_generate_v7()
returns uuid
as $$
select encode(
set_bit(
set_bit(
overlay(uuid_send(gen_random_uuid())
@@ -20,24 +16,22 @@ SELECT encode(
),
'hex')::uuid;
$$
LANGUAGE SQL
VOLATILE;
language SQL
volatile;
-- Version checking function for temporal data patterns
CREATE FUNCTION isInVersion(
_version int,
_addedVersion int,
_removedVersion int
)
RETURNS boolean AS $$
RETURNS boolean as $$
BEGIN
RETURN _version IS NULL OR (
RETURN _version is null OR (
_version >= _addedVersion AND
(_removedVersion IS NULL OR _version < _removedVersion)
(_removedVersion is null OR _version < _removedVersion)
);
END;
$$ LANGUAGE plpgsql;
-- Domain for unsigned small integers
CREATE DOMAIN unsignedsmallint AS SMALLINT
CHECK (VALUE >= 0);
internal/database/migrations/00000000000002_clients.down.sql
-4
View File
@@ -1,4 +0,0 @@
-- Migration 002: Clients (DOWN)
DROP TABLE IF EXISTS clientCanSync;
DROP TABLE IF EXISTS clients;
internal/database/migrations/00000000000002_clients.up.sql
-15
View File
@@ -1,15 +0,0 @@
-- Migration 002: Clients
-- Core client tables for customer configuration
CREATE TABLE clients (
clientId varchar(255) PRIMARY KEY,
name TEXT NOT NULL,
UNIQUE (name)
);
CREATE TABLE clientCanSync (
syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
clientId varchar(255) NOT NULL,
canSync boolean NOT NULL,
FOREIGN KEY (clientId) REFERENCES clients(clientId)
);
internal/database/migrations/00000000000002_queries.down.sql
+7
View File
@@ -0,0 +1,7 @@
DROP TABLE queries;
DROP TYPE queryType;
DROP TABLE requiredQueries;
DROP TABLE queryConfigs;
internal/database/migrations/00000000000002_queries.up.sql
+79
View File
@@ -0,0 +1,79 @@
CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor');
CREATE TABLE queries (
queryId uuid primary key DEFAULT uuid_generate_v7(),
queryType queryType not null
);
CREATE TABLE queryVersions (
queryId uuid not null,
versionId int not null,
addedAt timestamp not null default current_timestamp,
primary key (versionId, queryId),
foreign key (queryId) references queries(queryId)
);
CREATE OR REPLACE FUNCTION setQueryVersionNumber()
RETURNS TRIGGER AS $$
BEGIN
SELECT COALESCE(MAX(versionId), 0) + 1
INTO NEW.versionId
FROM queryVersions
WHERE queryId = NEW.queryId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER setQueryVersionNumberTrigger
BEFORE INSERT ON queryVersions
FOR EACH ROW
EXECUTE FUNCTION setQueryVersionNumber();
CREATE TABLE queryActiveVersions (
activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
versionId int not null,
foreign key (queryId, versionId) references queryVersions(queryId, versionId)
);
CREATE TABLE requiredQueries (
requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
requiredQueryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (requiredQueryId) references queries(queryId),
foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
unique (queryId, requiredQueryId, removedVersion)
);
CREATE TABLE queryConfigs (
configId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
config jsonb not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
unique (queryId, removedVersion)
);
CREATE OR REPLACE FUNCTION removeQueryConfig()
RETURNS TRIGGER AS $$
BEGIN
UPDATE queryConfigs
SET removedVersion = NEW.addedVersion
WHERE queryId = NEW.queryId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER removeQueryConfigTrigger
BEFORE INSERT ON queryConfigs
FOR EACH ROW
EXECUTE FUNCTION removeQueryConfig();
internal/database/migrations/00000000000003_clients.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE clients;
internal/database/migrations/00000000000003_clients.up.sql
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE clients (
clientId varchar(255) primary key,
name TEXT not null,
UNIQUE (name)
);
CREATE TABLE clientCanSync (
syncId uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
canSync boolean not null,
foreign key (clientId) references clients(clientId)
);
internal/database/migrations/00000000000003_collectors.down.sql
-9
View File
@@ -1,9 +0,0 @@
-- Migration 003: Collectors (DOWN)
DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions;
DROP FUNCTION IF EXISTS removeMinCleanVersion;
DROP TABLE IF EXISTS collectorMinCleanVersions;
DROP TABLE IF EXISTS collectorActiveVersions;
DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions;
DROP FUNCTION IF EXISTS setCollectorVersionNumber;
DROP TABLE IF EXISTS collectorVersions;
internal/database/migrations/00000000000003_collectors.up.sql
-62
View File
@@ -1,62 +0,0 @@
-- Migration 003: Collectors (Simplified)
-- Collector versioning and configuration
-- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used)
CREATE TABLE collectorVersions (
clientId varchar(255) NOT NULL,
id int NOT NULL,
addedAt timestamp NOT NULL DEFAULT current_timestamp,
FOREIGN KEY (clientId) REFERENCES clients(clientId),
PRIMARY KEY (id, clientId)
);
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
RETURNS TRIGGER AS $$
BEGIN
SELECT COALESCE(MAX(id), 0) + 1
INTO NEW.id
FROM collectorVersions
WHERE clientId = NEW.clientId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER setCollectorVersionNumberTrigger
BEFORE INSERT ON collectorVersions
FOR EACH ROW
EXECUTE FUNCTION setCollectorVersionNumber();
CREATE TABLE collectorActiveVersions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
clientId varchar(255) NOT NULL,
versionId int NOT NULL,
FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id)
);
CREATE TABLE collectorMinCleanVersions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
clientId varchar(255) NOT NULL,
versionId bigint NOT NULL,
addedVersion int NOT NULL,
removedVersion int,
FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id),
FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id),
FOREIGN KEY (clientId) REFERENCES clients(clientId)
);
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinCleanVersions
SET removedVersion = NEW.addedVersion
WHERE clientId = NEW.clientId AND removedVersion IS NULL;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER removeMinCleanVersionTrigger
BEFORE INSERT ON collectorMinCleanVersions
FOR EACH ROW
EXECUTE FUNCTION removeMinCleanVersion();
internal/database/migrations/00000000000004_batch_uploads.down.sql
-6
View File
@@ -1,6 +0,0 @@
-- Migration 004: Batch Uploads (DOWN)
DROP INDEX IF EXISTS idx_batch_uploads_status;
DROP INDEX IF EXISTS idx_batch_uploads_client_id;
DROP TABLE IF EXISTS batch_uploads;
DROP TYPE IF EXISTS batch_status;
internal/database/migrations/00000000000004_collectors.down.sql
+3
View File
@@ -0,0 +1,3 @@
DROP TABLE collectorQueries;
DROP TABLE collectors;
internal/database/migrations/00000000000004_collectors.up.sql
+99
View File
@@ -0,0 +1,99 @@
CREATE TABLE collectorVersions (
clientId varchar(255) not null,
id int not null,
addedAt timestamp not null default current_timestamp,
foreign key (clientId) references clients(clientId),
primary key (id, clientId)
);
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
RETURNS TRIGGER AS $$
BEGIN
SELECT COALESCE(MAX(id), 0) + 1
INTO NEW.id
FROM collectorVersions
WHERE clientId = NEW.clientId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER setCollectorVersionNumberTrigger
BEFORE INSERT ON collectorVersions
FOR EACH ROW
EXECUTE FUNCTION setCollectorVersionNumber();
CREATE TABLE collectorActiveVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
versionId int not null,
foreign key (clientId, versionId) references collectorVersions(clientId, id)
);
CREATE TABLE collectorMinCleanVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
versionId bigint not null,
addedVersion int not null,
removedVersion int,
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
foreign key (clientId) references clients(clientId)
);
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinCleanVersions
SET removedVersion = NEW.addedVersion
WHERE clientId = NEW.clientId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER removeMinCleanVersionTrigger
BEFORE INSERT ON collectorMinCleanVersions
FOR EACH ROW
EXECUTE FUNCTION removeMinCleanVersion();
CREATE TABLE collectorMinTextVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
versionId bigint not null,
addedVersion int not null,
removedVersion int,
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
foreign key (clientId) references clients(clientId)
);
CREATE OR REPLACE FUNCTION removeMinTextVersion()
RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinTextVersions
SET removedVersion = NEW.addedVersion
WHERE clientId = NEW.clientId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER removeMinTextVersionTrigger
BEFORE INSERT ON collectorMinTextVersions
FOR EACH ROW
EXECUTE FUNCTION removeMinTextVersion();
CREATE TABLE collectorQueries (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
name varchar(255) not null,
queryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (clientId) references clients(clientId),
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
unique (clientId, name, removedVersion)
);
internal/database/migrations/00000000000005_documents.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE documents;
internal/database/migrations/00000000000005_documents.up.sql
+78
View File
@@ -0,0 +1,78 @@
CREATE TABLE documents (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
hash text not null,
foreign key (clientId) references clients(clientId),
unique(clientId, hash)
);
CREATE TABLE documentUploads (
id uuid primary key DEFAULT uuid_generate_v7(),
bucket text not null,
key text not null,
clientId varchar(255) not null,
part unsignedsmallint not null,
createdAt timestamp not null,
foreign key (clientId) references clients(clientId)
);
CREATE TABLE documentEntries (
id uuid primary key DEFAULT uuid_generate_v7(),
documentId uuid not null,
bucket text not null,
key text not null,
foreign key (documentId) references documents(id)
);
CREATE TYPE cleanFailType AS ENUM (
'invalid_mimetype',
'invalid_read',
'invalid_read_pages',
'zero_page_count',
'large_file',
'small_dimensions',
'large_dimensions',
'small_dpi',
'large_dpi'
);
CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
CREATE TABLE documentCleans (
id uuid primary key DEFAULT uuid_generate_v7(),
documentId uuid not null,
bucket text,
key text,
hash text,
mimetype cleanMimeType,
fail cleanFailType,
foreign key (documentId) references documents(id),
CONSTRAINT bucket_and_key_together CHECK ((bucket IS NULL) = (key IS NULL) and (bucket is null) = (mimetype is null) and (bucket is null) = (hash is null)),
CONSTRAINT location_xor_fail CHECK (
(bucket IS NOT NULL) != (fail IS NOT NULL)
)
);
CREATE TABLE documentCleanEntries (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanId uuid not null,
version bigint not null,
foreign key (cleanId) references documentCleans(id)
);
CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanId uuid not null,
bucket text not null,
key text not null,
hash text not null,
createdAt timestamp not null,
part unsignedsmallint not null,
foreign key (cleanId) references documentCleans(id)
);
CREATE TABLE documentTextExtractionEntries (
id uuid primary key DEFAULT uuid_generate_v7(),
textId uuid not null,
version bigint not null,
foreign key (textId) references documentTextExtractions(id)
);
internal/database/migrations/00000000000006_documents.down.sql
-31
View File
@@ -1,31 +0,0 @@
-- Migration 006: Documents (DOWN)
-- Drop document clean entries first (depends on documentCleans)
DROP TABLE IF EXISTS documentCleanEntries;
-- Drop document cleans
DROP TABLE IF EXISTS documentCleans;
-- Drop document entries
DROP TABLE IF EXISTS documentEntries;
-- Drop documentUploads indexes
DROP INDEX IF EXISTS idx_documentuploads_folder_id;
DROP INDEX IF EXISTS idx_documentuploads_clientid;
DROP INDEX IF EXISTS idx_documentuploads_batch_id;
DROP INDEX IF EXISTS idx_documentuploads_filename;
-- Drop documentUploads table
DROP TABLE IF EXISTS documentUploads;
-- Drop documents indexes
DROP INDEX IF EXISTS idx_documents_folderid;
DROP INDEX IF EXISTS idx_documents_filename;
DROP INDEX IF EXISTS idx_documents_batch_id;
-- Drop documents table
DROP TABLE IF EXISTS documents;
-- Drop enums
DROP TYPE IF EXISTS cleanMimeType;
DROP TYPE IF EXISTS cleanFailType;
internal/database/migrations/00000000000006_documents.up.sql
-125
View File
@@ -1,125 +0,0 @@
-- Migration 006: Documents (Consolidated)
-- Document storage and processing tables
-- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118
-- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used)
-- Enum for document clean failure types
CREATE TYPE cleanFailType AS ENUM (
'invalid_mimetype',
'invalid_read',
'invalid_read_pages',
'zero_page_count',
'large_file',
'small_dimensions',
'large_dimensions',
'small_dpi',
'large_dpi'
);
-- Enum for clean document mime types
CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
-- Main documents table with all columns
CREATE TABLE documents (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
clientId varchar(255) NOT NULL,
hash text NOT NULL,
-- Batch reference (from migration 103)
batch_id uuid,
-- Filename (from migration 105)
filename TEXT,
-- Folder organization (from migration 111)
folderId uuid,
originalPath text,
-- File size (from migration 118)
file_size_bytes BIGINT,
FOREIGN KEY (clientId) REFERENCES clients(clientId),
FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
FOREIGN KEY (folderId) REFERENCES folders(id),
UNIQUE(clientId, hash)
);
-- Indexes for documents table
CREATE INDEX idx_documents_batch_id ON documents(batch_id);
CREATE INDEX idx_documents_filename ON documents(filename);
CREATE INDEX idx_documents_folderid ON documents(folderId);
-- Comments for immutable fields (from migration 111)
COMMENT ON COLUMN documents.originalPath IS
'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
-- Document uploads table (staging area for incoming documents)
CREATE TABLE documentUploads (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
bucket text NOT NULL,
key text NOT NULL,
clientId varchar(255) NOT NULL,
part unsignedsmallint NOT NULL,
createdAt timestamp NOT NULL,
-- Filename (from migration 106)
filename TEXT,
-- Batch reference (from migration 107)
batch_id uuid,
-- Folder reference (from migration 116)
folder_id uuid,
FOREIGN KEY (clientId) REFERENCES clients(clientId),
FOREIGN KEY (folder_id) REFERENCES folders(id)
);
-- Indexes for documentUploads
CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
-- Document entries (S3 storage records)
CREATE TABLE documentEntries (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
documentId uuid NOT NULL,
bucket text NOT NULL,
key text NOT NULL,
FOREIGN KEY (documentId) REFERENCES documents(id)
);
-- Comments for immutable fields (from migration 111)
COMMENT ON COLUMN documentEntries.key IS
'S3 object key. IMMUTABLE after creation - never modify this value.';
COMMENT ON COLUMN documentEntries.bucket IS
'S3 bucket name. IMMUTABLE after creation - never modify this value.';
-- Document cleans (processed/cleaned documents)
CREATE TABLE documentCleans (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
documentId uuid NOT NULL,
bucket text,
key text,
hash text,
mimetype cleanMimeType,
fail cleanFailType,
FOREIGN KEY (documentId) REFERENCES documents(id),
CONSTRAINT bucket_and_key_together CHECK (
(bucket IS NULL) = (key IS NULL) AND
(bucket IS NULL) = (mimetype IS NULL) AND
(bucket IS NULL) = (hash IS NULL)
),
CONSTRAINT location_xor_fail CHECK (
(bucket IS NOT NULL) != (fail IS NOT NULL)
)
);
-- Document clean entries (version tracking for cleans)
CREATE TABLE documentCleanEntries (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
cleanId uuid NOT NULL,
version bigint NOT NULL,
FOREIGN KEY (cleanId) REFERENCES documentCleans(id)
);
internal/database/migrations/00000000000006_results.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE results;
internal/database/migrations/00000000000006_results.up.sql
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE results (
id uuid primary key DEFAULT uuid_generate_v7(),
textEntryId uuid not null,
queryId uuid not null,
value TEXT not null,
queryVersion int not null,
foreign key (queryId) references queries(queryId),
foreign key (textEntryId) references documentTextExtractions(id),
foreign key (queryId, queryVersion) references queryVersions(queryId, versionId)
);
CREATE TABLE resultDependencies (
resultId uuid not null,
requiredResultId uuid not null,
foreign key (resultId) references results(id),
foreign key (requiredResultId) references results(id),
CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
);
internal/database/migrations/00000000000008_field_extractions.down.sql
-19
View File
@@ -1,19 +0,0 @@
-- Migration 008: Field Extractions (DOWN)
-- Drop array fields table and indexes
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
-- Drop versions table and indexes
DROP INDEX IF EXISTS idx_unique_version_per_document;
DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
DROP TABLE IF EXISTS documentFieldExtractionVersions;
-- Drop main field extractions table and indexes
DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
DROP TABLE IF EXISTS documentFieldExtractions;
internal/database/migrations/00000000000010_collector_views.down.sql
-6
View File
@@ -1,6 +0,0 @@
-- Migration 010: Collector Views (DOWN)
DROP VIEW IF EXISTS fullActiveCollectors;
DROP VIEW IF EXISTS currentCollectorMinCleanVersions;
DROP VIEW IF EXISTS collectorLatestVersions;
DROP VIEW IF EXISTS collectorCurrentActiveVersions;
internal/database/migrations/00000000000010_collector_views.up.sql
-43
View File
@@ -1,43 +0,0 @@
-- Migration 010: Collector Views (Simplified)
-- Views for collector configuration
-- NOTE: Query-related views and text version views removed (no longer used)
CREATE VIEW collectorCurrentActiveVersions AS
SELECT
c.clientId,
COALESCE(v.versionId, 0)::int AS activeVersion
FROM
clients c
LEFT JOIN (
SELECT DISTINCT ON (clientId)
clientId,
versionId
FROM
collectorActiveVersions
ORDER BY clientId, id DESC
) v ON c.clientId = v.clientId;
CREATE VIEW collectorLatestVersions AS
SELECT
j.clientId,
COALESCE(MAX(v.id), 0)::int AS latestVersion
FROM clients AS j
LEFT JOIN collectorVersions AS v ON v.clientId = j.clientId
GROUP BY j.clientId;
CREATE VIEW currentCollectorMinCleanVersions AS
SELECT DISTINCT
av.clientId,
COALESCE(ctv.versionId, 0) AS minCleanVersion
FROM collectorCurrentActiveVersions AS av
LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId
AND isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
-- Final fullActiveCollectors view (simplified without minTextVersion or fields)
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion,
av.activeVersion, lv.latestVersion
FROM collectorCurrentActiveVersions AS av
JOIN collectorLatestVersions AS lv ON lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
internal/database/migrations/00000000000011_document_views.down.sql
-7
View File
@@ -1,7 +0,0 @@
-- Migration 011: Document Views (DOWN)
DROP FUNCTION IF EXISTS listValidDocumentResults;
DROP VIEW IF EXISTS fullClients;
DROP VIEW IF EXISTS currentClientCanSync;
DROP VIEW IF EXISTS currentCleanEntries;
DROP FUNCTION IF EXISTS listDocumentIDs;
internal/database/migrations/00000000000100_query_views.down.sql
+1
View File
@@ -0,0 +1 @@
DROP VIEW fullActiveQueries;
internal/database/migrations/00000000000100_query_views.up.sql
+71
View File
@@ -0,0 +1,71 @@
CREATE VIEW queryCurrentActiveVersions as
SELECT DISTINCT
q.queryId,
coalesce(
(FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)),
0
)::int as activeVersion
FROM queries AS q
LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId;
CREATE VIEW queryLatestVersions as
SELECT
q.queryId,
coalesce(max(v.versionId), 0)::int as latestVersion
FROM queries AS q
LEFT JOIN queryVersions as v on v.queryId = q.queryId
GROUP BY q.queryId;
CREATE VIEW queryCurrentConfigs as
SELECT av.queryId, c.config
FROM queryCurrentActiveVersions as av
LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
CREATE VIEW queryCurrentRequiredIds as
SELECT DISTINCT av.queryId, r.requiredQueryId
FROM queryCurrentActiveVersions as av
LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
CREATE VIEW queryCurrentRequiredIdsAGG as
SELECT queryId,
coalesce(
ARRAY_AGG(DISTINCT requiredQueryId)
FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM queryCurrentRequiredIds
GROUP BY queryId;
CREATE VIEW fullActiveQueries AS
SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
FROM queries AS q
JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.queryId
JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId
JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId;
CREATE VIEW queryActiveDependencies AS
WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
SELECT
queryId,
requiredQueryId,
ARRAY[queryId, requiredQueryId]::uuid[] AS path,
false AS cycle
FROM queryCurrentRequiredIds
UNION ALL
SELECT
q.queryId,
qd.requiredQueryId,
path || qd.requiredQueryId,
qd.requiredQueryId = ANY(path) AS cycle
FROM queryCurrentRequiredIds as q
JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
WHERE NOT qd.cycle -- Stop if we detect a cycle
)
SELECT DISTINCT queryId as id, requiredQueryId
FROM queryActiveDependencies
WHERE NOT cycle;
internal/database/migrations/00000000000101_collector_views.down.sql
+3
View File
@@ -0,0 +1,3 @@
DROP VIEW fullActiveCollectors;
DROP VIEW collectorQueryDependencyTree;
internal/database/migrations/00000000000101_collector_views.up.sql
+123
View File
@@ -0,0 +1,123 @@
CREATE VIEW collectorCurrentActiveVersions AS
SELECT
c.clientId,
COALESCE(v.versionId, 0)::int AS activeVersion
FROM
clients c
LEFT JOIN (
SELECT DISTINCT ON (clientId)
clientId,
versionId
FROM
collectorActiveVersions
ORDER BY clientId, id DESC
) v ON c.clientId = v.clientId;
CREATE VIEW collectorLatestVersions as
SELECT
j.clientId,
coalesce(max(v.id), 0)::int as latestVersion
FROM clients as j
LEFT JOIN collectorVersions as v on v.clientId = j.clientId
GROUP BY j.clientId;
CREATE VIEW currentCollectorMinTextVersions as
SELECT DISTINCT
av.clientId,
coalesce(ctv.versionId, 0) as minTextVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorMinCleanVersions as
SELECT DISTINCT
av.clientId,
coalesce(ctv.versionId, 0) as minCleanVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
CREATE VIEW currentCollectorQueries as
SELECT DISTINCT
av.clientId,
q.name,
q.queryId
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId
and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
CREATE VIEW currentCollectorQueriesJSONAGG as
SELECT DISTINCT
clientId,
jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
FROM currentCollectorQueries
GROUP BY clientId;
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion,
q.fields
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId;
CREATE VIEW collectorQueryDependencyTree AS
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.clientId, cq.queryId, ri.requiredIds
FROM currentCollectorQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.clientId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.queryId = ct.queryId;
CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient(
_clientId varchar(255)
)
RETURNS TABLE (
clientId varchar(255),
queryId uuid,
queryType queryType,
queryVersion int,
requiredIds uuid[]
) AS $$
BEGIN
RETURN QUERY
WITH clientQueries as (
SELECT q.clientId, q.queryId
FROM currentCollectorQueries as q
WHERE q.clientId = _clientId
),
dependencyTree as (
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.clientId, cq.queryId, ri.requiredIds
FROM clientQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.clientId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.queryId = ct.queryId
)
SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds
FROM dependencyTree as t
WHERE t.clientID is not null;
END;
$$ LANGUAGE plpgsql;
internal/database/migrations/00000000000102_document_views.down.sql
+1
View File
@@ -0,0 +1 @@
DROP VIEW listDocumentIDs;
internal/database/migrations/00000000000011_document_views.up.sql → internal/database/migrations/00000000000102_document_views.up.sql
+40 -19
View File
@@ -1,7 +1,3 @@
-- Migration 011: Document Views (Simplified)
-- Views and functions for document queries
-- NOTE: currentTextEntries and listValidDocumentResults removed (no longer used)
CREATE OR REPLACE FUNCTION listDocumentIDs(
_clientId varchar(255),
_batchSize INTEGER,
@@ -15,7 +11,7 @@ BEGIN
RETURN QUERY
SELECT d.id, totalCount
FROM documents AS d
FROM documents as d
WHERE d.clientId = _clientId
ORDER BY d.id
LIMIT _batchSize
@@ -23,7 +19,7 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
CREATE VIEW currentCleanEntries AS
CREATE VIEW currentCleanEntries as
WITH RankedExtractions AS (
SELECT
dc.id,
@@ -35,7 +31,7 @@ WITH RankedExtractions AS (
dc.hash,
dce.version,
d.clientId,
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) AS row_num
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) as row_num
FROM documents d
JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId
JOIN documentCleans dc ON dc.documentId = d.id
@@ -55,30 +51,55 @@ SELECT
FROM RankedExtractions re
WHERE row_num = 1;
CREATE VIEW currentClientCanSync AS
CREATE VIEW currentTextEntries as
WITH RankedExtractions AS (
SELECT
extract.id,
cc.documentId,
extract.bucket,
extract.key,
extract.hash,
extract.cleanId,
dtee.version as extractionVersion,
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
FROM documents d
JOIN currentCleanEntries cc on cc.documentId = d.id
JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
JOIN documentTextExtractions extract ON extract.cleanId = cc.id
JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
AND dtee.version >= ctv.minTextVersion
)
SELECT
id,
documentId,
bucket,
key,
hash,
cleanId,
extractionVersion
FROM RankedExtractions
WHERE row_num = 1;
CREATE VIEW currentClientCanSync as
WITH RankedExtractions AS (
SELECT
ccs.clientId,
ccs.canSync,
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) AS row_num
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) as row_num
FROM clients c
JOIN clientCanSync ccs ON c.clientId = ccs.clientId
JOIN clientCanSync ccs on c.clientId = ccs.clientId
)
SELECT
c.clientId,
COALESCE(r.canSync, false) AS canSync
coalesce(r.canSync, false) as canSync
FROM clients c
LEFT JOIN RankedExtractions r ON r.clientId = c.clientId AND r.row_num = 1;
LEFT JOIN RankedExtractions r on r.clientId = c.clientId and r.row_num = 1;
CREATE VIEW fullClients AS
CREATE VIEW fullClients as
SELECT c.clientId, c.name, cs.canSync
FROM clients AS c
JOIN currentClientCanSync AS cs ON cs.clientId = c.clientId;
FROM clients as c
JOIN currentClientCanSync as cs on cs.clientId = c.clientId;
-- NOTE: This function exists for schema compatibility with original migrations.
-- It references tables/functions that no longer exist (results, resultDependencies,
-- collectorQueryDependencyTreeByClient, currentTextEntries) and will fail if called.
-- It was created in original migration 102 but never explicitly dropped.
CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid)
RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$
DECLARE
internal/database/migrations/00000000000103_batch_uploads.down.sql
+13
View File
@@ -0,0 +1,13 @@
-- Drop indexes
DROP INDEX IF EXISTS idx_documents_batch_id;
DROP INDEX IF EXISTS idx_batch_uploads_status;
DROP INDEX IF EXISTS idx_batch_uploads_client_id;
-- Remove batch_id from documents table
ALTER TABLE documents DROP COLUMN IF EXISTS batch_id;
-- Drop batch uploads table
DROP TABLE IF EXISTS batch_uploads;
-- Drop batch status ENUM type
DROP TYPE IF EXISTS batch_status;
internal/database/migrations/00000000000004_batch_uploads.up.sql → internal/database/migrations/00000000000103_batch_uploads.up.sql
+9 -17
View File
@@ -1,7 +1,3 @@
-- Migration 004: Batch Uploads
-- Batch upload tracking for ZIP file uploads
-- Merges original migrations 103 + 104
-- Create batch status ENUM type
CREATE TYPE batch_status AS ENUM (
'processing', -- Currently processing
@@ -10,7 +6,7 @@ CREATE TYPE batch_status AS ENUM (
'cancelled' -- Cancelled by user
);
-- Create batch uploads table with all columns
-- Create batch uploads table
CREATE TABLE batch_uploads (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
client_id varchar(255) NOT NULL,
@@ -33,20 +29,16 @@ CREATE TABLE batch_uploads (
created_at timestamp NOT NULL DEFAULT NOW(),
completed_at timestamp,
-- Storage metadata (from migration 104)
archive_bucket text,
archive_key text,
file_size_bytes bigint,
FOREIGN KEY (client_id) REFERENCES clients(clientId),
-- Constraint: storage metadata consistency
CONSTRAINT batch_storage_consistent CHECK (
(archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
(archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
)
FOREIGN KEY (client_id) REFERENCES clients(clientId)
);
-- Create indexes for efficient queries
CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id);
CREATE INDEX idx_batch_uploads_status ON batch_uploads(status);
-- Add batch_id to documents table
ALTER TABLE documents ADD COLUMN batch_id uuid NULL;
ALTER TABLE documents ADD FOREIGN KEY (batch_id) REFERENCES batch_uploads(id);
-- Create index on batch_id for efficient batch document queries
CREATE INDEX idx_documents_batch_id ON documents(batch_id);
internal/database/migrations/00000000000104_add_batch_storage_columns.down.sql
+5
View File
@@ -0,0 +1,5 @@
-- Remove storage columns and constraint
ALTER TABLE batch_uploads DROP CONSTRAINT IF EXISTS batch_storage_required;
ALTER TABLE batch_uploads DROP COLUMN IF EXISTS file_size_bytes;
ALTER TABLE batch_uploads DROP COLUMN IF EXISTS archive_key;
ALTER TABLE batch_uploads DROP COLUMN IF EXISTS archive_bucket;
internal/database/migrations/00000000000104_add_batch_storage_columns.up.sql
+11
View File
@@ -0,0 +1,11 @@
-- Add storage metadata columns to batch_uploads table
ALTER TABLE batch_uploads ADD COLUMN archive_bucket text;
ALTER TABLE batch_uploads ADD COLUMN archive_key text;
ALTER TABLE batch_uploads ADD COLUMN file_size_bytes bigint;
-- Add constraint to ensure storage metadata is consistent (if any storage field is set, bucket and key must both be set)
ALTER TABLE batch_uploads ADD CONSTRAINT batch_storage_consistent
CHECK (
(archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
(archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
);
internal/database/migrations/00000000000105_add_document_filename.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Remove filename column and index
DROP INDEX IF EXISTS idx_documents_filename;
ALTER TABLE documents DROP COLUMN IF EXISTS filename;
internal/database/migrations/00000000000105_add_document_filename.up.sql
+5
View File
@@ -0,0 +1,5 @@
-- Add filename column to documents table to store original filename with path
ALTER TABLE documents ADD COLUMN filename TEXT;
-- Add index for efficient filename searches
CREATE INDEX idx_documents_filename ON documents(filename);
internal/database/migrations/00000000000106_add_documentuploads_filename.down.sql
+2
View File
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS idx_documentuploads_filename;
ALTER TABLE documentUploads DROP COLUMN IF EXISTS filename;
internal/database/migrations/00000000000106_add_documentuploads_filename.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE documentUploads ADD COLUMN filename TEXT;
CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
internal/database/migrations/00000000000107_add_documentuploads_batch_id.down.sql
+2
View File
@@ -0,0 +1,2 @@
DROP INDEX IF EXISTS idx_documentuploads_batch_id;
ALTER TABLE documentUploads DROP COLUMN IF EXISTS batch_id;
internal/database/migrations/00000000000107_add_documentuploads_batch_id.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE documentUploads ADD COLUMN batch_id uuid;
CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
internal/database/migrations/00000000000108_add_documentuploads_clientid_index.down.sql
+1
View File
@@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_documentuploads_clientid;
internal/database/migrations/00000000000108_add_documentuploads_clientid_index.up.sql
+1
View File
@@ -0,0 +1 @@
CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
internal/database/migrations/00000000000005_folders.down.sql → internal/database/migrations/00000000000109_create_folders_table.down.sql
+1 -2
View File
@@ -1,5 +1,4 @@
-- Migration 005: Folders (DOWN)
-- Rollback: Drop folders table and its indexes
DROP INDEX IF EXISTS idx_folders_path;
DROP INDEX IF EXISTS idx_folders_clientid;
DROP INDEX IF EXISTS idx_folders_parentid;
internal/database/migrations/00000000000005_folders.up.sql → internal/database/migrations/00000000000109_create_folders_table.up.sql
+1 -9
View File
@@ -1,6 +1,4 @@
-- Migration 005: Folders
-- Virtual folder hierarchy for document organization
-- Create folders table for hierarchical document organization
CREATE TABLE folders (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
path text NOT NULL,
@@ -18,9 +16,3 @@ CREATE TABLE folders (
CREATE INDEX idx_folders_parentid ON folders(parentId);
CREATE INDEX idx_folders_clientid ON folders(clientId);
CREATE INDEX idx_folders_path ON folders(path);
-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
COMMENT ON TABLE folders IS
'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
COMMENT ON COLUMN folders.path IS
'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
internal/database/migrations/00000000000007_labels.down.sql → internal/database/migrations/00000000000110_create_labels_tables.down.sql
+1 -2
View File
@@ -1,5 +1,4 @@
-- Migration 007: Labels (DOWN)
-- Rollback: Drop documentLabels and labels tables with indexes
DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
DROP INDEX IF EXISTS idx_documentlabels_appliedat;
DROP INDEX IF EXISTS idx_documentlabels_label;
internal/database/migrations/00000000000007_labels.up.sql → internal/database/migrations/00000000000110_create_labels_tables.up.sql
-3
View File
@@ -1,6 +1,3 @@
-- Migration 007: Labels
-- Document labeling system for tracking processing status
-- Create labels lookup table
CREATE TABLE labels (
label text PRIMARY KEY,
internal/database/migrations/00000000000111_add_documents_folder_fields.down.sql
+18
View File
@@ -0,0 +1,18 @@
-- Rollback: Remove folderId, originalPath columns and related constraints/indexes from documents table
-- Remove comments from folders table
COMMENT ON COLUMN folders.path IS NULL;
COMMENT ON TABLE folders IS NULL;
-- Remove comments from documentEntries table
COMMENT ON COLUMN documentEntries.bucket IS NULL;
COMMENT ON COLUMN documentEntries.key IS NULL;
-- Remove originalPath column
COMMENT ON COLUMN documents.originalPath IS NULL;
ALTER TABLE documents DROP COLUMN IF EXISTS originalPath;
-- Remove folderId column
DROP INDEX IF EXISTS idx_documents_folderid;
ALTER TABLE documents DROP CONSTRAINT IF EXISTS fk_documents_folderid;
ALTER TABLE documents DROP COLUMN IF EXISTS folderId;
internal/database/migrations/00000000000111_add_documents_folder_fields.up.sql
+29
View File
@@ -0,0 +1,29 @@
-- Add folderId column to documents table for folder organization
-- Note: filename column already exists from migration 105
ALTER TABLE documents ADD COLUMN folderId uuid;
-- Add foreign key constraint to folders table
ALTER TABLE documents ADD CONSTRAINT fk_documents_folderid
FOREIGN KEY (folderId) REFERENCES folders(id);
-- Add index for efficient folder-based document queries
CREATE INDEX idx_documents_folderid ON documents(folderId);
-- Add originalPath column to store the exact path provided during upload
-- This value is IMMUTABLE after creation - it preserves the original upload path
-- even if the document is later moved to a different virtual folder
ALTER TABLE documents ADD COLUMN originalPath text;
COMMENT ON COLUMN documents.originalPath IS
'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
-- Add comments to clarify immutability of S3 storage fields
COMMENT ON COLUMN documentEntries.key IS
'S3 object key. IMMUTABLE after creation - never modify this value.';
COMMENT ON COLUMN documentEntries.bucket IS
'S3 bucket name. IMMUTABLE after creation - never modify this value.';
-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
COMMENT ON TABLE folders IS
'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
COMMENT ON COLUMN folders.path IS
'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
internal/database/migrations/00000000000112_create_field_extractions_main.down.sql
+5
View File
@@ -0,0 +1,5 @@
-- Rollback: Drop documentFieldExtractions table and its indexes
DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
DROP TABLE IF EXISTS documentFieldExtractions;
internal/database/migrations/00000000000112_create_field_extractions_main.up.sql
+48
View File
@@ -0,0 +1,48 @@
-- Create documentFieldExtractions table for single-value fields
CREATE TABLE documentFieldExtractions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
documentId uuid NOT NULL,
-- Single-value fields (19 fields total - 1:1 relationship)
-- Document identification
fileName text,
contractTitle text,
aareteDerivedAmendmentNum int,
-- Party information
clientName text,
payerName text,
payerState text,
providerState text,
-- Tax identification numbers (stored as text to preserve leading zeros)
filenameTin text,
provGroupTin text,
provGroupNpi text,
provGroupNameFull text,
provOtherTin text,
provOtherNpi text,
provOtherNameFull text,
-- Contract dates
aareteDerivedEffectiveDt date,
aareteDerivedTerminationDt date,
-- Renewal information
autoRenewalInd boolean,
autoRenewalTerm text,
-- Metadata
createdAt timestamp NOT NULL DEFAULT NOW(),
createdBy varchar(255) NOT NULL,
FOREIGN KEY (documentId) REFERENCES documents(id)
);
-- Indexes for efficient queries
CREATE INDEX idx_documentfieldextractions_documentid
ON documentFieldExtractions(documentId);
CREATE INDEX idx_documentfieldextractions_createdby
ON documentFieldExtractions(createdBy);
CREATE INDEX idx_documentfieldextractions_filename
ON documentFieldExtractions(fileName);
internal/database/migrations/00000000000113_create_field_extraction_versions.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Rollback: Drop documentFieldExtractionVersions table and its indexes
DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
DROP TABLE IF EXISTS documentFieldExtractionVersions;
internal/database/migrations/00000000000113_create_field_extraction_versions.up.sql
+16
View File
@@ -0,0 +1,16 @@
-- Create documentFieldExtractionVersions table for version tracking
CREATE TABLE documentFieldExtractionVersions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
fieldExtractionId uuid NOT NULL,
version bigint NOT NULL,
createdBy varchar(255) NOT NULL,
createdAt timestamp NOT NULL DEFAULT NOW(),
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id)
);
-- Indexes for version queries
CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
ON documentFieldExtractionVersions(fieldExtractionId);
CREATE INDEX idx_documentfieldextractionversions_version
ON documentFieldExtractionVersions(version);
internal/database/migrations/00000000000114_create_field_extraction_arrays.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Rollback: Drop documentFieldExtractionArrayFields table and its indexes
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
internal/database/migrations/00000000000008_field_extractions.up.sql → internal/database/migrations/00000000000114_create_field_extraction_arrays.up.sql
-81
View File
@@ -1,84 +1,3 @@
-- Migration 008: Field Extractions (Consolidated)
-- Document field extraction storage for GenAI-extracted data
-- Merges: 112 + 113 + 114 + 117
-- Create documentFieldExtractions table for single-value fields
CREATE TABLE documentFieldExtractions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
documentId uuid NOT NULL,
-- Single-value fields (19 fields total - 1:1 relationship)
-- Document identification
fileName text,
contractTitle text,
aareteDerivedAmendmentNum int,
-- Party information
clientName text,
payerName text,
payerState text,
providerState text,
-- Tax identification numbers (stored as text to preserve leading zeros)
filenameTin text,
provGroupTin text,
provGroupNpi text,
provGroupNameFull text,
provOtherTin text,
provOtherNpi text,
provOtherNameFull text,
-- Contract dates
aareteDerivedEffectiveDt date,
aareteDerivedTerminationDt date,
-- Renewal information
autoRenewalInd boolean,
autoRenewalTerm text,
-- Metadata
createdAt timestamp NOT NULL DEFAULT NOW(),
createdBy varchar(255) NOT NULL,
FOREIGN KEY (documentId) REFERENCES documents(id)
);
-- Indexes for efficient queries
CREATE INDEX idx_documentfieldextractions_documentid
ON documentFieldExtractions(documentId);
CREATE INDEX idx_documentfieldextractions_createdby
ON documentFieldExtractions(createdBy);
CREATE INDEX idx_documentfieldextractions_filename
ON documentFieldExtractions(fileName);
-- Create documentFieldExtractionVersions table for version tracking
-- Includes documentId column from migration 117 for unique constraint
CREATE TABLE documentFieldExtractionVersions (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
fieldExtractionId uuid NOT NULL,
version bigint NOT NULL,
createdBy varchar(255) NOT NULL,
createdAt timestamp NOT NULL DEFAULT NOW(),
-- documentId for unique version constraint (from migration 117)
documentId uuid NOT NULL,
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
FOREIGN KEY (documentId) REFERENCES documents(id)
);
-- Indexes for version queries
CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
ON documentFieldExtractionVersions(fieldExtractionId);
CREATE INDEX idx_documentfieldextractionversions_version
ON documentFieldExtractionVersions(version);
CREATE INDEX idx_documentfieldextractionversions_documentid
ON documentFieldExtractionVersions(documentId);
-- Unique constraint to prevent duplicate versions per document (from migration 117)
CREATE UNIQUE INDEX idx_unique_version_per_document
ON documentFieldExtractionVersions(documentId, version);
-- Create documentFieldExtractionArrayFields table for 1:N array fields
CREATE TABLE documentFieldExtractionArrayFields (
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
internal/database/migrations/00000000000012_field_extraction_views.down.sql → internal/database/migrations/00000000000115_create_field_extraction_views.down.sql
+1 -2
View File
@@ -1,4 +1,3 @@
-- Migration 012: Field Extraction Views (DOWN)
-- Rollback: Drop field extraction views
DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount;
DROP VIEW IF EXISTS currentFieldExtractions;
internal/database/migrations/00000000000012_field_extraction_views.up.sql → internal/database/migrations/00000000000115_create_field_extraction_views.up.sql
+2 -5
View File
@@ -1,6 +1,3 @@
-- Migration 012: Field Extraction Views
-- Views for querying current field extractions
-- Create view for current (newest) field extraction per document
CREATE VIEW currentFieldExtractions AS
SELECT DISTINCT ON (dfe.documentId)
@@ -36,12 +33,12 @@ ORDER BY dfe.documentId, dfev.id DESC;
CREATE VIEW currentFieldExtractionsWithArrayCount AS
SELECT
cfe.*,
COALESCE(array_counts.arraySize, 0) AS arraySize
COALESCE(array_counts.arraySize, 0) as arraySize
FROM currentFieldExtractions cfe
LEFT JOIN (
SELECT
fieldExtractionId,
COUNT(*) AS arraySize
COUNT(*) as arraySize
FROM documentFieldExtractionArrayFields
GROUP BY fieldExtractionId
) array_counts ON array_counts.fieldExtractionId = cfe.id;
internal/database/migrations/00000000000116_add_documentuploads_folder_id.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Remove folder_id column from documentUploads table
DROP INDEX IF EXISTS idx_documentuploads_folder_id;
ALTER TABLE documentUploads DROP CONSTRAINT IF EXISTS fk_documentuploads_folder_id;
ALTER TABLE documentUploads DROP COLUMN IF EXISTS folder_id;
internal/database/migrations/00000000000116_add_documentuploads_folder_id.up.sql
+6
View File
@@ -0,0 +1,6 @@
-- Add folder_id column to documentUploads table to persist folder association during batch uploads
ALTER TABLE documentUploads ADD COLUMN folder_id uuid;
-- Add foreign key constraint referencing folders table
ALTER TABLE documentUploads ADD CONSTRAINT fk_documentuploads_folder_id FOREIGN KEY (folder_id) REFERENCES folders(id);
-- Add index for efficient lookup by folder
CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
internal/database/migrations/00000000000117_add_version_unique_constraint.down.sql
+10
View File
@@ -0,0 +1,10 @@
-- Remove the unique constraint and documentId column
DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
DROP INDEX IF EXISTS idx_unique_version_per_document;
ALTER TABLE documentFieldExtractionVersions
DROP CONSTRAINT IF EXISTS fk_documentfieldextractionversions_documentid;
ALTER TABLE documentFieldExtractionVersions
DROP COLUMN IF EXISTS documentId;
internal/database/migrations/00000000000117_add_version_unique_constraint.up.sql
+30
View File
@@ -0,0 +1,30 @@
-- Add unique constraint to prevent duplicate versions per document
-- This ensures that concurrent requests cannot create the same version number
-- First, add documentId column to versions table for easier constraint enforcement
ALTER TABLE documentFieldExtractionVersions
ADD COLUMN documentId uuid;
-- Populate documentId from the joined extraction record
UPDATE documentFieldExtractionVersions dfev
SET documentId = dfe.documentId
FROM documentFieldExtractions dfe
WHERE dfev.fieldExtractionId = dfe.id;
-- Make documentId NOT NULL after populating
ALTER TABLE documentFieldExtractionVersions
ALTER COLUMN documentId SET NOT NULL;
-- Add foreign key constraint (documents table has 'id' as primary key)
ALTER TABLE documentFieldExtractionVersions
ADD CONSTRAINT fk_documentfieldextractionversions_documentid
FOREIGN KEY (documentId) REFERENCES documents(id);
-- Add unique constraint on documentId + version
-- This prevents duplicate version numbers for the same document
CREATE UNIQUE INDEX idx_unique_version_per_document
ON documentFieldExtractionVersions(documentId, version);
-- Add index for efficient lookups by documentId
CREATE INDEX idx_documentfieldextractionversions_documentid
ON documentFieldExtractionVersions(documentId);
internal/database/migrations/00000000000118_add_document_file_size.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Remove file_size_bytes column from documents table
ALTER TABLE documents DROP COLUMN file_size_bytes;
internal/database/migrations/00000000000118_add_document_file_size.up.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add file_size_bytes column to documents table
-- Stores the size of the document in bytes as measured from S3 HeadObject
-- NULL for legacy documents (before this feature was added)
ALTER TABLE documents ADD COLUMN file_size_bytes BIGINT;
internal/database/migrations/00000000000119_remove_queries.down.sql
+287
View File
@@ -0,0 +1,287 @@
-- Down migration to restore query functionality
-- This recreates all query-related tables, views, and functions
-- Drop the simplified fullActiveCollectors view
DROP VIEW IF EXISTS fullActiveCollectors;
-- Recreate query type enum
CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor');
-- Recreate queries table
CREATE TABLE queries (
queryId uuid primary key DEFAULT uuid_generate_v7(),
queryType queryType not null
);
-- Recreate queryVersions table
CREATE TABLE queryVersions (
queryId uuid not null,
versionId int not null,
addedAt timestamp not null default current_timestamp,
primary key (versionId, queryId),
foreign key (queryId) references queries(queryId)
);
-- Recreate version number function and trigger
CREATE OR REPLACE FUNCTION setQueryVersionNumber()
RETURNS TRIGGER AS $$
BEGIN
SELECT COALESCE(MAX(versionId), 0) + 1
INTO NEW.versionId
FROM queryVersions
WHERE queryId = NEW.queryId;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER setQueryVersionNumberTrigger
BEFORE INSERT ON queryVersions
FOR EACH ROW
EXECUTE FUNCTION setQueryVersionNumber();
-- Recreate queryActiveVersions table
CREATE TABLE queryActiveVersions (
activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
versionId int not null,
foreign key (queryId, versionId) references queryVersions(queryId, versionId)
);
-- Recreate requiredQueries table
CREATE TABLE requiredQueries (
requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
requiredQueryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (requiredQueryId) references queries(queryId),
foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
unique (queryId, requiredQueryId, removedVersion)
);
-- Recreate queryConfigs table
CREATE TABLE queryConfigs (
configId uuid primary key DEFAULT uuid_generate_v7(),
queryId uuid not null,
config jsonb not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (queryId, addedVersion) references queryVersions(queryId, versionId),
foreign key (queryId, removedVersion) references queryVersions(queryId, versionId),
unique (queryId, removedVersion)
);
-- Recreate removeQueryConfig function and trigger
CREATE OR REPLACE FUNCTION removeQueryConfig()
RETURNS TRIGGER AS $$
BEGIN
UPDATE queryConfigs
SET removedVersion = NEW.addedVersion
WHERE queryId = NEW.queryId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER removeQueryConfigTrigger
BEFORE INSERT ON queryConfigs
FOR EACH ROW
EXECUTE FUNCTION removeQueryConfig();
-- Recreate collectorQueries table
CREATE TABLE collectorQueries (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
name varchar(255) not null,
queryId uuid not null,
addedVersion int not null,
removedVersion int,
foreign key (queryId) references queries(queryId),
foreign key (clientId) references clients(clientId),
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
unique (clientId, name, removedVersion)
);
-- Recreate results table
CREATE TABLE results (
id uuid primary key DEFAULT uuid_generate_v7(),
textEntryId uuid not null,
queryId uuid not null,
value TEXT not null,
queryVersion int not null,
foreign key (queryId) references queries(queryId),
foreign key (textEntryId) references documentTextExtractions(id),
foreign key (queryId, queryVersion) references queryVersions(queryId, versionId)
);
-- Recreate resultDependencies table
CREATE TABLE resultDependencies (
resultId uuid not null,
requiredResultId uuid not null,
foreign key (resultId) references results(id),
foreign key (requiredResultId) references results(id),
CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId)
);
-- Recreate query views
CREATE VIEW queryCurrentActiveVersions as
SELECT DISTINCT
q.queryId,
coalesce(
(FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)),
0
)::int as activeVersion
FROM queries AS q
LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId;
CREATE VIEW queryLatestVersions as
SELECT
q.queryId,
coalesce(max(v.versionId), 0)::int as latestVersion
FROM queries AS q
LEFT JOIN queryVersions as v on v.queryId = q.queryId
GROUP BY q.queryId;
CREATE VIEW queryCurrentConfigs as
SELECT av.queryId, c.config
FROM queryCurrentActiveVersions as av
LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId
and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion);
CREATE VIEW queryCurrentRequiredIds as
SELECT DISTINCT av.queryId, r.requiredQueryId
FROM queryCurrentActiveVersions as av
LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId
and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion);
CREATE VIEW queryCurrentRequiredIdsAGG as
SELECT queryId,
coalesce(
ARRAY_AGG(DISTINCT requiredQueryId)
FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[],
array[]::uuid[]
)::uuid[] as requiredIds
FROM queryCurrentRequiredIds
GROUP BY queryId;
CREATE VIEW fullActiveQueries AS
SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds
FROM queries AS q
JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId
JOIN queryLatestVersions as lv on lv.queryId = q.queryId
JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId
JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId;
CREATE VIEW queryActiveDependencies AS
WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS (
SELECT
queryId,
requiredQueryId,
ARRAY[queryId, requiredQueryId]::uuid[] AS path,
false AS cycle
FROM queryCurrentRequiredIds
UNION ALL
SELECT
q.queryId,
qd.requiredQueryId,
path || qd.requiredQueryId,
qd.requiredQueryId = ANY(path) AS cycle
FROM queryCurrentRequiredIds as q
JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId
WHERE NOT qd.cycle
)
SELECT DISTINCT queryId as id, requiredQueryId
FROM queryActiveDependencies
WHERE NOT cycle;
-- Recreate collector views that depend on queries
CREATE VIEW currentCollectorQueries as
SELECT DISTINCT
av.clientId,
q.name,
q.queryId
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId
and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion);
CREATE VIEW currentCollectorQueriesJSONAGG as
SELECT DISTINCT
clientId,
jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields
FROM currentCollectorQueries
GROUP BY clientId;
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion,
q.fields
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId;
CREATE VIEW collectorQueryDependencyTree AS
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.clientId, cq.queryId, ri.requiredIds
FROM currentCollectorQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.clientId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.queryId = ct.queryId;
CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient(
_clientId varchar(255)
)
RETURNS TABLE (
clientId varchar(255),
queryId uuid,
queryType queryType,
queryVersion int,
requiredIds uuid[]
) AS $$
BEGIN
RETURN QUERY
WITH clientQueries as (
SELECT q.clientId, q.queryId
FROM currentCollectorQueries as q
WHERE q.clientId = _clientId
),
dependencyTree as (
WITH RECURSIVE collectorQueryDependencyTree AS (
SELECT cq.clientId, cq.queryId, ri.requiredIds
FROM clientQueries as cq
JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId
UNION ALL
SELECT acq.clientId, q.queryId, q.requiredIds
FROM queryCurrentRequiredIdsAGG as q
JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds)
)
SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds
FROM collectorQueryDependencyTree as ct
JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId
JOIN queries as q on q.queryId = ct.queryId
)
SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds
FROM dependencyTree as t
WHERE t.clientID is not null;
END;
$$ LANGUAGE plpgsql;
internal/database/migrations/00000000000119_remove_queries.up.sql
+53
View File
@@ -0,0 +1,53 @@
-- Migration to remove query functionality from the database
-- This drops all query-related tables, views, and functions
-- First drop the function that depends on query views
DROP FUNCTION IF EXISTS collectorQueryDependencyTreeByClient;
-- Drop the collector views that depend on query tables (in order of dependencies)
DROP VIEW IF EXISTS collectorQueryDependencyTree;
DROP VIEW IF EXISTS fullActiveCollectors;
DROP VIEW IF EXISTS currentCollectorQueriesJSONAGG;
DROP VIEW IF EXISTS currentCollectorQueries;
-- Drop query views (in order of dependencies)
DROP VIEW IF EXISTS fullActiveQueries;
DROP VIEW IF EXISTS queryActiveDependencies;
DROP VIEW IF EXISTS queryCurrentRequiredIdsAGG;
DROP VIEW IF EXISTS queryCurrentRequiredIds;
DROP VIEW IF EXISTS queryCurrentConfigs;
DROP VIEW IF EXISTS queryLatestVersions;
DROP VIEW IF EXISTS queryCurrentActiveVersions;
-- Drop result tables
DROP TABLE IF EXISTS resultDependencies;
DROP TABLE IF EXISTS results;
-- Drop collectorQueries table (links collectors to queries)
DROP TABLE IF EXISTS collectorQueries;
-- Drop query configuration and dependency tables
DROP TABLE IF EXISTS queryConfigs;
DROP TABLE IF EXISTS requiredQueries;
DROP TABLE IF EXISTS queryActiveVersions;
DROP TABLE IF EXISTS queryVersions;
DROP TABLE IF EXISTS queries;
-- Drop triggers and functions
DROP TRIGGER IF EXISTS setQueryVersionNumberTrigger ON queryVersions;
DROP FUNCTION IF EXISTS setQueryVersionNumber;
DROP TRIGGER IF EXISTS removeQueryConfigTrigger ON queryConfigs;
DROP FUNCTION IF EXISTS removeQueryConfig;
-- Drop the query type enum
DROP TYPE IF EXISTS queryType;
-- Recreate the fullActiveCollectors view without the fields column
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId;
internal/database/migrations/00000000000119_stub.down.sql
-2
View File
@@ -1,2 +0,0 @@
-- Stub migration for version compatibility
-- No actual changes needed.
internal/database/migrations/00000000000119_stub.up.sql
-4
View File
@@ -1,4 +0,0 @@
-- Stub migration for version compatibility
-- This file exists to satisfy the migrate tool for databases that were
-- previously at version 119 (before migration collapse).
-- No actual changes - the schema is already correct.
internal/database/migrations/00000000000120_remove_text_extraction.down.sql
+104
View File
@@ -0,0 +1,104 @@
-- Rollback migration to restore text extraction functionality
-- NOTE: This recreates text extraction tables and views
-- First drop the simplified fullActiveCollectors view
DROP VIEW IF EXISTS fullActiveCollectors;
-- Recreate documentTextExtractions table
CREATE TABLE documentTextExtractions (
id uuid primary key DEFAULT uuid_generate_v7(),
cleanId uuid not null,
bucket text not null,
key text not null,
hash text not null,
createdAt timestamp not null,
part unsignedsmallint not null,
foreign key (cleanId) references documentCleans(id)
);
-- Recreate documentTextExtractionEntries table
CREATE TABLE documentTextExtractionEntries (
id uuid primary key DEFAULT uuid_generate_v7(),
textId uuid not null,
version bigint not null,
foreign key (textId) references documentTextExtractions(id)
);
-- Recreate collectorMinTextVersions table
CREATE TABLE collectorMinTextVersions (
id uuid primary key DEFAULT uuid_generate_v7(),
clientId varchar(255) not null,
versionId bigint not null,
addedVersion int not null,
removedVersion int,
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
foreign key (clientId) references clients(clientId)
);
-- Recreate removeMinTextVersion function
CREATE OR REPLACE FUNCTION removeMinTextVersion()
RETURNS TRIGGER AS $$
BEGIN
UPDATE collectorMinTextVersions
SET removedVersion = NEW.addedVersion
WHERE clientId = NEW.clientId and removedVersion is null;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- Recreate trigger for collectorMinTextVersions
CREATE TRIGGER removeMinTextVersionTrigger
BEFORE INSERT ON collectorMinTextVersions
FOR EACH ROW
EXECUTE FUNCTION removeMinTextVersion();
-- Recreate currentCollectorMinTextVersions view
CREATE VIEW currentCollectorMinTextVersions as
SELECT DISTINCT
av.clientId,
coalesce(ctv.versionId, 0) as minTextVersion
FROM collectorCurrentActiveVersions as av
LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
-- Recreate fullActiveCollectors view with minTextVersion
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion, ctv.minTextVersion,
av.activeVersion, lv.latestVersion
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId;
-- Recreate currentTextEntries view
CREATE VIEW currentTextEntries as
WITH RankedExtractions AS (
SELECT
extract.id,
cc.documentId,
extract.bucket,
extract.key,
extract.hash,
extract.cleanId,
dtee.version as extractionVersion,
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
FROM documents d
JOIN currentCleanEntries cc on cc.documentId = d.id
JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
JOIN documentTextExtractions extract ON extract.cleanId = cc.id
JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
AND dtee.version >= ctv.minTextVersion
)
SELECT
id,
documentId,
bucket,
key,
hash,
cleanId,
extractionVersion
FROM RankedExtractions
WHERE row_num = 1;
internal/database/migrations/00000000000120_remove_text_extraction.up.sql
+32
View File
@@ -0,0 +1,32 @@
-- Migration to remove text extraction functionality from the database
-- This drops all text extraction related tables, views, and functions
-- NOTE: Field extractions are separate and NOT affected by this migration
-- First drop the view that depends on text extraction tables
DROP VIEW IF EXISTS currentTextEntries;
-- Drop fullActiveCollectors view that depends on currentCollectorMinTextVersions
DROP VIEW IF EXISTS fullActiveCollectors;
-- Drop the currentCollectorMinTextVersions view
DROP VIEW IF EXISTS currentCollectorMinTextVersions;
-- Drop the trigger and function for collectorMinTextVersions
DROP TRIGGER IF EXISTS removeMinTextVersionTrigger ON collectorMinTextVersions;
DROP FUNCTION IF EXISTS removeMinTextVersion;
-- Drop the collectorMinTextVersions table
DROP TABLE IF EXISTS collectorMinTextVersions;
-- Drop the text extraction tables (child first for FK constraints)
DROP TABLE IF EXISTS documentTextExtractionEntries;
DROP TABLE IF EXISTS documentTextExtractions;
-- Recreate fullActiveCollectors view WITHOUT minTextVersion
CREATE VIEW fullActiveCollectors AS
SELECT DISTINCT av.clientId,
ccv.minCleanVersion,
av.activeVersion, lv.latestVersion
FROM collectorCurrentActiveVersions as av
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
internal/database/migrations/00000000000120_stub.down.sql
-2
View File
@@ -1,2 +0,0 @@
-- Stub migration for version compatibility
-- No actual changes needed.
internal/database/migrations/00000000000120_stub.up.sql
-4
View File
@@ -1,4 +0,0 @@
-- Stub migration for version compatibility
-- This file exists to satisfy the migrate tool for databases that were
-- previously at version 120 (before migration collapse).
-- No actual changes - the schema is already correct.