diff --git a/internal/database/migrations/00000000000001_extensions.down.sql b/internal/database/migrations/00000000000001_extensions.down.sql index 2f39878e..d3b7d69f 100644 --- a/internal/database/migrations/00000000000001_extensions.down.sql +++ b/internal/database/migrations/00000000000001_extensions.down.sql @@ -1,7 +1 @@ --- Migration 001: Extensions and Core Functions (DOWN) --- Drops in reverse order of creation - -DROP DOMAIN IF EXISTS unsignedsmallint; -DROP FUNCTION IF EXISTS isInVersion; -DROP FUNCTION IF EXISTS uuid_generate_v7; -DROP EXTENSION IF EXISTS "pgcrypto"; +DROP EXTENSION "pgcrypto"; \ No newline at end of file diff --git a/internal/database/migrations/00000000000001_extensions.up.sql b/internal/database/migrations/00000000000001_extensions.up.sql index 5a072aa3..25ac191b 100644 --- a/internal/database/migrations/00000000000001_extensions.up.sql +++ b/internal/database/migrations/00000000000001_extensions.up.sql @@ -1,13 +1,9 @@ --- Migration 001: Extensions and Core Functions --- Creates PostgreSQL extensions and utility functions used throughout the schema - CREATE EXTENSION IF NOT EXISTS "pgcrypto"; --- UUID v7 generator function (time-ordered UUIDs) -CREATE OR REPLACE FUNCTION uuid_generate_v7() -RETURNS uuid -AS $$ -SELECT encode( +create or replace function uuid_generate_v7() +returns uuid +as $$ +select encode( set_bit( set_bit( overlay(uuid_send(gen_random_uuid()) @@ -20,24 +16,22 @@ SELECT encode( ), 'hex')::uuid; $$ -LANGUAGE SQL -VOLATILE; +language SQL +volatile; --- Version checking function for temporal data patterns CREATE FUNCTION isInVersion( _version int, _addedVersion int, _removedVersion int ) -RETURNS boolean AS $$ +RETURNS boolean as $$ BEGIN - RETURN _version IS NULL OR ( + RETURN _version is null OR ( _version >= _addedVersion AND - (_removedVersion IS NULL OR _version < _removedVersion) + (_removedVersion is null OR _version < _removedVersion) ); END; $$ LANGUAGE plpgsql; --- Domain for unsigned small integers CREATE DOMAIN unsignedsmallint AS SMALLINT CHECK (VALUE >= 0); diff --git a/internal/database/migrations/00000000000002_clients.down.sql b/internal/database/migrations/00000000000002_clients.down.sql deleted file mode 100644 index fa6970a3..00000000 --- a/internal/database/migrations/00000000000002_clients.down.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Migration 002: Clients (DOWN) - -DROP TABLE IF EXISTS clientCanSync; -DROP TABLE IF EXISTS clients; diff --git a/internal/database/migrations/00000000000002_clients.up.sql b/internal/database/migrations/00000000000002_clients.up.sql deleted file mode 100644 index ca31f5b8..00000000 --- a/internal/database/migrations/00000000000002_clients.up.sql +++ /dev/null @@ -1,15 +0,0 @@ --- Migration 002: Clients --- Core client tables for customer configuration - -CREATE TABLE clients ( - clientId varchar(255) PRIMARY KEY, - name TEXT NOT NULL, - UNIQUE (name) -); - -CREATE TABLE clientCanSync ( - syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - clientId varchar(255) NOT NULL, - canSync boolean NOT NULL, - FOREIGN KEY (clientId) REFERENCES clients(clientId) -); diff --git a/internal/database/migrations/00000000000002_queries.down.sql b/internal/database/migrations/00000000000002_queries.down.sql new file mode 100644 index 00000000..60000287 --- /dev/null +++ b/internal/database/migrations/00000000000002_queries.down.sql @@ -0,0 +1,7 @@ +DROP TABLE queries; + +DROP TYPE queryType; + +DROP TABLE requiredQueries; + +DROP TABLE queryConfigs; \ No newline at end of file diff --git a/internal/database/migrations/00000000000002_queries.up.sql b/internal/database/migrations/00000000000002_queries.up.sql new file mode 100644 index 00000000..b28878a4 --- /dev/null +++ b/internal/database/migrations/00000000000002_queries.up.sql @@ -0,0 +1,79 @@ +CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor'); + +CREATE TABLE queries ( + queryId uuid primary key DEFAULT uuid_generate_v7(), + queryType queryType not null +); + +CREATE TABLE queryVersions ( + queryId uuid not null, + versionId int not null, + addedAt timestamp not null default current_timestamp, + primary key (versionId, queryId), + foreign key (queryId) references queries(queryId) +); + +CREATE OR REPLACE FUNCTION setQueryVersionNumber() +RETURNS TRIGGER AS $$ +BEGIN + SELECT COALESCE(MAX(versionId), 0) + 1 + INTO NEW.versionId + FROM queryVersions + WHERE queryId = NEW.queryId; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER setQueryVersionNumberTrigger +BEFORE INSERT ON queryVersions +FOR EACH ROW +EXECUTE FUNCTION setQueryVersionNumber(); + +CREATE TABLE queryActiveVersions ( + activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + versionId int not null, + foreign key (queryId, versionId) references queryVersions(queryId, versionId) +); + +CREATE TABLE requiredQueries ( + requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + requiredQueryId uuid not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (requiredQueryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), + unique (queryId, requiredQueryId, removedVersion) +); + +CREATE TABLE queryConfigs ( + configId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + config jsonb not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), + unique (queryId, removedVersion) +); + +CREATE OR REPLACE FUNCTION removeQueryConfig() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE queryConfigs + SET removedVersion = NEW.addedVersion + WHERE queryId = NEW.queryId and removedVersion is null; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER removeQueryConfigTrigger +BEFORE INSERT ON queryConfigs +FOR EACH ROW +EXECUTE FUNCTION removeQueryConfig(); diff --git a/internal/database/migrations/00000000000003_clients.down.sql b/internal/database/migrations/00000000000003_clients.down.sql new file mode 100644 index 00000000..05c08227 --- /dev/null +++ b/internal/database/migrations/00000000000003_clients.down.sql @@ -0,0 +1 @@ +DROP TABLE clients; \ No newline at end of file diff --git a/internal/database/migrations/00000000000003_clients.up.sql b/internal/database/migrations/00000000000003_clients.up.sql new file mode 100644 index 00000000..8525ef2d --- /dev/null +++ b/internal/database/migrations/00000000000003_clients.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE clients ( + clientId varchar(255) primary key, + name TEXT not null, + UNIQUE (name) +); + +CREATE TABLE clientCanSync ( + syncId uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + canSync boolean not null, + foreign key (clientId) references clients(clientId) +); diff --git a/internal/database/migrations/00000000000003_collectors.down.sql b/internal/database/migrations/00000000000003_collectors.down.sql deleted file mode 100644 index ed04f2a9..00000000 --- a/internal/database/migrations/00000000000003_collectors.down.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Migration 003: Collectors (DOWN) - -DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions; -DROP FUNCTION IF EXISTS removeMinCleanVersion; -DROP TABLE IF EXISTS collectorMinCleanVersions; -DROP TABLE IF EXISTS collectorActiveVersions; -DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions; -DROP FUNCTION IF EXISTS setCollectorVersionNumber; -DROP TABLE IF EXISTS collectorVersions; diff --git a/internal/database/migrations/00000000000003_collectors.up.sql b/internal/database/migrations/00000000000003_collectors.up.sql deleted file mode 100644 index 70afe051..00000000 --- a/internal/database/migrations/00000000000003_collectors.up.sql +++ /dev/null @@ -1,62 +0,0 @@ --- Migration 003: Collectors (Simplified) --- Collector versioning and configuration --- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used) - -CREATE TABLE collectorVersions ( - clientId varchar(255) NOT NULL, - id int NOT NULL, - addedAt timestamp NOT NULL DEFAULT current_timestamp, - FOREIGN KEY (clientId) REFERENCES clients(clientId), - PRIMARY KEY (id, clientId) -); - -CREATE OR REPLACE FUNCTION setCollectorVersionNumber() -RETURNS TRIGGER AS $$ -BEGIN - SELECT COALESCE(MAX(id), 0) + 1 - INTO NEW.id - FROM collectorVersions - WHERE clientId = NEW.clientId; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER setCollectorVersionNumberTrigger -BEFORE INSERT ON collectorVersions -FOR EACH ROW -EXECUTE FUNCTION setCollectorVersionNumber(); - -CREATE TABLE collectorActiveVersions ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - clientId varchar(255) NOT NULL, - versionId int NOT NULL, - FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id) -); - -CREATE TABLE collectorMinCleanVersions ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - clientId varchar(255) NOT NULL, - versionId bigint NOT NULL, - addedVersion int NOT NULL, - removedVersion int, - FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id), - FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id), - FOREIGN KEY (clientId) REFERENCES clients(clientId) -); - -CREATE OR REPLACE FUNCTION removeMinCleanVersion() -RETURNS TRIGGER AS $$ -BEGIN - UPDATE collectorMinCleanVersions - SET removedVersion = NEW.addedVersion - WHERE clientId = NEW.clientId AND removedVersion IS NULL; - - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - -CREATE TRIGGER removeMinCleanVersionTrigger -BEFORE INSERT ON collectorMinCleanVersions -FOR EACH ROW -EXECUTE FUNCTION removeMinCleanVersion(); diff --git a/internal/database/migrations/00000000000004_batch_uploads.down.sql b/internal/database/migrations/00000000000004_batch_uploads.down.sql deleted file mode 100644 index e5ce8815..00000000 --- a/internal/database/migrations/00000000000004_batch_uploads.down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Migration 004: Batch Uploads (DOWN) - -DROP INDEX IF EXISTS idx_batch_uploads_status; -DROP INDEX IF EXISTS idx_batch_uploads_client_id; -DROP TABLE IF EXISTS batch_uploads; -DROP TYPE IF EXISTS batch_status; diff --git a/internal/database/migrations/00000000000004_collectors.down.sql b/internal/database/migrations/00000000000004_collectors.down.sql new file mode 100644 index 00000000..02fe466c --- /dev/null +++ b/internal/database/migrations/00000000000004_collectors.down.sql @@ -0,0 +1,3 @@ +DROP TABLE collectorQueries; + +DROP TABLE collectors; diff --git a/internal/database/migrations/00000000000004_collectors.up.sql b/internal/database/migrations/00000000000004_collectors.up.sql new file mode 100644 index 00000000..60695d90 --- /dev/null +++ b/internal/database/migrations/00000000000004_collectors.up.sql @@ -0,0 +1,99 @@ +CREATE TABLE collectorVersions ( + clientId varchar(255) not null, + id int not null, + addedAt timestamp not null default current_timestamp, + foreign key (clientId) references clients(clientId), + primary key (id, clientId) +); + +CREATE OR REPLACE FUNCTION setCollectorVersionNumber() +RETURNS TRIGGER AS $$ +BEGIN + SELECT COALESCE(MAX(id), 0) + 1 + INTO NEW.id + FROM collectorVersions + WHERE clientId = NEW.clientId; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER setCollectorVersionNumberTrigger +BEFORE INSERT ON collectorVersions +FOR EACH ROW +EXECUTE FUNCTION setCollectorVersionNumber(); + +CREATE TABLE collectorActiveVersions ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + versionId int not null, + foreign key (clientId, versionId) references collectorVersions(clientId, id) +); + +CREATE TABLE collectorMinCleanVersions ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + versionId bigint not null, + addedVersion int not null, + removedVersion int, + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + foreign key (clientId) references clients(clientId) +); + +CREATE OR REPLACE FUNCTION removeMinCleanVersion() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE collectorMinCleanVersions + SET removedVersion = NEW.addedVersion + WHERE clientId = NEW.clientId and removedVersion is null; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER removeMinCleanVersionTrigger +BEFORE INSERT ON collectorMinCleanVersions +FOR EACH ROW +EXECUTE FUNCTION removeMinCleanVersion(); + +CREATE TABLE collectorMinTextVersions ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + versionId bigint not null, + addedVersion int not null, + removedVersion int, + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + foreign key (clientId) references clients(clientId) +); + +CREATE OR REPLACE FUNCTION removeMinTextVersion() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE collectorMinTextVersions + SET removedVersion = NEW.addedVersion + WHERE clientId = NEW.clientId and removedVersion is null; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER removeMinTextVersionTrigger +BEFORE INSERT ON collectorMinTextVersions +FOR EACH ROW +EXECUTE FUNCTION removeMinTextVersion(); + +CREATE TABLE collectorQueries ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + name varchar(255) not null, + queryId uuid not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (clientId) references clients(clientId), + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + unique (clientId, name, removedVersion) +); diff --git a/internal/database/migrations/00000000000005_documents.down.sql b/internal/database/migrations/00000000000005_documents.down.sql new file mode 100644 index 00000000..b51b6a91 --- /dev/null +++ b/internal/database/migrations/00000000000005_documents.down.sql @@ -0,0 +1 @@ +DROP TABLE documents; \ No newline at end of file diff --git a/internal/database/migrations/00000000000005_documents.up.sql b/internal/database/migrations/00000000000005_documents.up.sql new file mode 100644 index 00000000..9d9ef069 --- /dev/null +++ b/internal/database/migrations/00000000000005_documents.up.sql @@ -0,0 +1,78 @@ +CREATE TABLE documents ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + hash text not null, + foreign key (clientId) references clients(clientId), + unique(clientId, hash) +); + +CREATE TABLE documentUploads ( + id uuid primary key DEFAULT uuid_generate_v7(), + bucket text not null, + key text not null, + clientId varchar(255) not null, + part unsignedsmallint not null, + createdAt timestamp not null, + foreign key (clientId) references clients(clientId) +); + +CREATE TABLE documentEntries ( + id uuid primary key DEFAULT uuid_generate_v7(), + documentId uuid not null, + bucket text not null, + key text not null, + foreign key (documentId) references documents(id) +); + +CREATE TYPE cleanFailType AS ENUM ( + 'invalid_mimetype', + 'invalid_read', + 'invalid_read_pages', + 'zero_page_count', + 'large_file', + 'small_dimensions', + 'large_dimensions', + 'small_dpi', + 'large_dpi' +); +CREATE TYPE cleanMimeType AS ENUM ('application/pdf'); + +CREATE TABLE documentCleans ( + id uuid primary key DEFAULT uuid_generate_v7(), + documentId uuid not null, + bucket text, + key text, + hash text, + mimetype cleanMimeType, + fail cleanFailType, + foreign key (documentId) references documents(id), + CONSTRAINT bucket_and_key_together CHECK ((bucket IS NULL) = (key IS NULL) and (bucket is null) = (mimetype is null) and (bucket is null) = (hash is null)), + CONSTRAINT location_xor_fail CHECK ( + (bucket IS NOT NULL) != (fail IS NOT NULL) + ) +); + +CREATE TABLE documentCleanEntries ( + id uuid primary key DEFAULT uuid_generate_v7(), + cleanId uuid not null, + version bigint not null, + foreign key (cleanId) references documentCleans(id) +); + +CREATE TABLE documentTextExtractions ( + id uuid primary key DEFAULT uuid_generate_v7(), + cleanId uuid not null, + bucket text not null, + key text not null, + hash text not null, + createdAt timestamp not null, + part unsignedsmallint not null, + foreign key (cleanId) references documentCleans(id) +); + +CREATE TABLE documentTextExtractionEntries ( + id uuid primary key DEFAULT uuid_generate_v7(), + textId uuid not null, + version bigint not null, + foreign key (textId) references documentTextExtractions(id) +); diff --git a/internal/database/migrations/00000000000006_documents.down.sql b/internal/database/migrations/00000000000006_documents.down.sql deleted file mode 100644 index a75f3001..00000000 --- a/internal/database/migrations/00000000000006_documents.down.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Migration 006: Documents (DOWN) - --- Drop document clean entries first (depends on documentCleans) -DROP TABLE IF EXISTS documentCleanEntries; - --- Drop document cleans -DROP TABLE IF EXISTS documentCleans; - --- Drop document entries -DROP TABLE IF EXISTS documentEntries; - --- Drop documentUploads indexes -DROP INDEX IF EXISTS idx_documentuploads_folder_id; -DROP INDEX IF EXISTS idx_documentuploads_clientid; -DROP INDEX IF EXISTS idx_documentuploads_batch_id; -DROP INDEX IF EXISTS idx_documentuploads_filename; - --- Drop documentUploads table -DROP TABLE IF EXISTS documentUploads; - --- Drop documents indexes -DROP INDEX IF EXISTS idx_documents_folderid; -DROP INDEX IF EXISTS idx_documents_filename; -DROP INDEX IF EXISTS idx_documents_batch_id; - --- Drop documents table -DROP TABLE IF EXISTS documents; - --- Drop enums -DROP TYPE IF EXISTS cleanMimeType; -DROP TYPE IF EXISTS cleanFailType; diff --git a/internal/database/migrations/00000000000006_documents.up.sql b/internal/database/migrations/00000000000006_documents.up.sql deleted file mode 100644 index 7dbafb12..00000000 --- a/internal/database/migrations/00000000000006_documents.up.sql +++ /dev/null @@ -1,125 +0,0 @@ --- Migration 006: Documents (Consolidated) --- Document storage and processing tables --- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118 --- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used) - --- Enum for document clean failure types -CREATE TYPE cleanFailType AS ENUM ( - 'invalid_mimetype', - 'invalid_read', - 'invalid_read_pages', - 'zero_page_count', - 'large_file', - 'small_dimensions', - 'large_dimensions', - 'small_dpi', - 'large_dpi' -); - --- Enum for clean document mime types -CREATE TYPE cleanMimeType AS ENUM ('application/pdf'); - --- Main documents table with all columns -CREATE TABLE documents ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - clientId varchar(255) NOT NULL, - hash text NOT NULL, - - -- Batch reference (from migration 103) - batch_id uuid, - - -- Filename (from migration 105) - filename TEXT, - - -- Folder organization (from migration 111) - folderId uuid, - originalPath text, - - -- File size (from migration 118) - file_size_bytes BIGINT, - - FOREIGN KEY (clientId) REFERENCES clients(clientId), - FOREIGN KEY (batch_id) REFERENCES batch_uploads(id), - FOREIGN KEY (folderId) REFERENCES folders(id), - UNIQUE(clientId, hash) -); - --- Indexes for documents table -CREATE INDEX idx_documents_batch_id ON documents(batch_id); -CREATE INDEX idx_documents_filename ON documents(filename); -CREATE INDEX idx_documents_folderid ON documents(folderId); - --- Comments for immutable fields (from migration 111) -COMMENT ON COLUMN documents.originalPath IS - 'Original path provided during upload. IMMUTABLE after creation - never modify this value.'; - --- Document uploads table (staging area for incoming documents) -CREATE TABLE documentUploads ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - bucket text NOT NULL, - key text NOT NULL, - clientId varchar(255) NOT NULL, - part unsignedsmallint NOT NULL, - createdAt timestamp NOT NULL, - - -- Filename (from migration 106) - filename TEXT, - - -- Batch reference (from migration 107) - batch_id uuid, - - -- Folder reference (from migration 116) - folder_id uuid, - - FOREIGN KEY (clientId) REFERENCES clients(clientId), - FOREIGN KEY (folder_id) REFERENCES folders(id) -); - --- Indexes for documentUploads -CREATE INDEX idx_documentuploads_filename ON documentUploads(filename); -CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id); -CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId); -CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id); - --- Document entries (S3 storage records) -CREATE TABLE documentEntries ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - documentId uuid NOT NULL, - bucket text NOT NULL, - key text NOT NULL, - FOREIGN KEY (documentId) REFERENCES documents(id) -); - --- Comments for immutable fields (from migration 111) -COMMENT ON COLUMN documentEntries.key IS - 'S3 object key. IMMUTABLE after creation - never modify this value.'; -COMMENT ON COLUMN documentEntries.bucket IS - 'S3 bucket name. IMMUTABLE after creation - never modify this value.'; - --- Document cleans (processed/cleaned documents) -CREATE TABLE documentCleans ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - documentId uuid NOT NULL, - bucket text, - key text, - hash text, - mimetype cleanMimeType, - fail cleanFailType, - FOREIGN KEY (documentId) REFERENCES documents(id), - CONSTRAINT bucket_and_key_together CHECK ( - (bucket IS NULL) = (key IS NULL) AND - (bucket IS NULL) = (mimetype IS NULL) AND - (bucket IS NULL) = (hash IS NULL) - ), - CONSTRAINT location_xor_fail CHECK ( - (bucket IS NOT NULL) != (fail IS NOT NULL) - ) -); - --- Document clean entries (version tracking for cleans) -CREATE TABLE documentCleanEntries ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - cleanId uuid NOT NULL, - version bigint NOT NULL, - FOREIGN KEY (cleanId) REFERENCES documentCleans(id) -); diff --git a/internal/database/migrations/00000000000006_results.down.sql b/internal/database/migrations/00000000000006_results.down.sql new file mode 100644 index 00000000..9ceb26f1 --- /dev/null +++ b/internal/database/migrations/00000000000006_results.down.sql @@ -0,0 +1 @@ +DROP TABLE results; \ No newline at end of file diff --git a/internal/database/migrations/00000000000006_results.up.sql b/internal/database/migrations/00000000000006_results.up.sql new file mode 100644 index 00000000..e27a128a --- /dev/null +++ b/internal/database/migrations/00000000000006_results.up.sql @@ -0,0 +1,18 @@ +CREATE TABLE results ( + id uuid primary key DEFAULT uuid_generate_v7(), + textEntryId uuid not null, + queryId uuid not null, + value TEXT not null, + queryVersion int not null, + foreign key (queryId) references queries(queryId), + foreign key (textEntryId) references documentTextExtractions(id), + foreign key (queryId, queryVersion) references queryVersions(queryId, versionId) +); + +CREATE TABLE resultDependencies ( + resultId uuid not null, + requiredResultId uuid not null, + foreign key (resultId) references results(id), + foreign key (requiredResultId) references results(id), + CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId) +); diff --git a/internal/database/migrations/00000000000008_field_extractions.down.sql b/internal/database/migrations/00000000000008_field_extractions.down.sql deleted file mode 100644 index d48c95da..00000000 --- a/internal/database/migrations/00000000000008_field_extractions.down.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Migration 008: Field Extractions (DOWN) - --- Drop array fields table and indexes -DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index; -DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid; -DROP TABLE IF EXISTS documentFieldExtractionArrayFields; - --- Drop versions table and indexes -DROP INDEX IF EXISTS idx_unique_version_per_document; -DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid; -DROP INDEX IF EXISTS idx_documentfieldextractionversions_version; -DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid; -DROP TABLE IF EXISTS documentFieldExtractionVersions; - --- Drop main field extractions table and indexes -DROP INDEX IF EXISTS idx_documentfieldextractions_filename; -DROP INDEX IF EXISTS idx_documentfieldextractions_createdby; -DROP INDEX IF EXISTS idx_documentfieldextractions_documentid; -DROP TABLE IF EXISTS documentFieldExtractions; diff --git a/internal/database/migrations/00000000000010_collector_views.down.sql b/internal/database/migrations/00000000000010_collector_views.down.sql deleted file mode 100644 index 3d47b9f8..00000000 --- a/internal/database/migrations/00000000000010_collector_views.down.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Migration 010: Collector Views (DOWN) - -DROP VIEW IF EXISTS fullActiveCollectors; -DROP VIEW IF EXISTS currentCollectorMinCleanVersions; -DROP VIEW IF EXISTS collectorLatestVersions; -DROP VIEW IF EXISTS collectorCurrentActiveVersions; diff --git a/internal/database/migrations/00000000000010_collector_views.up.sql b/internal/database/migrations/00000000000010_collector_views.up.sql deleted file mode 100644 index 58c2a6db..00000000 --- a/internal/database/migrations/00000000000010_collector_views.up.sql +++ /dev/null @@ -1,43 +0,0 @@ --- Migration 010: Collector Views (Simplified) --- Views for collector configuration --- NOTE: Query-related views and text version views removed (no longer used) - -CREATE VIEW collectorCurrentActiveVersions AS -SELECT - c.clientId, - COALESCE(v.versionId, 0)::int AS activeVersion -FROM - clients c -LEFT JOIN ( - SELECT DISTINCT ON (clientId) - clientId, - versionId - FROM - collectorActiveVersions - ORDER BY clientId, id DESC -) v ON c.clientId = v.clientId; - -CREATE VIEW collectorLatestVersions AS - SELECT - j.clientId, - COALESCE(MAX(v.id), 0)::int AS latestVersion - FROM clients AS j - LEFT JOIN collectorVersions AS v ON v.clientId = j.clientId - GROUP BY j.clientId; - -CREATE VIEW currentCollectorMinCleanVersions AS -SELECT DISTINCT - av.clientId, - COALESCE(ctv.versionId, 0) AS minCleanVersion - FROM collectorCurrentActiveVersions AS av - LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId - AND isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); - --- Final fullActiveCollectors view (simplified without minTextVersion or fields) -CREATE VIEW fullActiveCollectors AS - SELECT DISTINCT av.clientId, - ccv.minCleanVersion, - av.activeVersion, lv.latestVersion - FROM collectorCurrentActiveVersions AS av - JOIN collectorLatestVersions AS lv ON lv.clientId = av.clientId - JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId; diff --git a/internal/database/migrations/00000000000011_document_views.down.sql b/internal/database/migrations/00000000000011_document_views.down.sql deleted file mode 100644 index c08aec93..00000000 --- a/internal/database/migrations/00000000000011_document_views.down.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Migration 011: Document Views (DOWN) - -DROP FUNCTION IF EXISTS listValidDocumentResults; -DROP VIEW IF EXISTS fullClients; -DROP VIEW IF EXISTS currentClientCanSync; -DROP VIEW IF EXISTS currentCleanEntries; -DROP FUNCTION IF EXISTS listDocumentIDs; diff --git a/internal/database/migrations/00000000000100_query_views.down.sql b/internal/database/migrations/00000000000100_query_views.down.sql new file mode 100644 index 00000000..cf880abe --- /dev/null +++ b/internal/database/migrations/00000000000100_query_views.down.sql @@ -0,0 +1 @@ +DROP VIEW fullActiveQueries; diff --git a/internal/database/migrations/00000000000100_query_views.up.sql b/internal/database/migrations/00000000000100_query_views.up.sql new file mode 100644 index 00000000..3cbbb367 --- /dev/null +++ b/internal/database/migrations/00000000000100_query_views.up.sql @@ -0,0 +1,71 @@ +CREATE VIEW queryCurrentActiveVersions as +SELECT DISTINCT + q.queryId, + coalesce( + (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)), + 0 + )::int as activeVersion + FROM queries AS q + LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId; + +CREATE VIEW queryLatestVersions as + SELECT + q.queryId, + coalesce(max(v.versionId), 0)::int as latestVersion + FROM queries AS q + LEFT JOIN queryVersions as v on v.queryId = q.queryId + GROUP BY q.queryId; + +CREATE VIEW queryCurrentConfigs as + SELECT av.queryId, c.config + FROM queryCurrentActiveVersions as av + LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId + and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion); + +CREATE VIEW queryCurrentRequiredIds as + SELECT DISTINCT av.queryId, r.requiredQueryId + FROM queryCurrentActiveVersions as av + LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId + and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion); + +CREATE VIEW queryCurrentRequiredIdsAGG as + SELECT queryId, + coalesce( + ARRAY_AGG(DISTINCT requiredQueryId) + FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], + array[]::uuid[] + )::uuid[] as requiredIds + FROM queryCurrentRequiredIds + GROUP BY queryId; + +CREATE VIEW fullActiveQueries AS +SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds + FROM queries AS q + JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.queryId + JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId + JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId; + +CREATE VIEW queryActiveDependencies AS +WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS ( + SELECT + queryId, + requiredQueryId, + ARRAY[queryId, requiredQueryId]::uuid[] AS path, + false AS cycle + FROM queryCurrentRequiredIds + + UNION ALL + + SELECT + q.queryId, + qd.requiredQueryId, + path || qd.requiredQueryId, + qd.requiredQueryId = ANY(path) AS cycle + FROM queryCurrentRequiredIds as q + JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId + WHERE NOT qd.cycle -- Stop if we detect a cycle + ) + SELECT DISTINCT queryId as id, requiredQueryId + FROM queryActiveDependencies + WHERE NOT cycle; diff --git a/internal/database/migrations/00000000000101_collector_views.down.sql b/internal/database/migrations/00000000000101_collector_views.down.sql new file mode 100644 index 00000000..8553c48b --- /dev/null +++ b/internal/database/migrations/00000000000101_collector_views.down.sql @@ -0,0 +1,3 @@ +DROP VIEW fullActiveCollectors; + +DROP VIEW collectorQueryDependencyTree; \ No newline at end of file diff --git a/internal/database/migrations/00000000000101_collector_views.up.sql b/internal/database/migrations/00000000000101_collector_views.up.sql new file mode 100644 index 00000000..adb3b265 --- /dev/null +++ b/internal/database/migrations/00000000000101_collector_views.up.sql @@ -0,0 +1,123 @@ +CREATE VIEW collectorCurrentActiveVersions AS +SELECT + c.clientId, + COALESCE(v.versionId, 0)::int AS activeVersion +FROM + clients c +LEFT JOIN ( + SELECT DISTINCT ON (clientId) + clientId, + versionId + FROM + collectorActiveVersions + ORDER BY clientId, id DESC +) v ON c.clientId = v.clientId; + +CREATE VIEW collectorLatestVersions as + SELECT + j.clientId, + coalesce(max(v.id), 0)::int as latestVersion + FROM clients as j + LEFT JOIN collectorVersions as v on v.clientId = j.clientId + GROUP BY j.clientId; + +CREATE VIEW currentCollectorMinTextVersions as +SELECT DISTINCT + av.clientId, + coalesce(ctv.versionId, 0) as minTextVersion + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId + and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); + +CREATE VIEW currentCollectorMinCleanVersions as +SELECT DISTINCT + av.clientId, + coalesce(ctv.versionId, 0) as minCleanVersion + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId + and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); + +CREATE VIEW currentCollectorQueries as +SELECT DISTINCT + av.clientId, + q.name, + q.queryId + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId + and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion); + +CREATE VIEW currentCollectorQueriesJSONAGG as +SELECT DISTINCT + clientId, + jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields + FROM currentCollectorQueries + GROUP BY clientId; + +CREATE VIEW fullActiveCollectors AS + SELECT DISTINCT av.clientId, + ccv.minCleanVersion, ctv.minTextVersion, + av.activeVersion, lv.latestVersion, + q.fields + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId + JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId + JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId; + +CREATE VIEW collectorQueryDependencyTree AS + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.clientId, cq.queryId, ri.requiredIds + FROM currentCollectorQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId + + UNION ALL + + SELECT acq.clientId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) + ) + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.queryId = ct.queryId; + + +CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient( + _clientId varchar(255) +) +RETURNS TABLE ( + clientId varchar(255), + queryId uuid, + queryType queryType, + queryVersion int, + requiredIds uuid[] +) AS $$ +BEGIN + RETURN QUERY + WITH clientQueries as ( + SELECT q.clientId, q.queryId + FROM currentCollectorQueries as q + WHERE q.clientId = _clientId + ), + dependencyTree as ( + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.clientId, cq.queryId, ri.requiredIds + FROM clientQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId + + UNION ALL + + SELECT acq.clientId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) + ) + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.queryId = ct.queryId + ) + SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds + FROM dependencyTree as t + WHERE t.clientID is not null; +END; +$$ LANGUAGE plpgsql; diff --git a/internal/database/migrations/00000000000102_document_views.down.sql b/internal/database/migrations/00000000000102_document_views.down.sql new file mode 100644 index 00000000..6b386842 --- /dev/null +++ b/internal/database/migrations/00000000000102_document_views.down.sql @@ -0,0 +1 @@ +DROP VIEW listDocumentIDs; diff --git a/internal/database/migrations/00000000000011_document_views.up.sql b/internal/database/migrations/00000000000102_document_views.up.sql similarity index 80% rename from internal/database/migrations/00000000000011_document_views.up.sql rename to internal/database/migrations/00000000000102_document_views.up.sql index 905f3175..59b627f8 100644 --- a/internal/database/migrations/00000000000011_document_views.up.sql +++ b/internal/database/migrations/00000000000102_document_views.up.sql @@ -1,7 +1,3 @@ --- Migration 011: Document Views (Simplified) --- Views and functions for document queries --- NOTE: currentTextEntries and listValidDocumentResults removed (no longer used) - CREATE OR REPLACE FUNCTION listDocumentIDs( _clientId varchar(255), _batchSize INTEGER, @@ -15,7 +11,7 @@ BEGIN RETURN QUERY SELECT d.id, totalCount - FROM documents AS d + FROM documents as d WHERE d.clientId = _clientId ORDER BY d.id LIMIT _batchSize @@ -23,7 +19,7 @@ BEGIN END; $$ LANGUAGE plpgsql; -CREATE VIEW currentCleanEntries AS +CREATE VIEW currentCleanEntries as WITH RankedExtractions AS ( SELECT dc.id, @@ -35,7 +31,7 @@ WITH RankedExtractions AS ( dc.hash, dce.version, d.clientId, - ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) AS row_num + ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) as row_num FROM documents d JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId JOIN documentCleans dc ON dc.documentId = d.id @@ -55,30 +51,55 @@ SELECT FROM RankedExtractions re WHERE row_num = 1; -CREATE VIEW currentClientCanSync AS +CREATE VIEW currentTextEntries as +WITH RankedExtractions AS ( + SELECT + extract.id, + cc.documentId, + extract.bucket, + extract.key, + extract.hash, + extract.cleanId, + dtee.version as extractionVersion, + ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num + FROM documents d + JOIN currentCleanEntries cc on cc.documentId = d.id + JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId + JOIN documentTextExtractions extract ON extract.cleanId = cc.id + JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id + AND dtee.version >= ctv.minTextVersion +) +SELECT + id, + documentId, + bucket, + key, + hash, + cleanId, + extractionVersion +FROM RankedExtractions +WHERE row_num = 1; + +CREATE VIEW currentClientCanSync as WITH RankedExtractions AS ( SELECT ccs.clientId, ccs.canSync, - ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) AS row_num + ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) as row_num FROM clients c - JOIN clientCanSync ccs ON c.clientId = ccs.clientId + JOIN clientCanSync ccs on c.clientId = ccs.clientId ) SELECT c.clientId, - COALESCE(r.canSync, false) AS canSync + coalesce(r.canSync, false) as canSync FROM clients c -LEFT JOIN RankedExtractions r ON r.clientId = c.clientId AND r.row_num = 1; +LEFT JOIN RankedExtractions r on r.clientId = c.clientId and r.row_num = 1; -CREATE VIEW fullClients AS +CREATE VIEW fullClients as SELECT c.clientId, c.name, cs.canSync - FROM clients AS c - JOIN currentClientCanSync AS cs ON cs.clientId = c.clientId; + FROM clients as c + JOIN currentClientCanSync as cs on cs.clientId = c.clientId; --- NOTE: This function exists for schema compatibility with original migrations. --- It references tables/functions that no longer exist (results, resultDependencies, --- collectorQueryDependencyTreeByClient, currentTextEntries) and will fail if called. --- It was created in original migration 102 but never explicitly dropped. CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid) RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$ DECLARE diff --git a/internal/database/migrations/00000000000103_batch_uploads.down.sql b/internal/database/migrations/00000000000103_batch_uploads.down.sql new file mode 100644 index 00000000..cf6ffe6b --- /dev/null +++ b/internal/database/migrations/00000000000103_batch_uploads.down.sql @@ -0,0 +1,13 @@ +-- Drop indexes +DROP INDEX IF EXISTS idx_documents_batch_id; +DROP INDEX IF EXISTS idx_batch_uploads_status; +DROP INDEX IF EXISTS idx_batch_uploads_client_id; + +-- Remove batch_id from documents table +ALTER TABLE documents DROP COLUMN IF EXISTS batch_id; + +-- Drop batch uploads table +DROP TABLE IF EXISTS batch_uploads; + +-- Drop batch status ENUM type +DROP TYPE IF EXISTS batch_status; \ No newline at end of file diff --git a/internal/database/migrations/00000000000004_batch_uploads.up.sql b/internal/database/migrations/00000000000103_batch_uploads.up.sql similarity index 66% rename from internal/database/migrations/00000000000004_batch_uploads.up.sql rename to internal/database/migrations/00000000000103_batch_uploads.up.sql index f8ecd6ca..cbbfc352 100644 --- a/internal/database/migrations/00000000000004_batch_uploads.up.sql +++ b/internal/database/migrations/00000000000103_batch_uploads.up.sql @@ -1,7 +1,3 @@ --- Migration 004: Batch Uploads --- Batch upload tracking for ZIP file uploads --- Merges original migrations 103 + 104 - -- Create batch status ENUM type CREATE TYPE batch_status AS ENUM ( 'processing', -- Currently processing @@ -10,43 +6,39 @@ CREATE TYPE batch_status AS ENUM ( 'cancelled' -- Cancelled by user ); --- Create batch uploads table with all columns +-- Create batch uploads table CREATE TABLE batch_uploads ( id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), client_id varchar(255) NOT NULL, original_filename text NOT NULL, - + -- Essential batch metrics total_documents integer NOT NULL, processed_documents integer NOT NULL DEFAULT 0, failed_documents integer NOT NULL DEFAULT 0, invalid_type_documents integer NOT NULL DEFAULT 0, - + -- Status and progress status batch_status NOT NULL DEFAULT 'processing', progress_percent integer NOT NULL DEFAULT 0, - + -- Failed document filenames (JSON array) failed_filenames jsonb DEFAULT '[]'::jsonb, - + -- Timestamps created_at timestamp NOT NULL DEFAULT NOW(), completed_at timestamp, - - -- Storage metadata (from migration 104) - archive_bucket text, - archive_key text, - file_size_bytes bigint, - - FOREIGN KEY (client_id) REFERENCES clients(clientId), - - -- Constraint: storage metadata consistency - CONSTRAINT batch_storage_consistent CHECK ( - (archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR - (archive_bucket IS NOT NULL AND archive_key IS NOT NULL) - ) + + FOREIGN KEY (client_id) REFERENCES clients(clientId) ); -- Create indexes for efficient queries CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id); CREATE INDEX idx_batch_uploads_status ON batch_uploads(status); + +-- Add batch_id to documents table +ALTER TABLE documents ADD COLUMN batch_id uuid NULL; +ALTER TABLE documents ADD FOREIGN KEY (batch_id) REFERENCES batch_uploads(id); + +-- Create index on batch_id for efficient batch document queries +CREATE INDEX idx_documents_batch_id ON documents(batch_id); \ No newline at end of file diff --git a/internal/database/migrations/00000000000104_add_batch_storage_columns.down.sql b/internal/database/migrations/00000000000104_add_batch_storage_columns.down.sql new file mode 100644 index 00000000..8126eccd --- /dev/null +++ b/internal/database/migrations/00000000000104_add_batch_storage_columns.down.sql @@ -0,0 +1,5 @@ +-- Remove storage columns and constraint +ALTER TABLE batch_uploads DROP CONSTRAINT IF EXISTS batch_storage_required; +ALTER TABLE batch_uploads DROP COLUMN IF EXISTS file_size_bytes; +ALTER TABLE batch_uploads DROP COLUMN IF EXISTS archive_key; +ALTER TABLE batch_uploads DROP COLUMN IF EXISTS archive_bucket; \ No newline at end of file diff --git a/internal/database/migrations/00000000000104_add_batch_storage_columns.up.sql b/internal/database/migrations/00000000000104_add_batch_storage_columns.up.sql new file mode 100644 index 00000000..02acef97 --- /dev/null +++ b/internal/database/migrations/00000000000104_add_batch_storage_columns.up.sql @@ -0,0 +1,11 @@ +-- Add storage metadata columns to batch_uploads table +ALTER TABLE batch_uploads ADD COLUMN archive_bucket text; +ALTER TABLE batch_uploads ADD COLUMN archive_key text; +ALTER TABLE batch_uploads ADD COLUMN file_size_bytes bigint; + +-- Add constraint to ensure storage metadata is consistent (if any storage field is set, bucket and key must both be set) +ALTER TABLE batch_uploads ADD CONSTRAINT batch_storage_consistent +CHECK ( + (archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR + (archive_bucket IS NOT NULL AND archive_key IS NOT NULL) +); \ No newline at end of file diff --git a/internal/database/migrations/00000000000105_add_document_filename.down.sql b/internal/database/migrations/00000000000105_add_document_filename.down.sql new file mode 100644 index 00000000..db8c3593 --- /dev/null +++ b/internal/database/migrations/00000000000105_add_document_filename.down.sql @@ -0,0 +1,3 @@ +-- Remove filename column and index +DROP INDEX IF EXISTS idx_documents_filename; +ALTER TABLE documents DROP COLUMN IF EXISTS filename; \ No newline at end of file diff --git a/internal/database/migrations/00000000000105_add_document_filename.up.sql b/internal/database/migrations/00000000000105_add_document_filename.up.sql new file mode 100644 index 00000000..248a4fc1 --- /dev/null +++ b/internal/database/migrations/00000000000105_add_document_filename.up.sql @@ -0,0 +1,5 @@ +-- Add filename column to documents table to store original filename with path +ALTER TABLE documents ADD COLUMN filename TEXT; + +-- Add index for efficient filename searches +CREATE INDEX idx_documents_filename ON documents(filename); \ No newline at end of file diff --git a/internal/database/migrations/00000000000106_add_documentuploads_filename.down.sql b/internal/database/migrations/00000000000106_add_documentuploads_filename.down.sql new file mode 100644 index 00000000..07223a4a --- /dev/null +++ b/internal/database/migrations/00000000000106_add_documentuploads_filename.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_documentuploads_filename; +ALTER TABLE documentUploads DROP COLUMN IF EXISTS filename; \ No newline at end of file diff --git a/internal/database/migrations/00000000000106_add_documentuploads_filename.up.sql b/internal/database/migrations/00000000000106_add_documentuploads_filename.up.sql new file mode 100644 index 00000000..939f831d --- /dev/null +++ b/internal/database/migrations/00000000000106_add_documentuploads_filename.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE documentUploads ADD COLUMN filename TEXT; +CREATE INDEX idx_documentuploads_filename ON documentUploads(filename); \ No newline at end of file diff --git a/internal/database/migrations/00000000000107_add_documentuploads_batch_id.down.sql b/internal/database/migrations/00000000000107_add_documentuploads_batch_id.down.sql new file mode 100644 index 00000000..36bec50d --- /dev/null +++ b/internal/database/migrations/00000000000107_add_documentuploads_batch_id.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_documentuploads_batch_id; +ALTER TABLE documentUploads DROP COLUMN IF EXISTS batch_id; \ No newline at end of file diff --git a/internal/database/migrations/00000000000107_add_documentuploads_batch_id.up.sql b/internal/database/migrations/00000000000107_add_documentuploads_batch_id.up.sql new file mode 100644 index 00000000..a5f5b66e --- /dev/null +++ b/internal/database/migrations/00000000000107_add_documentuploads_batch_id.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE documentUploads ADD COLUMN batch_id uuid; +CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id); \ No newline at end of file diff --git a/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.down.sql b/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.down.sql new file mode 100644 index 00000000..d0413062 --- /dev/null +++ b/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_documentuploads_clientid; \ No newline at end of file diff --git a/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.up.sql b/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.up.sql new file mode 100644 index 00000000..e16cc76e --- /dev/null +++ b/internal/database/migrations/00000000000108_add_documentuploads_clientid_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId); \ No newline at end of file diff --git a/internal/database/migrations/00000000000005_folders.down.sql b/internal/database/migrations/00000000000109_create_folders_table.down.sql similarity index 76% rename from internal/database/migrations/00000000000005_folders.down.sql rename to internal/database/migrations/00000000000109_create_folders_table.down.sql index 692ca138..c9e00134 100644 --- a/internal/database/migrations/00000000000005_folders.down.sql +++ b/internal/database/migrations/00000000000109_create_folders_table.down.sql @@ -1,5 +1,4 @@ --- Migration 005: Folders (DOWN) - +-- Rollback: Drop folders table and its indexes DROP INDEX IF EXISTS idx_folders_path; DROP INDEX IF EXISTS idx_folders_clientid; DROP INDEX IF EXISTS idx_folders_parentid; diff --git a/internal/database/migrations/00000000000005_folders.up.sql b/internal/database/migrations/00000000000109_create_folders_table.up.sql similarity index 57% rename from internal/database/migrations/00000000000005_folders.up.sql rename to internal/database/migrations/00000000000109_create_folders_table.up.sql index 98b092d0..e8171a2f 100644 --- a/internal/database/migrations/00000000000005_folders.up.sql +++ b/internal/database/migrations/00000000000109_create_folders_table.up.sql @@ -1,6 +1,4 @@ --- Migration 005: Folders --- Virtual folder hierarchy for document organization - +-- Create folders table for hierarchical document organization CREATE TABLE folders ( id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), path text NOT NULL, @@ -18,9 +16,3 @@ CREATE TABLE folders ( CREATE INDEX idx_folders_parentid ON folders(parentId); CREATE INDEX idx_folders_clientid ON folders(clientId); CREATE INDEX idx_folders_path ON folders(path); - --- Add comment to clarify that folders are virtual (database-only, no S3 relationship) -COMMENT ON TABLE folders IS - 'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.'; -COMMENT ON COLUMN folders.path IS - 'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.'; diff --git a/internal/database/migrations/00000000000007_labels.down.sql b/internal/database/migrations/00000000000110_create_labels_tables.down.sql similarity index 81% rename from internal/database/migrations/00000000000007_labels.down.sql rename to internal/database/migrations/00000000000110_create_labels_tables.down.sql index 4ac89d9c..b586e505 100644 --- a/internal/database/migrations/00000000000007_labels.down.sql +++ b/internal/database/migrations/00000000000110_create_labels_tables.down.sql @@ -1,5 +1,4 @@ --- Migration 007: Labels (DOWN) - +-- Rollback: Drop documentLabels and labels tables with indexes DROP INDEX IF EXISTS idx_documentlabels_document_label_time; DROP INDEX IF EXISTS idx_documentlabels_appliedat; DROP INDEX IF EXISTS idx_documentlabels_label; diff --git a/internal/database/migrations/00000000000007_labels.up.sql b/internal/database/migrations/00000000000110_create_labels_tables.up.sql similarity index 93% rename from internal/database/migrations/00000000000007_labels.up.sql rename to internal/database/migrations/00000000000110_create_labels_tables.up.sql index c80e3e4e..cd1da1f5 100644 --- a/internal/database/migrations/00000000000007_labels.up.sql +++ b/internal/database/migrations/00000000000110_create_labels_tables.up.sql @@ -1,6 +1,3 @@ --- Migration 007: Labels --- Document labeling system for tracking processing status - -- Create labels lookup table CREATE TABLE labels ( label text PRIMARY KEY, diff --git a/internal/database/migrations/00000000000111_add_documents_folder_fields.down.sql b/internal/database/migrations/00000000000111_add_documents_folder_fields.down.sql new file mode 100644 index 00000000..46bc1308 --- /dev/null +++ b/internal/database/migrations/00000000000111_add_documents_folder_fields.down.sql @@ -0,0 +1,18 @@ +-- Rollback: Remove folderId, originalPath columns and related constraints/indexes from documents table + +-- Remove comments from folders table +COMMENT ON COLUMN folders.path IS NULL; +COMMENT ON TABLE folders IS NULL; + +-- Remove comments from documentEntries table +COMMENT ON COLUMN documentEntries.bucket IS NULL; +COMMENT ON COLUMN documentEntries.key IS NULL; + +-- Remove originalPath column +COMMENT ON COLUMN documents.originalPath IS NULL; +ALTER TABLE documents DROP COLUMN IF EXISTS originalPath; + +-- Remove folderId column +DROP INDEX IF EXISTS idx_documents_folderid; +ALTER TABLE documents DROP CONSTRAINT IF EXISTS fk_documents_folderid; +ALTER TABLE documents DROP COLUMN IF EXISTS folderId; diff --git a/internal/database/migrations/00000000000111_add_documents_folder_fields.up.sql b/internal/database/migrations/00000000000111_add_documents_folder_fields.up.sql new file mode 100644 index 00000000..eed58aca --- /dev/null +++ b/internal/database/migrations/00000000000111_add_documents_folder_fields.up.sql @@ -0,0 +1,29 @@ +-- Add folderId column to documents table for folder organization +-- Note: filename column already exists from migration 105 +ALTER TABLE documents ADD COLUMN folderId uuid; + +-- Add foreign key constraint to folders table +ALTER TABLE documents ADD CONSTRAINT fk_documents_folderid + FOREIGN KEY (folderId) REFERENCES folders(id); + +-- Add index for efficient folder-based document queries +CREATE INDEX idx_documents_folderid ON documents(folderId); + +-- Add originalPath column to store the exact path provided during upload +-- This value is IMMUTABLE after creation - it preserves the original upload path +-- even if the document is later moved to a different virtual folder +ALTER TABLE documents ADD COLUMN originalPath text; +COMMENT ON COLUMN documents.originalPath IS + 'Original path provided during upload. IMMUTABLE after creation - never modify this value.'; + +-- Add comments to clarify immutability of S3 storage fields +COMMENT ON COLUMN documentEntries.key IS + 'S3 object key. IMMUTABLE after creation - never modify this value.'; +COMMENT ON COLUMN documentEntries.bucket IS + 'S3 bucket name. IMMUTABLE after creation - never modify this value.'; + +-- Add comment to clarify that folders are virtual (database-only, no S3 relationship) +COMMENT ON TABLE folders IS + 'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.'; +COMMENT ON COLUMN folders.path IS + 'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.'; diff --git a/internal/database/migrations/00000000000112_create_field_extractions_main.down.sql b/internal/database/migrations/00000000000112_create_field_extractions_main.down.sql new file mode 100644 index 00000000..cd26d6f6 --- /dev/null +++ b/internal/database/migrations/00000000000112_create_field_extractions_main.down.sql @@ -0,0 +1,5 @@ +-- Rollback: Drop documentFieldExtractions table and its indexes +DROP INDEX IF EXISTS idx_documentfieldextractions_filename; +DROP INDEX IF EXISTS idx_documentfieldextractions_createdby; +DROP INDEX IF EXISTS idx_documentfieldextractions_documentid; +DROP TABLE IF EXISTS documentFieldExtractions; diff --git a/internal/database/migrations/00000000000112_create_field_extractions_main.up.sql b/internal/database/migrations/00000000000112_create_field_extractions_main.up.sql new file mode 100644 index 00000000..df6986f2 --- /dev/null +++ b/internal/database/migrations/00000000000112_create_field_extractions_main.up.sql @@ -0,0 +1,48 @@ +-- Create documentFieldExtractions table for single-value fields +CREATE TABLE documentFieldExtractions ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), + documentId uuid NOT NULL, + + -- Single-value fields (19 fields total - 1:1 relationship) + -- Document identification + fileName text, + contractTitle text, + aareteDerivedAmendmentNum int, + + -- Party information + clientName text, + payerName text, + payerState text, + providerState text, + + -- Tax identification numbers (stored as text to preserve leading zeros) + filenameTin text, + provGroupTin text, + provGroupNpi text, + provGroupNameFull text, + provOtherTin text, + provOtherNpi text, + provOtherNameFull text, + + -- Contract dates + aareteDerivedEffectiveDt date, + aareteDerivedTerminationDt date, + + -- Renewal information + autoRenewalInd boolean, + autoRenewalTerm text, + + -- Metadata + createdAt timestamp NOT NULL DEFAULT NOW(), + createdBy varchar(255) NOT NULL, + + FOREIGN KEY (documentId) REFERENCES documents(id) +); + +-- Indexes for efficient queries +CREATE INDEX idx_documentfieldextractions_documentid + ON documentFieldExtractions(documentId); +CREATE INDEX idx_documentfieldextractions_createdby + ON documentFieldExtractions(createdBy); +CREATE INDEX idx_documentfieldextractions_filename + ON documentFieldExtractions(fileName); diff --git a/internal/database/migrations/00000000000113_create_field_extraction_versions.down.sql b/internal/database/migrations/00000000000113_create_field_extraction_versions.down.sql new file mode 100644 index 00000000..2761ad75 --- /dev/null +++ b/internal/database/migrations/00000000000113_create_field_extraction_versions.down.sql @@ -0,0 +1,4 @@ +-- Rollback: Drop documentFieldExtractionVersions table and its indexes +DROP INDEX IF EXISTS idx_documentfieldextractionversions_version; +DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid; +DROP TABLE IF EXISTS documentFieldExtractionVersions; diff --git a/internal/database/migrations/00000000000113_create_field_extraction_versions.up.sql b/internal/database/migrations/00000000000113_create_field_extraction_versions.up.sql new file mode 100644 index 00000000..5c0a4a72 --- /dev/null +++ b/internal/database/migrations/00000000000113_create_field_extraction_versions.up.sql @@ -0,0 +1,16 @@ +-- Create documentFieldExtractionVersions table for version tracking +CREATE TABLE documentFieldExtractionVersions ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), + fieldExtractionId uuid NOT NULL, + version bigint NOT NULL, + createdBy varchar(255) NOT NULL, + createdAt timestamp NOT NULL DEFAULT NOW(), + + FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id) +); + +-- Indexes for version queries +CREATE INDEX idx_documentfieldextractionversions_fieldextractionid + ON documentFieldExtractionVersions(fieldExtractionId); +CREATE INDEX idx_documentfieldextractionversions_version + ON documentFieldExtractionVersions(version); diff --git a/internal/database/migrations/00000000000114_create_field_extraction_arrays.down.sql b/internal/database/migrations/00000000000114_create_field_extraction_arrays.down.sql new file mode 100644 index 00000000..1d4884d0 --- /dev/null +++ b/internal/database/migrations/00000000000114_create_field_extraction_arrays.down.sql @@ -0,0 +1,4 @@ +-- Rollback: Drop documentFieldExtractionArrayFields table and its indexes +DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index; +DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid; +DROP TABLE IF EXISTS documentFieldExtractionArrayFields; diff --git a/internal/database/migrations/00000000000008_field_extractions.up.sql b/internal/database/migrations/00000000000114_create_field_extraction_arrays.up.sql similarity index 64% rename from internal/database/migrations/00000000000008_field_extractions.up.sql rename to internal/database/migrations/00000000000114_create_field_extraction_arrays.up.sql index 0bf6eed8..feb8e5d3 100644 --- a/internal/database/migrations/00000000000008_field_extractions.up.sql +++ b/internal/database/migrations/00000000000114_create_field_extraction_arrays.up.sql @@ -1,84 +1,3 @@ --- Migration 008: Field Extractions (Consolidated) --- Document field extraction storage for GenAI-extracted data --- Merges: 112 + 113 + 114 + 117 - --- Create documentFieldExtractions table for single-value fields -CREATE TABLE documentFieldExtractions ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - documentId uuid NOT NULL, - - -- Single-value fields (19 fields total - 1:1 relationship) - -- Document identification - fileName text, - contractTitle text, - aareteDerivedAmendmentNum int, - - -- Party information - clientName text, - payerName text, - payerState text, - providerState text, - - -- Tax identification numbers (stored as text to preserve leading zeros) - filenameTin text, - provGroupTin text, - provGroupNpi text, - provGroupNameFull text, - provOtherTin text, - provOtherNpi text, - provOtherNameFull text, - - -- Contract dates - aareteDerivedEffectiveDt date, - aareteDerivedTerminationDt date, - - -- Renewal information - autoRenewalInd boolean, - autoRenewalTerm text, - - -- Metadata - createdAt timestamp NOT NULL DEFAULT NOW(), - createdBy varchar(255) NOT NULL, - - FOREIGN KEY (documentId) REFERENCES documents(id) -); - --- Indexes for efficient queries -CREATE INDEX idx_documentfieldextractions_documentid - ON documentFieldExtractions(documentId); -CREATE INDEX idx_documentfieldextractions_createdby - ON documentFieldExtractions(createdBy); -CREATE INDEX idx_documentfieldextractions_filename - ON documentFieldExtractions(fileName); - --- Create documentFieldExtractionVersions table for version tracking --- Includes documentId column from migration 117 for unique constraint -CREATE TABLE documentFieldExtractionVersions ( - id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), - fieldExtractionId uuid NOT NULL, - version bigint NOT NULL, - createdBy varchar(255) NOT NULL, - createdAt timestamp NOT NULL DEFAULT NOW(), - - -- documentId for unique version constraint (from migration 117) - documentId uuid NOT NULL, - - FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id), - FOREIGN KEY (documentId) REFERENCES documents(id) -); - --- Indexes for version queries -CREATE INDEX idx_documentfieldextractionversions_fieldextractionid - ON documentFieldExtractionVersions(fieldExtractionId); -CREATE INDEX idx_documentfieldextractionversions_version - ON documentFieldExtractionVersions(version); -CREATE INDEX idx_documentfieldextractionversions_documentid - ON documentFieldExtractionVersions(documentId); - --- Unique constraint to prevent duplicate versions per document (from migration 117) -CREATE UNIQUE INDEX idx_unique_version_per_document - ON documentFieldExtractionVersions(documentId, version); - -- Create documentFieldExtractionArrayFields table for 1:N array fields CREATE TABLE documentFieldExtractionArrayFields ( id uuid PRIMARY KEY DEFAULT uuid_generate_v7(), diff --git a/internal/database/migrations/00000000000012_field_extraction_views.down.sql b/internal/database/migrations/00000000000115_create_field_extraction_views.down.sql similarity index 67% rename from internal/database/migrations/00000000000012_field_extraction_views.down.sql rename to internal/database/migrations/00000000000115_create_field_extraction_views.down.sql index 70d0eabc..7566d529 100644 --- a/internal/database/migrations/00000000000012_field_extraction_views.down.sql +++ b/internal/database/migrations/00000000000115_create_field_extraction_views.down.sql @@ -1,4 +1,3 @@ --- Migration 012: Field Extraction Views (DOWN) - +-- Rollback: Drop field extraction views DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount; DROP VIEW IF EXISTS currentFieldExtractions; diff --git a/internal/database/migrations/00000000000012_field_extraction_views.up.sql b/internal/database/migrations/00000000000115_create_field_extraction_views.up.sql similarity index 87% rename from internal/database/migrations/00000000000012_field_extraction_views.up.sql rename to internal/database/migrations/00000000000115_create_field_extraction_views.up.sql index 4e86d161..88f853c4 100644 --- a/internal/database/migrations/00000000000012_field_extraction_views.up.sql +++ b/internal/database/migrations/00000000000115_create_field_extraction_views.up.sql @@ -1,6 +1,3 @@ --- Migration 012: Field Extraction Views --- Views for querying current field extractions - -- Create view for current (newest) field extraction per document CREATE VIEW currentFieldExtractions AS SELECT DISTINCT ON (dfe.documentId) @@ -36,12 +33,12 @@ ORDER BY dfe.documentId, dfev.id DESC; CREATE VIEW currentFieldExtractionsWithArrayCount AS SELECT cfe.*, - COALESCE(array_counts.arraySize, 0) AS arraySize + COALESCE(array_counts.arraySize, 0) as arraySize FROM currentFieldExtractions cfe LEFT JOIN ( SELECT fieldExtractionId, - COUNT(*) AS arraySize + COUNT(*) as arraySize FROM documentFieldExtractionArrayFields GROUP BY fieldExtractionId ) array_counts ON array_counts.fieldExtractionId = cfe.id; diff --git a/internal/database/migrations/00000000000116_add_documentuploads_folder_id.down.sql b/internal/database/migrations/00000000000116_add_documentuploads_folder_id.down.sql new file mode 100644 index 00000000..42ca5406 --- /dev/null +++ b/internal/database/migrations/00000000000116_add_documentuploads_folder_id.down.sql @@ -0,0 +1,4 @@ +-- Remove folder_id column from documentUploads table +DROP INDEX IF EXISTS idx_documentuploads_folder_id; +ALTER TABLE documentUploads DROP CONSTRAINT IF EXISTS fk_documentuploads_folder_id; +ALTER TABLE documentUploads DROP COLUMN IF EXISTS folder_id; diff --git a/internal/database/migrations/00000000000116_add_documentuploads_folder_id.up.sql b/internal/database/migrations/00000000000116_add_documentuploads_folder_id.up.sql new file mode 100644 index 00000000..bf0b0f3b --- /dev/null +++ b/internal/database/migrations/00000000000116_add_documentuploads_folder_id.up.sql @@ -0,0 +1,6 @@ +-- Add folder_id column to documentUploads table to persist folder association during batch uploads +ALTER TABLE documentUploads ADD COLUMN folder_id uuid; +-- Add foreign key constraint referencing folders table +ALTER TABLE documentUploads ADD CONSTRAINT fk_documentuploads_folder_id FOREIGN KEY (folder_id) REFERENCES folders(id); +-- Add index for efficient lookup by folder +CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id); diff --git a/internal/database/migrations/00000000000117_add_version_unique_constraint.down.sql b/internal/database/migrations/00000000000117_add_version_unique_constraint.down.sql new file mode 100644 index 00000000..92488e11 --- /dev/null +++ b/internal/database/migrations/00000000000117_add_version_unique_constraint.down.sql @@ -0,0 +1,10 @@ +-- Remove the unique constraint and documentId column + +DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid; +DROP INDEX IF EXISTS idx_unique_version_per_document; + +ALTER TABLE documentFieldExtractionVersions +DROP CONSTRAINT IF EXISTS fk_documentfieldextractionversions_documentid; + +ALTER TABLE documentFieldExtractionVersions +DROP COLUMN IF EXISTS documentId; diff --git a/internal/database/migrations/00000000000117_add_version_unique_constraint.up.sql b/internal/database/migrations/00000000000117_add_version_unique_constraint.up.sql new file mode 100644 index 00000000..95892fec --- /dev/null +++ b/internal/database/migrations/00000000000117_add_version_unique_constraint.up.sql @@ -0,0 +1,30 @@ +-- Add unique constraint to prevent duplicate versions per document +-- This ensures that concurrent requests cannot create the same version number + +-- First, add documentId column to versions table for easier constraint enforcement +ALTER TABLE documentFieldExtractionVersions +ADD COLUMN documentId uuid; + +-- Populate documentId from the joined extraction record +UPDATE documentFieldExtractionVersions dfev +SET documentId = dfe.documentId +FROM documentFieldExtractions dfe +WHERE dfev.fieldExtractionId = dfe.id; + +-- Make documentId NOT NULL after populating +ALTER TABLE documentFieldExtractionVersions +ALTER COLUMN documentId SET NOT NULL; + +-- Add foreign key constraint (documents table has 'id' as primary key) +ALTER TABLE documentFieldExtractionVersions +ADD CONSTRAINT fk_documentfieldextractionversions_documentid +FOREIGN KEY (documentId) REFERENCES documents(id); + +-- Add unique constraint on documentId + version +-- This prevents duplicate version numbers for the same document +CREATE UNIQUE INDEX idx_unique_version_per_document +ON documentFieldExtractionVersions(documentId, version); + +-- Add index for efficient lookups by documentId +CREATE INDEX idx_documentfieldextractionversions_documentid +ON documentFieldExtractionVersions(documentId); diff --git a/internal/database/migrations/00000000000118_add_document_file_size.down.sql b/internal/database/migrations/00000000000118_add_document_file_size.down.sql new file mode 100644 index 00000000..4ff05c19 --- /dev/null +++ b/internal/database/migrations/00000000000118_add_document_file_size.down.sql @@ -0,0 +1,2 @@ +-- Remove file_size_bytes column from documents table +ALTER TABLE documents DROP COLUMN file_size_bytes; diff --git a/internal/database/migrations/00000000000118_add_document_file_size.up.sql b/internal/database/migrations/00000000000118_add_document_file_size.up.sql new file mode 100644 index 00000000..80d5ac9d --- /dev/null +++ b/internal/database/migrations/00000000000118_add_document_file_size.up.sql @@ -0,0 +1,4 @@ +-- Add file_size_bytes column to documents table +-- Stores the size of the document in bytes as measured from S3 HeadObject +-- NULL for legacy documents (before this feature was added) +ALTER TABLE documents ADD COLUMN file_size_bytes BIGINT; diff --git a/internal/database/migrations/00000000000119_remove_queries.down.sql b/internal/database/migrations/00000000000119_remove_queries.down.sql new file mode 100644 index 00000000..2ce31be9 --- /dev/null +++ b/internal/database/migrations/00000000000119_remove_queries.down.sql @@ -0,0 +1,287 @@ +-- Down migration to restore query functionality +-- This recreates all query-related tables, views, and functions + +-- Drop the simplified fullActiveCollectors view +DROP VIEW IF EXISTS fullActiveCollectors; + +-- Recreate query type enum +CREATE TYPE queryType AS ENUM ('context_full', 'json_extractor'); + +-- Recreate queries table +CREATE TABLE queries ( + queryId uuid primary key DEFAULT uuid_generate_v7(), + queryType queryType not null +); + +-- Recreate queryVersions table +CREATE TABLE queryVersions ( + queryId uuid not null, + versionId int not null, + addedAt timestamp not null default current_timestamp, + primary key (versionId, queryId), + foreign key (queryId) references queries(queryId) +); + +-- Recreate version number function and trigger +CREATE OR REPLACE FUNCTION setQueryVersionNumber() +RETURNS TRIGGER AS $$ +BEGIN + SELECT COALESCE(MAX(versionId), 0) + 1 + INTO NEW.versionId + FROM queryVersions + WHERE queryId = NEW.queryId; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER setQueryVersionNumberTrigger +BEFORE INSERT ON queryVersions +FOR EACH ROW +EXECUTE FUNCTION setQueryVersionNumber(); + +-- Recreate queryActiveVersions table +CREATE TABLE queryActiveVersions ( + activeVersionEntryId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + versionId int not null, + foreign key (queryId, versionId) references queryVersions(queryId, versionId) +); + +-- Recreate requiredQueries table +CREATE TABLE requiredQueries ( + requiredQueryEntryId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + requiredQueryId uuid not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (requiredQueryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), + unique (queryId, requiredQueryId, removedVersion) +); + +-- Recreate queryConfigs table +CREATE TABLE queryConfigs ( + configId uuid primary key DEFAULT uuid_generate_v7(), + queryId uuid not null, + config jsonb not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (queryId, addedVersion) references queryVersions(queryId, versionId), + foreign key (queryId, removedVersion) references queryVersions(queryId, versionId), + unique (queryId, removedVersion) +); + +-- Recreate removeQueryConfig function and trigger +CREATE OR REPLACE FUNCTION removeQueryConfig() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE queryConfigs + SET removedVersion = NEW.addedVersion + WHERE queryId = NEW.queryId and removedVersion is null; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER removeQueryConfigTrigger +BEFORE INSERT ON queryConfigs +FOR EACH ROW +EXECUTE FUNCTION removeQueryConfig(); + +-- Recreate collectorQueries table +CREATE TABLE collectorQueries ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + name varchar(255) not null, + queryId uuid not null, + addedVersion int not null, + removedVersion int, + foreign key (queryId) references queries(queryId), + foreign key (clientId) references clients(clientId), + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + unique (clientId, name, removedVersion) +); + +-- Recreate results table +CREATE TABLE results ( + id uuid primary key DEFAULT uuid_generate_v7(), + textEntryId uuid not null, + queryId uuid not null, + value TEXT not null, + queryVersion int not null, + foreign key (queryId) references queries(queryId), + foreign key (textEntryId) references documentTextExtractions(id), + foreign key (queryId, queryVersion) references queryVersions(queryId, versionId) +); + +-- Recreate resultDependencies table +CREATE TABLE resultDependencies ( + resultId uuid not null, + requiredResultId uuid not null, + foreign key (resultId) references results(id), + foreign key (requiredResultId) references results(id), + CONSTRAINT result_not_self_dependent CHECK (resultId != requiredResultId) +); + +-- Recreate query views +CREATE VIEW queryCurrentActiveVersions as +SELECT DISTINCT + q.queryId, + coalesce( + (FIRST_VALUE(av.versionId) OVER (PARTITION BY q.queryId ORDER BY av.activeVersionEntryId DESC)), + 0 + )::int as activeVersion + FROM queries AS q + LEFT JOIN queryActiveVersions as av on av.queryId = q.queryId; + +CREATE VIEW queryLatestVersions as + SELECT + q.queryId, + coalesce(max(v.versionId), 0)::int as latestVersion + FROM queries AS q + LEFT JOIN queryVersions as v on v.queryId = q.queryId + GROUP BY q.queryId; + +CREATE VIEW queryCurrentConfigs as + SELECT av.queryId, c.config + FROM queryCurrentActiveVersions as av + LEFT JOIN queryConfigs AS c ON av.queryId = c.queryId + and isInVersion(av.activeVersion, c.addedVersion, c.removedVersion); + +CREATE VIEW queryCurrentRequiredIds as + SELECT DISTINCT av.queryId, r.requiredQueryId + FROM queryCurrentActiveVersions as av + LEFT JOIN requiredQueries AS r ON av.queryId = r.queryId + and isInVersion(av.activeVersion, r.addedVersion, r.removedVersion); + +CREATE VIEW queryCurrentRequiredIdsAGG as + SELECT queryId, + coalesce( + ARRAY_AGG(DISTINCT requiredQueryId) + FILTER (WHERE requiredQueryId != '00000000-0000-0000-0000-000000000000')::uuid[], + array[]::uuid[] + )::uuid[] as requiredIds + FROM queryCurrentRequiredIds + GROUP BY queryId; + +CREATE VIEW fullActiveQueries AS +SELECT DISTINCT q.queryId, q.queryType, av.activeVersion, lv.latestVersion, c.config, r.requiredIds + FROM queries AS q + JOIN queryCurrentActiveVersions as av on q.queryId = av.queryId + JOIN queryLatestVersions as lv on lv.queryId = q.queryId + JOIN queryCurrentConfigs AS c ON q.queryId = c.queryId + JOIN queryCurrentRequiredIdsAGG AS r ON q.queryId = r.queryId; + +CREATE VIEW queryActiveDependencies AS +WITH RECURSIVE queryActiveDependencies(queryId, requiredQueryId, path, cycle) AS ( + SELECT + queryId, + requiredQueryId, + ARRAY[queryId, requiredQueryId]::uuid[] AS path, + false AS cycle + FROM queryCurrentRequiredIds + + UNION ALL + + SELECT + q.queryId, + qd.requiredQueryId, + path || qd.requiredQueryId, + qd.requiredQueryId = ANY(path) AS cycle + FROM queryCurrentRequiredIds as q + JOIN queryActiveDependencies as qd ON q.queryId = qd.requiredQueryId + WHERE NOT qd.cycle + ) + SELECT DISTINCT queryId as id, requiredQueryId + FROM queryActiveDependencies + WHERE NOT cycle; + +-- Recreate collector views that depend on queries +CREATE VIEW currentCollectorQueries as +SELECT DISTINCT + av.clientId, + q.name, + q.queryId + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorQueries AS q ON av.clientId = q.clientId + and isInVersion(av.activeVersion, q.addedVersion, q.removedVersion); + +CREATE VIEW currentCollectorQueriesJSONAGG as +SELECT DISTINCT + clientId, + jsonb_object_agg(name, queryId) FILTER (WHERE name is not null) AS fields + FROM currentCollectorQueries + GROUP BY clientId; + +CREATE VIEW fullActiveCollectors AS + SELECT DISTINCT av.clientId, + ccv.minCleanVersion, ctv.minTextVersion, + av.activeVersion, lv.latestVersion, + q.fields + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId + JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId + JOIN currentCollectorQueriesJSONAGG AS q ON av.clientId = q.clientId; + +CREATE VIEW collectorQueryDependencyTree AS + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.clientId, cq.queryId, ri.requiredIds + FROM currentCollectorQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId + + UNION ALL + + SELECT acq.clientId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) + ) + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.queryId = ct.queryId; + +CREATE OR REPLACE FUNCTION collectorQueryDependencyTreeByClient( + _clientId varchar(255) +) +RETURNS TABLE ( + clientId varchar(255), + queryId uuid, + queryType queryType, + queryVersion int, + requiredIds uuid[] +) AS $$ +BEGIN + RETURN QUERY + WITH clientQueries as ( + SELECT q.clientId, q.queryId + FROM currentCollectorQueries as q + WHERE q.clientId = _clientId + ), + dependencyTree as ( + WITH RECURSIVE collectorQueryDependencyTree AS ( + SELECT cq.clientId, cq.queryId, ri.requiredIds + FROM clientQueries as cq + JOIN queryCurrentRequiredIdsAGG as ri on cq.queryId = ri.queryId + + UNION ALL + + SELECT acq.clientId, q.queryId, q.requiredIds + FROM queryCurrentRequiredIdsAGG as q + JOIN collectorQueryDependencyTree as acq on q.queryId = ANY(acq.requiredIds) + ) + SELECT DISTINCT ct.clientId, ct.queryId, q.queryType, av.activeVersion as queryVersion, ct.requiredIds + FROM collectorQueryDependencyTree as ct + JOIN queryCurrentActiveVersions as av on ct.queryId = av.queryId + JOIN queries as q on q.queryId = ct.queryId + ) + SELECT t.clientId, t.queryId, t.queryType, t.queryVersion, t.requiredIds + FROM dependencyTree as t + WHERE t.clientID is not null; +END; +$$ LANGUAGE plpgsql; diff --git a/internal/database/migrations/00000000000119_remove_queries.up.sql b/internal/database/migrations/00000000000119_remove_queries.up.sql new file mode 100644 index 00000000..7a4eb28d --- /dev/null +++ b/internal/database/migrations/00000000000119_remove_queries.up.sql @@ -0,0 +1,53 @@ +-- Migration to remove query functionality from the database +-- This drops all query-related tables, views, and functions + +-- First drop the function that depends on query views +DROP FUNCTION IF EXISTS collectorQueryDependencyTreeByClient; + +-- Drop the collector views that depend on query tables (in order of dependencies) +DROP VIEW IF EXISTS collectorQueryDependencyTree; +DROP VIEW IF EXISTS fullActiveCollectors; +DROP VIEW IF EXISTS currentCollectorQueriesJSONAGG; +DROP VIEW IF EXISTS currentCollectorQueries; + +-- Drop query views (in order of dependencies) +DROP VIEW IF EXISTS fullActiveQueries; +DROP VIEW IF EXISTS queryActiveDependencies; +DROP VIEW IF EXISTS queryCurrentRequiredIdsAGG; +DROP VIEW IF EXISTS queryCurrentRequiredIds; +DROP VIEW IF EXISTS queryCurrentConfigs; +DROP VIEW IF EXISTS queryLatestVersions; +DROP VIEW IF EXISTS queryCurrentActiveVersions; + +-- Drop result tables +DROP TABLE IF EXISTS resultDependencies; +DROP TABLE IF EXISTS results; + +-- Drop collectorQueries table (links collectors to queries) +DROP TABLE IF EXISTS collectorQueries; + +-- Drop query configuration and dependency tables +DROP TABLE IF EXISTS queryConfigs; +DROP TABLE IF EXISTS requiredQueries; +DROP TABLE IF EXISTS queryActiveVersions; +DROP TABLE IF EXISTS queryVersions; +DROP TABLE IF EXISTS queries; + +-- Drop triggers and functions +DROP TRIGGER IF EXISTS setQueryVersionNumberTrigger ON queryVersions; +DROP FUNCTION IF EXISTS setQueryVersionNumber; +DROP TRIGGER IF EXISTS removeQueryConfigTrigger ON queryConfigs; +DROP FUNCTION IF EXISTS removeQueryConfig; + +-- Drop the query type enum +DROP TYPE IF EXISTS queryType; + +-- Recreate the fullActiveCollectors view without the fields column +CREATE VIEW fullActiveCollectors AS + SELECT DISTINCT av.clientId, + ccv.minCleanVersion, ctv.minTextVersion, + av.activeVersion, lv.latestVersion + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId + JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId; diff --git a/internal/database/migrations/00000000000119_stub.down.sql b/internal/database/migrations/00000000000119_stub.down.sql deleted file mode 100644 index be879209..00000000 --- a/internal/database/migrations/00000000000119_stub.down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Stub migration for version compatibility --- No actual changes needed. diff --git a/internal/database/migrations/00000000000119_stub.up.sql b/internal/database/migrations/00000000000119_stub.up.sql deleted file mode 100644 index 026a33e6..00000000 --- a/internal/database/migrations/00000000000119_stub.up.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Stub migration for version compatibility --- This file exists to satisfy the migrate tool for databases that were --- previously at version 119 (before migration collapse). --- No actual changes - the schema is already correct. diff --git a/internal/database/migrations/00000000000120_remove_text_extraction.down.sql b/internal/database/migrations/00000000000120_remove_text_extraction.down.sql new file mode 100644 index 00000000..e3833b88 --- /dev/null +++ b/internal/database/migrations/00000000000120_remove_text_extraction.down.sql @@ -0,0 +1,104 @@ +-- Rollback migration to restore text extraction functionality +-- NOTE: This recreates text extraction tables and views + +-- First drop the simplified fullActiveCollectors view +DROP VIEW IF EXISTS fullActiveCollectors; + +-- Recreate documentTextExtractions table +CREATE TABLE documentTextExtractions ( + id uuid primary key DEFAULT uuid_generate_v7(), + cleanId uuid not null, + bucket text not null, + key text not null, + hash text not null, + createdAt timestamp not null, + part unsignedsmallint not null, + foreign key (cleanId) references documentCleans(id) +); + +-- Recreate documentTextExtractionEntries table +CREATE TABLE documentTextExtractionEntries ( + id uuid primary key DEFAULT uuid_generate_v7(), + textId uuid not null, + version bigint not null, + foreign key (textId) references documentTextExtractions(id) +); + +-- Recreate collectorMinTextVersions table +CREATE TABLE collectorMinTextVersions ( + id uuid primary key DEFAULT uuid_generate_v7(), + clientId varchar(255) not null, + versionId bigint not null, + addedVersion int not null, + removedVersion int, + foreign key (clientId, addedVersion) references collectorVersions(clientId, id), + foreign key (clientId, removedVersion) references collectorVersions(clientId, id), + foreign key (clientId) references clients(clientId) +); + +-- Recreate removeMinTextVersion function +CREATE OR REPLACE FUNCTION removeMinTextVersion() +RETURNS TRIGGER AS $$ +BEGIN + UPDATE collectorMinTextVersions + SET removedVersion = NEW.addedVersion + WHERE clientId = NEW.clientId and removedVersion is null; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Recreate trigger for collectorMinTextVersions +CREATE TRIGGER removeMinTextVersionTrigger +BEFORE INSERT ON collectorMinTextVersions +FOR EACH ROW +EXECUTE FUNCTION removeMinTextVersion(); + +-- Recreate currentCollectorMinTextVersions view +CREATE VIEW currentCollectorMinTextVersions as +SELECT DISTINCT + av.clientId, + coalesce(ctv.versionId, 0) as minTextVersion + FROM collectorCurrentActiveVersions as av + LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId + and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion); + +-- Recreate fullActiveCollectors view with minTextVersion +CREATE VIEW fullActiveCollectors AS + SELECT DISTINCT av.clientId, + ccv.minCleanVersion, ctv.minTextVersion, + av.activeVersion, lv.latestVersion + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId + JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId; + +-- Recreate currentTextEntries view +CREATE VIEW currentTextEntries as +WITH RankedExtractions AS ( + SELECT + extract.id, + cc.documentId, + extract.bucket, + extract.key, + extract.hash, + extract.cleanId, + dtee.version as extractionVersion, + ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num + FROM documents d + JOIN currentCleanEntries cc on cc.documentId = d.id + JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId + JOIN documentTextExtractions extract ON extract.cleanId = cc.id + JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id + AND dtee.version >= ctv.minTextVersion +) +SELECT + id, + documentId, + bucket, + key, + hash, + cleanId, + extractionVersion +FROM RankedExtractions +WHERE row_num = 1; diff --git a/internal/database/migrations/00000000000120_remove_text_extraction.up.sql b/internal/database/migrations/00000000000120_remove_text_extraction.up.sql new file mode 100644 index 00000000..da1fa313 --- /dev/null +++ b/internal/database/migrations/00000000000120_remove_text_extraction.up.sql @@ -0,0 +1,32 @@ +-- Migration to remove text extraction functionality from the database +-- This drops all text extraction related tables, views, and functions +-- NOTE: Field extractions are separate and NOT affected by this migration + +-- First drop the view that depends on text extraction tables +DROP VIEW IF EXISTS currentTextEntries; + +-- Drop fullActiveCollectors view that depends on currentCollectorMinTextVersions +DROP VIEW IF EXISTS fullActiveCollectors; + +-- Drop the currentCollectorMinTextVersions view +DROP VIEW IF EXISTS currentCollectorMinTextVersions; + +-- Drop the trigger and function for collectorMinTextVersions +DROP TRIGGER IF EXISTS removeMinTextVersionTrigger ON collectorMinTextVersions; +DROP FUNCTION IF EXISTS removeMinTextVersion; + +-- Drop the collectorMinTextVersions table +DROP TABLE IF EXISTS collectorMinTextVersions; + +-- Drop the text extraction tables (child first for FK constraints) +DROP TABLE IF EXISTS documentTextExtractionEntries; +DROP TABLE IF EXISTS documentTextExtractions; + +-- Recreate fullActiveCollectors view WITHOUT minTextVersion +CREATE VIEW fullActiveCollectors AS + SELECT DISTINCT av.clientId, + ccv.minCleanVersion, + av.activeVersion, lv.latestVersion + FROM collectorCurrentActiveVersions as av + JOIN collectorLatestVersions as lv on lv.clientId = av.clientId + JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId; diff --git a/internal/database/migrations/00000000000120_stub.down.sql b/internal/database/migrations/00000000000120_stub.down.sql deleted file mode 100644 index be879209..00000000 --- a/internal/database/migrations/00000000000120_stub.down.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Stub migration for version compatibility --- No actual changes needed. diff --git a/internal/database/migrations/00000000000120_stub.up.sql b/internal/database/migrations/00000000000120_stub.up.sql deleted file mode 100644 index ce66da66..00000000 --- a/internal/database/migrations/00000000000120_stub.up.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Stub migration for version compatibility --- This file exists to satisfy the migrate tool for databases that were --- previously at version 120 (before migration collapse). --- No actual changes - the schema is already correct.