Merged in feature/remove-mocks (pull request #202)
Feature/remove mocks * remove mocks * cleanup db migrations
This commit is contained in:
Jay Brown
@@ -1 +1,7 @@
|
||||
DROP EXTENSION "pgcrypto";
|
||||
-- Migration 001: Extensions and Core Functions (DOWN)
|
||||
-- Drops in reverse order of creation
|
||||
|
||||
DROP DOMAIN IF EXISTS unsignedsmallint;
|
||||
DROP FUNCTION IF EXISTS isInVersion;
|
||||
DROP FUNCTION IF EXISTS uuid_generate_v7;
|
||||
DROP EXTENSION IF EXISTS "pgcrypto";
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
-- Migration 001: Extensions and Core Functions
|
||||
-- Creates PostgreSQL extensions and utility functions used throughout the schema
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
create or replace function uuid_generate_v7()
|
||||
returns uuid
|
||||
as $$
|
||||
select encode(
|
||||
-- UUID v7 generator function (time-ordered UUIDs)
|
||||
CREATE OR REPLACE FUNCTION uuid_generate_v7()
|
||||
RETURNS uuid
|
||||
AS $$
|
||||
SELECT encode(
|
||||
set_bit(
|
||||
set_bit(
|
||||
overlay(uuid_send(gen_random_uuid())
|
||||
@@ -16,22 +20,24 @@ select encode(
|
||||
),
|
||||
'hex')::uuid;
|
||||
$$
|
||||
language SQL
|
||||
volatile;
|
||||
LANGUAGE SQL
|
||||
VOLATILE;
|
||||
|
||||
-- Version checking function for temporal data patterns
|
||||
CREATE FUNCTION isInVersion(
|
||||
_version int,
|
||||
_addedVersion int,
|
||||
_removedVersion int
|
||||
)
|
||||
RETURNS boolean as $$
|
||||
RETURNS boolean AS $$
|
||||
BEGIN
|
||||
RETURN _version is null OR (
|
||||
RETURN _version IS NULL OR (
|
||||
_version >= _addedVersion AND
|
||||
(_removedVersion is null OR _version < _removedVersion)
|
||||
(_removedVersion IS NULL OR _version < _removedVersion)
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Domain for unsigned small integers
|
||||
CREATE DOMAIN unsignedsmallint AS SMALLINT
|
||||
CHECK (VALUE >= 0);
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Migration 002: Clients (DOWN)
|
||||
|
||||
DROP TABLE IF EXISTS clientCanSync;
|
||||
DROP TABLE IF EXISTS clients;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Migration 002: Clients
|
||||
-- Core client tables for customer configuration
|
||||
|
||||
CREATE TABLE clients (
|
||||
clientId varchar(255) PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE clientCanSync (
|
||||
syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
canSync boolean NOT NULL,
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId)
|
||||
);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 003: Collectors (DOWN)
|
||||
|
||||
DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions;
|
||||
DROP FUNCTION IF EXISTS removeMinCleanVersion;
|
||||
DROP TABLE IF EXISTS collectorMinCleanVersions;
|
||||
DROP TABLE IF EXISTS collectorActiveVersions;
|
||||
DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions;
|
||||
DROP FUNCTION IF EXISTS setCollectorVersionNumber;
|
||||
DROP TABLE IF EXISTS collectorVersions;
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Migration 003: Collectors (Simplified)
|
||||
-- Collector versioning and configuration
|
||||
-- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used)
|
||||
|
||||
CREATE TABLE collectorVersions (
|
||||
clientId varchar(255) NOT NULL,
|
||||
id int NOT NULL,
|
||||
addedAt timestamp NOT NULL DEFAULT current_timestamp,
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
PRIMARY KEY (id, clientId)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
SELECT COALESCE(MAX(id), 0) + 1
|
||||
INTO NEW.id
|
||||
FROM collectorVersions
|
||||
WHERE clientId = NEW.clientId;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER setCollectorVersionNumberTrigger
|
||||
BEFORE INSERT ON collectorVersions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION setCollectorVersionNumber();
|
||||
|
||||
CREATE TABLE collectorActiveVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
versionId int NOT NULL,
|
||||
FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id)
|
||||
);
|
||||
|
||||
CREATE TABLE collectorMinCleanVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
versionId bigint NOT NULL,
|
||||
addedVersion int NOT NULL,
|
||||
removedVersion int,
|
||||
FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id),
|
||||
FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id),
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE collectorMinCleanVersions
|
||||
SET removedVersion = NEW.addedVersion
|
||||
WHERE clientId = NEW.clientId AND removedVersion IS NULL;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER removeMinCleanVersionTrigger
|
||||
BEFORE INSERT ON collectorMinCleanVersions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION removeMinCleanVersion();
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 004: Batch Uploads (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_batch_uploads_status;
|
||||
DROP INDEX IF EXISTS idx_batch_uploads_client_id;
|
||||
DROP TABLE IF EXISTS batch_uploads;
|
||||
DROP TYPE IF EXISTS batch_status;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- Migration 004: Batch Uploads
|
||||
-- Batch upload tracking for ZIP file uploads
|
||||
-- Merges original migrations 103 + 104
|
||||
|
||||
-- Create batch status ENUM type
|
||||
CREATE TYPE batch_status AS ENUM (
|
||||
'processing', -- Currently processing
|
||||
'completed', -- All documents processed (may have individual failures)
|
||||
'failed', -- Batch-level failure (ZIP corrupt, etc.)
|
||||
'cancelled' -- Cancelled by user
|
||||
);
|
||||
|
||||
-- Create batch uploads table with all columns
|
||||
CREATE TABLE batch_uploads (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
client_id varchar(255) NOT NULL,
|
||||
original_filename text NOT NULL,
|
||||
|
||||
-- Essential batch metrics
|
||||
total_documents integer NOT NULL,
|
||||
processed_documents integer NOT NULL DEFAULT 0,
|
||||
failed_documents integer NOT NULL DEFAULT 0,
|
||||
invalid_type_documents integer NOT NULL DEFAULT 0,
|
||||
|
||||
-- Status and progress
|
||||
status batch_status NOT NULL DEFAULT 'processing',
|
||||
progress_percent integer NOT NULL DEFAULT 0,
|
||||
|
||||
-- Failed document filenames (JSON array)
|
||||
failed_filenames jsonb DEFAULT '[]'::jsonb,
|
||||
|
||||
-- Timestamps
|
||||
created_at timestamp NOT NULL DEFAULT NOW(),
|
||||
completed_at timestamp,
|
||||
|
||||
-- Storage metadata (from migration 104)
|
||||
archive_bucket text,
|
||||
archive_key text,
|
||||
file_size_bytes bigint,
|
||||
|
||||
FOREIGN KEY (client_id) REFERENCES clients(clientId),
|
||||
|
||||
-- Constraint: storage metadata consistency
|
||||
CONSTRAINT batch_storage_consistent CHECK (
|
||||
(archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
|
||||
(archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Create indexes for efficient queries
|
||||
CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id);
|
||||
CREATE INDEX idx_batch_uploads_status ON batch_uploads(status);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 005: Folders (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_folders_path;
|
||||
DROP INDEX IF EXISTS idx_folders_clientid;
|
||||
DROP INDEX IF EXISTS idx_folders_parentid;
|
||||
DROP TABLE IF EXISTS folders;
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Migration 005: Folders
|
||||
-- Virtual folder hierarchy for document organization
|
||||
|
||||
CREATE TABLE folders (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
path text NOT NULL,
|
||||
parentId uuid,
|
||||
clientId varchar(255) NOT NULL,
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
createdBy varchar(255) NOT NULL,
|
||||
FOREIGN KEY (parentId) REFERENCES folders(id),
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
UNIQUE(clientId, path),
|
||||
CHECK (id != parentId)
|
||||
);
|
||||
|
||||
-- Indexes for efficient folder queries
|
||||
CREATE INDEX idx_folders_parentid ON folders(parentId);
|
||||
CREATE INDEX idx_folders_clientid ON folders(clientId);
|
||||
CREATE INDEX idx_folders_path ON folders(path);
|
||||
|
||||
-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
|
||||
COMMENT ON TABLE folders IS
|
||||
'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
|
||||
COMMENT ON COLUMN folders.path IS
|
||||
'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
|
||||
@@ -0,0 +1,31 @@
|
||||
-- Migration 006: Documents (DOWN)
|
||||
|
||||
-- Drop document clean entries first (depends on documentCleans)
|
||||
DROP TABLE IF EXISTS documentCleanEntries;
|
||||
|
||||
-- Drop document cleans
|
||||
DROP TABLE IF EXISTS documentCleans;
|
||||
|
||||
-- Drop document entries
|
||||
DROP TABLE IF EXISTS documentEntries;
|
||||
|
||||
-- Drop documentUploads indexes
|
||||
DROP INDEX IF EXISTS idx_documentuploads_folder_id;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_clientid;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_batch_id;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_filename;
|
||||
|
||||
-- Drop documentUploads table
|
||||
DROP TABLE IF EXISTS documentUploads;
|
||||
|
||||
-- Drop documents indexes
|
||||
DROP INDEX IF EXISTS idx_documents_folderid;
|
||||
DROP INDEX IF EXISTS idx_documents_filename;
|
||||
DROP INDEX IF EXISTS idx_documents_batch_id;
|
||||
|
||||
-- Drop documents table
|
||||
DROP TABLE IF EXISTS documents;
|
||||
|
||||
-- Drop enums
|
||||
DROP TYPE IF EXISTS cleanMimeType;
|
||||
DROP TYPE IF EXISTS cleanFailType;
|
||||
@@ -0,0 +1,125 @@
|
||||
-- Migration 006: Documents (Consolidated)
|
||||
-- Document storage and processing tables
|
||||
-- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118
|
||||
-- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used)
|
||||
|
||||
-- Enum for document clean failure types
|
||||
CREATE TYPE cleanFailType AS ENUM (
|
||||
'invalid_mimetype',
|
||||
'invalid_read',
|
||||
'invalid_read_pages',
|
||||
'zero_page_count',
|
||||
'large_file',
|
||||
'small_dimensions',
|
||||
'large_dimensions',
|
||||
'small_dpi',
|
||||
'large_dpi'
|
||||
);
|
||||
|
||||
-- Enum for clean document mime types
|
||||
CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
|
||||
|
||||
-- Main documents table with all columns
|
||||
CREATE TABLE documents (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
hash text NOT NULL,
|
||||
|
||||
-- Batch reference (from migration 103)
|
||||
batch_id uuid,
|
||||
|
||||
-- Filename (from migration 105)
|
||||
filename TEXT,
|
||||
|
||||
-- Folder organization (from migration 111)
|
||||
folderId uuid,
|
||||
originalPath text,
|
||||
|
||||
-- File size (from migration 118)
|
||||
file_size_bytes BIGINT,
|
||||
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
|
||||
FOREIGN KEY (folderId) REFERENCES folders(id),
|
||||
UNIQUE(clientId, hash)
|
||||
);
|
||||
|
||||
-- Indexes for documents table
|
||||
CREATE INDEX idx_documents_batch_id ON documents(batch_id);
|
||||
CREATE INDEX idx_documents_filename ON documents(filename);
|
||||
CREATE INDEX idx_documents_folderid ON documents(folderId);
|
||||
|
||||
-- Comments for immutable fields (from migration 111)
|
||||
COMMENT ON COLUMN documents.originalPath IS
|
||||
'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
|
||||
|
||||
-- Document uploads table (staging area for incoming documents)
|
||||
CREATE TABLE documentUploads (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
bucket text NOT NULL,
|
||||
key text NOT NULL,
|
||||
clientId varchar(255) NOT NULL,
|
||||
part unsignedsmallint NOT NULL,
|
||||
createdAt timestamp NOT NULL,
|
||||
|
||||
-- Filename (from migration 106)
|
||||
filename TEXT,
|
||||
|
||||
-- Batch reference (from migration 107)
|
||||
batch_id uuid,
|
||||
|
||||
-- Folder reference (from migration 116)
|
||||
folder_id uuid,
|
||||
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
FOREIGN KEY (folder_id) REFERENCES folders(id)
|
||||
);
|
||||
|
||||
-- Indexes for documentUploads
|
||||
CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
|
||||
CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
|
||||
CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
|
||||
CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
|
||||
|
||||
-- Document entries (S3 storage records)
|
||||
CREATE TABLE documentEntries (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
bucket text NOT NULL,
|
||||
key text NOT NULL,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Comments for immutable fields (from migration 111)
|
||||
COMMENT ON COLUMN documentEntries.key IS
|
||||
'S3 object key. IMMUTABLE after creation - never modify this value.';
|
||||
COMMENT ON COLUMN documentEntries.bucket IS
|
||||
'S3 bucket name. IMMUTABLE after creation - never modify this value.';
|
||||
|
||||
-- Document cleans (processed/cleaned documents)
|
||||
CREATE TABLE documentCleans (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
bucket text,
|
||||
key text,
|
||||
hash text,
|
||||
mimetype cleanMimeType,
|
||||
fail cleanFailType,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id),
|
||||
CONSTRAINT bucket_and_key_together CHECK (
|
||||
(bucket IS NULL) = (key IS NULL) AND
|
||||
(bucket IS NULL) = (mimetype IS NULL) AND
|
||||
(bucket IS NULL) = (hash IS NULL)
|
||||
),
|
||||
CONSTRAINT location_xor_fail CHECK (
|
||||
(bucket IS NOT NULL) != (fail IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Document clean entries (version tracking for cleans)
|
||||
CREATE TABLE documentCleanEntries (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
cleanId uuid NOT NULL,
|
||||
version bigint NOT NULL,
|
||||
FOREIGN KEY (cleanId) REFERENCES documentCleans(id)
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Migration 007: Labels (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_appliedat;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_label;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_documentid;
|
||||
DROP TABLE IF EXISTS documentLabels;
|
||||
DROP TABLE IF EXISTS labels;
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Migration 007: Labels
|
||||
-- Document labeling system for tracking processing status
|
||||
|
||||
-- Create labels lookup table
|
||||
CREATE TABLE labels (
|
||||
label text PRIMARY KEY,
|
||||
description text NOT NULL
|
||||
);
|
||||
|
||||
-- Seed initial label values
|
||||
INSERT INTO labels (label, description) VALUES
|
||||
('Ingested', 'Document has been ingested into the system'),
|
||||
('OCR_Processed', 'OCR text extraction completed'),
|
||||
('GenAI_Processed', 'GenAI processing completed'),
|
||||
('Doczy_AI_Completed', 'Doczy.AI processing completed'),
|
||||
('Dashboard_Ready', 'Document ready for dashboard display');
|
||||
|
||||
-- Create documentLabels junction table
|
||||
CREATE TABLE documentLabels (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
label text NOT NULL,
|
||||
appliedAt timestamp NOT NULL DEFAULT NOW(),
|
||||
appliedBy varchar(255) NOT NULL,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (label) REFERENCES labels(label)
|
||||
);
|
||||
|
||||
-- Indexes for efficient label queries
|
||||
CREATE INDEX idx_documentlabels_documentid ON documentLabels(documentId);
|
||||
CREATE INDEX idx_documentlabels_label ON documentLabels(label);
|
||||
CREATE INDEX idx_documentlabels_appliedat ON documentLabels(appliedAt);
|
||||
-- Composite index for finding most recent label application
|
||||
CREATE INDEX idx_documentlabels_document_label_time ON documentLabels(documentId, label, appliedAt DESC);
|
||||
@@ -0,0 +1,19 @@
|
||||
-- Migration 008: Field Extractions (DOWN)
|
||||
|
||||
-- Drop array fields table and indexes
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
|
||||
|
||||
-- Drop versions table and indexes
|
||||
DROP INDEX IF EXISTS idx_unique_version_per_document;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractionVersions;
|
||||
|
||||
-- Drop main field extractions table and indexes
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractions;
|
||||
@@ -0,0 +1,254 @@
|
||||
-- Migration 008: Field Extractions (Consolidated)
|
||||
-- Document field extraction storage for GenAI-extracted data
|
||||
-- Merges: 112 + 113 + 114 + 117
|
||||
|
||||
-- Create documentFieldExtractions table for single-value fields
|
||||
CREATE TABLE documentFieldExtractions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
|
||||
-- Single-value fields (19 fields total - 1:1 relationship)
|
||||
-- Document identification
|
||||
fileName text,
|
||||
contractTitle text,
|
||||
aareteDerivedAmendmentNum int,
|
||||
|
||||
-- Party information
|
||||
clientName text,
|
||||
payerName text,
|
||||
payerState text,
|
||||
providerState text,
|
||||
|
||||
-- Tax identification numbers (stored as text to preserve leading zeros)
|
||||
filenameTin text,
|
||||
provGroupTin text,
|
||||
provGroupNpi text,
|
||||
provGroupNameFull text,
|
||||
provOtherTin text,
|
||||
provOtherNpi text,
|
||||
provOtherNameFull text,
|
||||
|
||||
-- Contract dates
|
||||
aareteDerivedEffectiveDt date,
|
||||
aareteDerivedTerminationDt date,
|
||||
|
||||
-- Renewal information
|
||||
autoRenewalInd boolean,
|
||||
autoRenewalTerm text,
|
||||
|
||||
-- Metadata
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
createdBy varchar(255) NOT NULL,
|
||||
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Indexes for efficient queries
|
||||
CREATE INDEX idx_documentfieldextractions_documentid
|
||||
ON documentFieldExtractions(documentId);
|
||||
CREATE INDEX idx_documentfieldextractions_createdby
|
||||
ON documentFieldExtractions(createdBy);
|
||||
CREATE INDEX idx_documentfieldextractions_filename
|
||||
ON documentFieldExtractions(fileName);
|
||||
|
||||
-- Create documentFieldExtractionVersions table for version tracking
|
||||
-- Includes documentId column from migration 117 for unique constraint
|
||||
CREATE TABLE documentFieldExtractionVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
fieldExtractionId uuid NOT NULL,
|
||||
version bigint NOT NULL,
|
||||
createdBy varchar(255) NOT NULL,
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
|
||||
-- documentId for unique version constraint (from migration 117)
|
||||
documentId uuid NOT NULL,
|
||||
|
||||
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Indexes for version queries
|
||||
CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
|
||||
ON documentFieldExtractionVersions(fieldExtractionId);
|
||||
CREATE INDEX idx_documentfieldextractionversions_version
|
||||
ON documentFieldExtractionVersions(version);
|
||||
CREATE INDEX idx_documentfieldextractionversions_documentid
|
||||
ON documentFieldExtractionVersions(documentId);
|
||||
|
||||
-- Unique constraint to prevent duplicate versions per document (from migration 117)
|
||||
CREATE UNIQUE INDEX idx_unique_version_per_document
|
||||
ON documentFieldExtractionVersions(documentId, version);
|
||||
|
||||
-- Create documentFieldExtractionArrayFields table for 1:N array fields
|
||||
CREATE TABLE documentFieldExtractionArrayFields (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
fieldExtractionId uuid NOT NULL,
|
||||
arrayIndex smallint NOT NULL,
|
||||
|
||||
-- Array-value fields (112 fields total - 1:N relationship)
|
||||
-- All fields at same arrayIndex form one "row" of the array
|
||||
|
||||
-- Exhibit information
|
||||
exhibitTitle text,
|
||||
exhibitPage text,
|
||||
|
||||
-- Reimbursement provider information
|
||||
reimbProvTin text,
|
||||
reimbProvNpi text,
|
||||
reimbProvName text,
|
||||
reimbEffectiveDt date,
|
||||
reimbTerminationDt date,
|
||||
|
||||
-- Claim and product codes
|
||||
aareteDerivedClaimTypeCd text,
|
||||
aareteDerivedProduct text,
|
||||
aareteDerivedLob text,
|
||||
aareteDerivedProgram text,
|
||||
aareteDerivedNetwork text,
|
||||
aareteDerivedProvType text,
|
||||
|
||||
-- Provider taxonomy and specialty
|
||||
provTaxonomyCd text,
|
||||
provTaxonomyCdDesc text,
|
||||
provSpecialtyCd text,
|
||||
provSpecialtyCdDesc text,
|
||||
|
||||
-- Service location
|
||||
placeOfServiceCd text,
|
||||
placeOfServiceCdDesc text,
|
||||
billTypeCd text,
|
||||
billTypeCdDesc text,
|
||||
|
||||
-- Patient demographics
|
||||
patientAgeMin text,
|
||||
patientAgeMax text,
|
||||
|
||||
-- Reimbursement terms
|
||||
reimbTerm text,
|
||||
lobProgramRelationship text,
|
||||
lobProductRelationship text,
|
||||
|
||||
-- Carveout and payment logic
|
||||
carveoutInd boolean,
|
||||
carveoutCd text,
|
||||
lesserOfInd boolean,
|
||||
greaterOfInd boolean,
|
||||
aareteDerivedReimbMethod text,
|
||||
unitOfMeasure text,
|
||||
|
||||
-- Reimbursement rates
|
||||
reimbPctRate numeric(10,4),
|
||||
reimbFeeRate numeric(12,2),
|
||||
reimbConversionFactor numeric(12,4),
|
||||
triggerCapThresholdAmt numeric(12,2),
|
||||
triggerBaseThreshold numeric(12,2),
|
||||
|
||||
-- Default and addition
|
||||
defaultInd boolean,
|
||||
additionDesc text,
|
||||
additionMaxFeeRateInc numeric(12,2),
|
||||
additionMaxPctRateInc numeric(10,4),
|
||||
aareteDerivedAdditionRateChangeTimeline text,
|
||||
|
||||
-- Fee schedule
|
||||
aareteDerivedFeeSchedule text,
|
||||
aareteDerivedFeeScheduleVersion text,
|
||||
|
||||
-- Service codes
|
||||
serviceTerm text,
|
||||
cpt4ProcCd text,
|
||||
cpt4ProcCdDesc text,
|
||||
cpt4ProcMod text,
|
||||
cpt4ProcModDesc text,
|
||||
revenueCd text,
|
||||
revenueCdDesc text,
|
||||
diagCd text,
|
||||
diagCdDesc text,
|
||||
ndcCd text,
|
||||
ndcCdDesc text,
|
||||
|
||||
-- Claim admit and status
|
||||
claimAdmitTypeCd text,
|
||||
authAdmitTypeDesc text,
|
||||
claimStatusCd text,
|
||||
claimStatusCdDesc text,
|
||||
|
||||
-- Grouper information
|
||||
grouperType text,
|
||||
grouperCd text,
|
||||
grouperCdDesc text,
|
||||
grouperPctRate numeric(10,4),
|
||||
grouperBaseRate numeric(12,2),
|
||||
aareteDerivedGrouperVersion text,
|
||||
grouperAlternativeLevelOfCare text,
|
||||
grouperSeverityInd boolean,
|
||||
grouperSeverity text,
|
||||
grouperRiskOfMortalitySubclass text,
|
||||
grouperTransferInd boolean,
|
||||
grouperReadmissionsInd boolean,
|
||||
grouperHacInd boolean,
|
||||
|
||||
-- Outlier terms
|
||||
outlierTerm text,
|
||||
outlierFirstDollarInd boolean,
|
||||
rangeNbrDays text,
|
||||
outlierFixedLossNbrDaysThreshold numeric(10,2),
|
||||
outlierFixedLossThreshold numeric(12,2),
|
||||
outlierMaximum numeric(12,2),
|
||||
outlierMaximumFrequency numeric(10,2),
|
||||
outlierPctRate numeric(10,4),
|
||||
outlierExclusionCd text,
|
||||
outlierExclusionCdDesc text,
|
||||
|
||||
-- Facility adjustments
|
||||
facilityAdjustmentTerm text,
|
||||
dshInd boolean,
|
||||
dshPctRate numeric(10,4),
|
||||
dshFeeRate numeric(12,2),
|
||||
imeInd boolean,
|
||||
imePctRate numeric(10,4),
|
||||
imeFeeRate numeric(12,2),
|
||||
ntapInd boolean,
|
||||
ntapPctRate numeric(10,4),
|
||||
ntapFeeRate numeric(12,2),
|
||||
ucInd boolean,
|
||||
ucPctRate numeric(10,4),
|
||||
ucFeeRate numeric(12,2),
|
||||
gmeInd boolean,
|
||||
gmePctRate numeric(10,4),
|
||||
gmeFeeRate numeric(12,2),
|
||||
|
||||
-- Rate escalator
|
||||
rateEscalatorInd boolean,
|
||||
rateEscalatorDesc text,
|
||||
rateEscalatorMaxRateIncPct numeric(10,4),
|
||||
rateEscalatorRateChangeTimeline numeric(10,2),
|
||||
|
||||
-- Stop loss
|
||||
stopLossTerm text,
|
||||
stopLossFirstDollarInd boolean,
|
||||
stopLossRangeNbrDays numeric(10,2),
|
||||
stopLossFixedLossThreshold numeric(12,2),
|
||||
stopLossMaximum numeric(12,2),
|
||||
stopLossMaximumFrequency numeric(10,2),
|
||||
stopLossDailyMaxRate numeric(12,2),
|
||||
stopLossPctRateOnExcessCharges numeric(10,4),
|
||||
stopLossExclusionCd text,
|
||||
stopLossExclusionDesc text,
|
||||
|
||||
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
|
||||
|
||||
-- Ensure unique index per extraction
|
||||
UNIQUE (fieldExtractionId, arrayIndex),
|
||||
|
||||
-- Ensure arrayIndex is non-negative
|
||||
CHECK (arrayIndex >= 0)
|
||||
);
|
||||
|
||||
-- Indexes for efficient array field queries
|
||||
CREATE INDEX idx_documentfieldextractionarrayfields_fieldextractionid
|
||||
ON documentFieldExtractionArrayFields(fieldExtractionId);
|
||||
|
||||
-- Composite index for efficient ordered retrieval
|
||||
CREATE INDEX idx_documentfieldextractionarrayfields_id_index
|
||||
ON documentFieldExtractionArrayFields(fieldExtractionId, arrayIndex);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 010: Collector Views (DOWN)
|
||||
|
||||
DROP VIEW IF EXISTS fullActiveCollectors;
|
||||
DROP VIEW IF EXISTS currentCollectorMinCleanVersions;
|
||||
DROP VIEW IF EXISTS collectorLatestVersions;
|
||||
DROP VIEW IF EXISTS collectorCurrentActiveVersions;
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Migration 010: Collector Views (Simplified)
|
||||
-- Views for collector configuration
|
||||
-- NOTE: Query-related views and text version views removed (no longer used)
|
||||
|
||||
CREATE VIEW collectorCurrentActiveVersions AS
|
||||
SELECT
|
||||
c.clientId,
|
||||
COALESCE(v.versionId, 0)::int AS activeVersion
|
||||
FROM
|
||||
clients c
|
||||
LEFT JOIN (
|
||||
SELECT DISTINCT ON (clientId)
|
||||
clientId,
|
||||
versionId
|
||||
FROM
|
||||
collectorActiveVersions
|
||||
ORDER BY clientId, id DESC
|
||||
) v ON c.clientId = v.clientId;
|
||||
|
||||
CREATE VIEW collectorLatestVersions AS
|
||||
SELECT
|
||||
j.clientId,
|
||||
COALESCE(MAX(v.id), 0)::int AS latestVersion
|
||||
FROM clients AS j
|
||||
LEFT JOIN collectorVersions AS v ON v.clientId = j.clientId
|
||||
GROUP BY j.clientId;
|
||||
|
||||
CREATE VIEW currentCollectorMinCleanVersions AS
|
||||
SELECT DISTINCT
|
||||
av.clientId,
|
||||
COALESCE(ctv.versionId, 0) AS minCleanVersion
|
||||
FROM collectorCurrentActiveVersions AS av
|
||||
LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId
|
||||
AND isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
|
||||
|
||||
-- Final fullActiveCollectors view (simplified without minTextVersion or fields)
|
||||
CREATE VIEW fullActiveCollectors AS
|
||||
SELECT DISTINCT av.clientId,
|
||||
ccv.minCleanVersion,
|
||||
av.activeVersion, lv.latestVersion
|
||||
FROM collectorCurrentActiveVersions AS av
|
||||
JOIN collectorLatestVersions AS lv ON lv.clientId = av.clientId
|
||||
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration 011: Document Views (DOWN)
|
||||
|
||||
DROP FUNCTION IF EXISTS listValidDocumentResults;
|
||||
DROP VIEW IF EXISTS fullClients;
|
||||
DROP VIEW IF EXISTS currentClientCanSync;
|
||||
DROP VIEW IF EXISTS currentCleanEntries;
|
||||
DROP FUNCTION IF EXISTS listDocumentIDs;
|
||||
@@ -0,0 +1,190 @@
|
||||
-- Migration 011: Document Views (Simplified)
|
||||
-- Views and functions for document queries
|
||||
-- NOTE: currentTextEntries and listValidDocumentResults removed (no longer used)
|
||||
|
||||
CREATE OR REPLACE FUNCTION listDocumentIDs(
|
||||
_clientId varchar(255),
|
||||
_batchSize INTEGER,
|
||||
_offset INTEGER
|
||||
)
|
||||
RETURNS TABLE (id uuid, totalCount BIGINT) AS $$
|
||||
DECLARE
|
||||
totalCount BIGINT := 0;
|
||||
BEGIN
|
||||
SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE clientId = _clientId;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT d.id, totalCount
|
||||
FROM documents AS d
|
||||
WHERE d.clientId = _clientId
|
||||
ORDER BY d.id
|
||||
LIMIT _batchSize
|
||||
OFFSET _offset;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE VIEW currentCleanEntries AS
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
dc.id,
|
||||
dc.documentId,
|
||||
dc.bucket,
|
||||
dc.key,
|
||||
dc.mimetype,
|
||||
dc.fail,
|
||||
dc.hash,
|
||||
dce.version,
|
||||
d.clientId,
|
||||
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) AS row_num
|
||||
FROM documents d
|
||||
JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId
|
||||
JOIN documentCleans dc ON dc.documentId = d.id
|
||||
JOIN documentCleanEntries dce ON dc.id = dce.cleanId
|
||||
AND dce.version >= ccv.minCleanVersion
|
||||
)
|
||||
SELECT
|
||||
re.id,
|
||||
re.documentId,
|
||||
re.bucket,
|
||||
re.key,
|
||||
re.version,
|
||||
re.hash,
|
||||
re.mimetype,
|
||||
re.fail,
|
||||
re.clientId
|
||||
FROM RankedExtractions re
|
||||
WHERE row_num = 1;
|
||||
|
||||
CREATE VIEW currentClientCanSync AS
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
ccs.clientId,
|
||||
ccs.canSync,
|
||||
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) AS row_num
|
||||
FROM clients c
|
||||
JOIN clientCanSync ccs ON c.clientId = ccs.clientId
|
||||
)
|
||||
SELECT
|
||||
c.clientId,
|
||||
COALESCE(r.canSync, false) AS canSync
|
||||
FROM clients c
|
||||
LEFT JOIN RankedExtractions r ON r.clientId = c.clientId AND r.row_num = 1;
|
||||
|
||||
CREATE VIEW fullClients AS
|
||||
SELECT c.clientId, c.name, cs.canSync
|
||||
FROM clients AS c
|
||||
JOIN currentClientCanSync AS cs ON cs.clientId = c.clientId;
|
||||
|
||||
-- NOTE: This function exists for schema compatibility with original migrations.
|
||||
-- It references tables/functions that no longer exist (results, resultDependencies,
|
||||
-- collectorQueryDependencyTreeByClient, currentTextEntries) and will fail if called.
|
||||
-- It was created in original migration 102 but never explicitly dropped.
|
||||
CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid)
|
||||
RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$
|
||||
DECLARE
|
||||
count_before INT;
|
||||
count_after INT;
|
||||
iterations INT := 0;
|
||||
BEGIN
|
||||
CREATE TEMPORARY TABLE allResults AS
|
||||
WITH
|
||||
docs AS (
|
||||
SELECT id as documentId, clientId
|
||||
FROM documents
|
||||
WHERE id = doc_id
|
||||
),
|
||||
tree as (
|
||||
select q.queryId, q.requiredIds, q.clientId, q.queryVersion
|
||||
from docs,
|
||||
LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
|
||||
),
|
||||
query_dependency_tree AS (
|
||||
WITH RECURSIVE query_deps AS (
|
||||
SELECT
|
||||
q.queryId,
|
||||
unnest(q.requiredIds) as requiredId
|
||||
FROM tree AS q
|
||||
WHERE array_length(q.requiredIds, 1) > 0
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
qd.queryId,
|
||||
unnest(q.requiredIds) as requiredId
|
||||
FROM query_deps qd
|
||||
JOIN tree q ON qd.requiredId = q.queryId
|
||||
WHERE array_length(q.requiredIds, 1) > 0
|
||||
)
|
||||
SELECT
|
||||
query_deps.queryId,
|
||||
array_agg(DISTINCT requiredId) AS all_required_ids
|
||||
FROM query_deps
|
||||
GROUP BY query_deps.queryId
|
||||
),
|
||||
docResults AS (
|
||||
SELECT
|
||||
d.documentId,
|
||||
q.queryId,
|
||||
r.id AS resultId,
|
||||
r.value,
|
||||
q.requiredIds as directRequiredIds,
|
||||
COALESCE(qdt.all_required_ids, ARRAY[]::uuid[]) AS allRequiredIds
|
||||
FROM docs AS d
|
||||
JOIN tree AS q ON q.clientId = d.clientId
|
||||
LEFT JOIN query_dependency_tree qdt ON q.queryId = qdt.queryId
|
||||
LEFT JOIN currentTextEntries AS tte ON tte.documentId = d.documentId
|
||||
LEFT JOIN results AS r
|
||||
ON q.queryId = r.queryId
|
||||
AND r.queryVersion = q.queryVersion
|
||||
AND tte.id = r.textEntryId
|
||||
WHERE r.value is not null
|
||||
)
|
||||
SELECT * FROM docResults dr;
|
||||
|
||||
CREATE TEMPORARY TABLE finalResults AS
|
||||
SELECT * from allResults where array_length(directRequiredIds, 1) is null;
|
||||
|
||||
SELECT COUNT(*) INTO count_before FROM finalResults;
|
||||
RAISE NOTICE 'Starting with % rows', count_before;
|
||||
|
||||
LOOP
|
||||
iterations := iterations + 1;
|
||||
|
||||
INSERT INTO finalResults
|
||||
SELECT dr.*
|
||||
FROM allResults dr
|
||||
WHERE
|
||||
dr.directRequiredIds IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(dr.directRequiredIds) AS req_id(queryId)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM allResults req_dr
|
||||
WHERE req_dr.queryId = req_id.queryId
|
||||
AND req_dr.resultId IN (SELECT fr.resultId FROM finalResults fr)
|
||||
AND exists (
|
||||
select 1 from resultDependencies rrd
|
||||
where dr.resultId = rrd.resultId
|
||||
and req_dr.resultId = rrd.requiredResultId
|
||||
)
|
||||
)
|
||||
)
|
||||
and dr.resultId NOT IN (SELECT fr.resultId FROM finalResults fr);
|
||||
|
||||
GET DIAGNOSTICS count_after = ROW_COUNT;
|
||||
RAISE NOTICE 'Iteration %: Additional Entries %',
|
||||
iterations, count_after;
|
||||
|
||||
IF count_after = 0 THEN
|
||||
EXIT;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
RETURN QUERY SELECT tr.documentId, tr.queryId, tr.resultId, tr.value
|
||||
FROM finalResults tr;
|
||||
|
||||
DROP TABLE finalResults;
|
||||
DROP TABLE allResults;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Migration 012: Field Extraction Views (DOWN)
|
||||
|
||||
DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount;
|
||||
DROP VIEW IF EXISTS currentFieldExtractions;
|
||||
@@ -0,0 +1,47 @@
|
||||
-- Migration 012: Field Extraction Views
|
||||
-- Views for querying current field extractions
|
||||
|
||||
-- Create view for current (newest) field extraction per document
|
||||
CREATE VIEW currentFieldExtractions AS
|
||||
SELECT DISTINCT ON (dfe.documentId)
|
||||
dfe.id,
|
||||
dfe.documentId,
|
||||
dfe.fileName,
|
||||
dfe.contractTitle,
|
||||
dfe.aareteDerivedAmendmentNum,
|
||||
dfe.clientName,
|
||||
dfe.payerName,
|
||||
dfe.payerState,
|
||||
dfe.providerState,
|
||||
dfe.filenameTin,
|
||||
dfe.provGroupTin,
|
||||
dfe.provGroupNpi,
|
||||
dfe.provGroupNameFull,
|
||||
dfe.provOtherTin,
|
||||
dfe.provOtherNpi,
|
||||
dfe.provOtherNameFull,
|
||||
dfe.aareteDerivedEffectiveDt,
|
||||
dfe.aareteDerivedTerminationDt,
|
||||
dfe.autoRenewalInd,
|
||||
dfe.autoRenewalTerm,
|
||||
dfev.version,
|
||||
dfev.createdBy,
|
||||
dfev.createdAt
|
||||
FROM documentFieldExtractions dfe
|
||||
JOIN documentFieldExtractionVersions dfev
|
||||
ON dfev.fieldExtractionId = dfe.id
|
||||
ORDER BY dfe.documentId, dfev.id DESC;
|
||||
|
||||
-- Create extended view with array field count
|
||||
CREATE VIEW currentFieldExtractionsWithArrayCount AS
|
||||
SELECT
|
||||
cfe.*,
|
||||
COALESCE(array_counts.arraySize, 0) AS arraySize
|
||||
FROM currentFieldExtractions cfe
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
fieldExtractionId,
|
||||
COUNT(*) AS arraySize
|
||||
FROM documentFieldExtractionArrayFields
|
||||
GROUP BY fieldExtractionId
|
||||
) array_counts ON array_counts.fieldExtractionId = cfe.id;
|
||||
@@ -0,0 +1 @@
|
||||
DROP EXTENSION "pgcrypto";
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
create or replace function uuid_generate_v7()
|
||||
returns uuid
|
||||
as $$
|
||||
select encode(
|
||||
set_bit(
|
||||
set_bit(
|
||||
overlay(uuid_send(gen_random_uuid())
|
||||
placing substring(int8send(floor(extract(epoch from clock_timestamp()) * 1000)::bigint) from 3)
|
||||
from 1 for 6
|
||||
),
|
||||
52, 1
|
||||
),
|
||||
53, 1
|
||||
),
|
||||
'hex')::uuid;
|
||||
$$
|
||||
language SQL
|
||||
volatile;
|
||||
|
||||
CREATE FUNCTION isInVersion(
|
||||
_version int,
|
||||
_addedVersion int,
|
||||
_removedVersion int
|
||||
)
|
||||
RETURNS boolean as $$
|
||||
BEGIN
|
||||
RETURN _version is null OR (
|
||||
_version >= _addedVersion AND
|
||||
(_removedVersion is null OR _version < _removedVersion)
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE DOMAIN unsignedsmallint AS SMALLINT
|
||||
CHECK (VALUE >= 0);
|
||||
@@ -0,0 +1,104 @@
|
||||
-- Rollback migration to restore text extraction functionality
|
||||
-- NOTE: This recreates text extraction tables and views
|
||||
|
||||
-- First drop the simplified fullActiveCollectors view
|
||||
DROP VIEW IF EXISTS fullActiveCollectors;
|
||||
|
||||
-- Recreate documentTextExtractions table
|
||||
CREATE TABLE documentTextExtractions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
cleanId uuid not null,
|
||||
bucket text not null,
|
||||
key text not null,
|
||||
hash text not null,
|
||||
createdAt timestamp not null,
|
||||
part unsignedsmallint not null,
|
||||
foreign key (cleanId) references documentCleans(id)
|
||||
);
|
||||
|
||||
-- Recreate documentTextExtractionEntries table
|
||||
CREATE TABLE documentTextExtractionEntries (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
textId uuid not null,
|
||||
version bigint not null,
|
||||
foreign key (textId) references documentTextExtractions(id)
|
||||
);
|
||||
|
||||
-- Recreate collectorMinTextVersions table
|
||||
CREATE TABLE collectorMinTextVersions (
|
||||
id uuid primary key DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) not null,
|
||||
versionId bigint not null,
|
||||
addedVersion int not null,
|
||||
removedVersion int,
|
||||
foreign key (clientId, addedVersion) references collectorVersions(clientId, id),
|
||||
foreign key (clientId, removedVersion) references collectorVersions(clientId, id),
|
||||
foreign key (clientId) references clients(clientId)
|
||||
);
|
||||
|
||||
-- Recreate removeMinTextVersion function
|
||||
CREATE OR REPLACE FUNCTION removeMinTextVersion()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE collectorMinTextVersions
|
||||
SET removedVersion = NEW.addedVersion
|
||||
WHERE clientId = NEW.clientId and removedVersion is null;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Recreate trigger for collectorMinTextVersions
|
||||
CREATE TRIGGER removeMinTextVersionTrigger
|
||||
BEFORE INSERT ON collectorMinTextVersions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION removeMinTextVersion();
|
||||
|
||||
-- Recreate currentCollectorMinTextVersions view
|
||||
CREATE VIEW currentCollectorMinTextVersions as
|
||||
SELECT DISTINCT
|
||||
av.clientId,
|
||||
coalesce(ctv.versionId, 0) as minTextVersion
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
LEFT JOIN collectorMinTextVersions AS ctv ON av.clientId = ctv.clientId
|
||||
and isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
|
||||
|
||||
-- Recreate fullActiveCollectors view with minTextVersion
|
||||
CREATE VIEW fullActiveCollectors AS
|
||||
SELECT DISTINCT av.clientId,
|
||||
ccv.minCleanVersion, ctv.minTextVersion,
|
||||
av.activeVersion, lv.latestVersion
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
|
||||
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId
|
||||
JOIN currentCollectorMinTextVersions AS ctv ON av.clientId = ctv.clientId;
|
||||
|
||||
-- Recreate currentTextEntries view
|
||||
CREATE VIEW currentTextEntries as
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
extract.id,
|
||||
cc.documentId,
|
||||
extract.bucket,
|
||||
extract.key,
|
||||
extract.hash,
|
||||
extract.cleanId,
|
||||
dtee.version as extractionVersion,
|
||||
ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
|
||||
FROM documents d
|
||||
JOIN currentCleanEntries cc on cc.documentId = d.id
|
||||
JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
|
||||
JOIN documentTextExtractions extract ON extract.cleanId = cc.id
|
||||
JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
|
||||
AND dtee.version >= ctv.minTextVersion
|
||||
)
|
||||
SELECT
|
||||
id,
|
||||
documentId,
|
||||
bucket,
|
||||
key,
|
||||
hash,
|
||||
cleanId,
|
||||
extractionVersion
|
||||
FROM RankedExtractions
|
||||
WHERE row_num = 1;
|
||||
@@ -0,0 +1,32 @@
|
||||
-- Migration to remove text extraction functionality from the database
|
||||
-- This drops all text extraction related tables, views, and functions
|
||||
-- NOTE: Field extractions are separate and NOT affected by this migration
|
||||
|
||||
-- First drop the view that depends on text extraction tables
|
||||
DROP VIEW IF EXISTS currentTextEntries;
|
||||
|
||||
-- Drop fullActiveCollectors view that depends on currentCollectorMinTextVersions
|
||||
DROP VIEW IF EXISTS fullActiveCollectors;
|
||||
|
||||
-- Drop the currentCollectorMinTextVersions view
|
||||
DROP VIEW IF EXISTS currentCollectorMinTextVersions;
|
||||
|
||||
-- Drop the trigger and function for collectorMinTextVersions
|
||||
DROP TRIGGER IF EXISTS removeMinTextVersionTrigger ON collectorMinTextVersions;
|
||||
DROP FUNCTION IF EXISTS removeMinTextVersion;
|
||||
|
||||
-- Drop the collectorMinTextVersions table
|
||||
DROP TABLE IF EXISTS collectorMinTextVersions;
|
||||
|
||||
-- Drop the text extraction tables (child first for FK constraints)
|
||||
DROP TABLE IF EXISTS documentTextExtractionEntries;
|
||||
DROP TABLE IF EXISTS documentTextExtractions;
|
||||
|
||||
-- Recreate fullActiveCollectors view WITHOUT minTextVersion
|
||||
CREATE VIEW fullActiveCollectors AS
|
||||
SELECT DISTINCT av.clientId,
|
||||
ccv.minCleanVersion,
|
||||
av.activeVersion, lv.latestVersion
|
||||
FROM collectorCurrentActiveVersions as av
|
||||
JOIN collectorLatestVersions as lv on lv.clientId = av.clientId
|
||||
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration 001: Extensions and Core Functions (DOWN)
|
||||
-- Drops in reverse order of creation
|
||||
|
||||
DROP DOMAIN IF EXISTS unsignedsmallint;
|
||||
DROP FUNCTION IF EXISTS isInVersion;
|
||||
DROP FUNCTION IF EXISTS uuid_generate_v7;
|
||||
DROP EXTENSION IF EXISTS "pgcrypto";
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Migration 001: Extensions and Core Functions
|
||||
-- Creates PostgreSQL extensions and utility functions used throughout the schema
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
|
||||
|
||||
-- UUID v7 generator function (time-ordered UUIDs)
|
||||
CREATE OR REPLACE FUNCTION uuid_generate_v7()
|
||||
RETURNS uuid
|
||||
AS $$
|
||||
SELECT encode(
|
||||
set_bit(
|
||||
set_bit(
|
||||
overlay(uuid_send(gen_random_uuid())
|
||||
placing substring(int8send(floor(extract(epoch from clock_timestamp()) * 1000)::bigint) from 3)
|
||||
from 1 for 6
|
||||
),
|
||||
52, 1
|
||||
),
|
||||
53, 1
|
||||
),
|
||||
'hex')::uuid;
|
||||
$$
|
||||
LANGUAGE SQL
|
||||
VOLATILE;
|
||||
|
||||
-- Version checking function for temporal data patterns
|
||||
CREATE FUNCTION isInVersion(
|
||||
_version int,
|
||||
_addedVersion int,
|
||||
_removedVersion int
|
||||
)
|
||||
RETURNS boolean AS $$
|
||||
BEGIN
|
||||
RETURN _version IS NULL OR (
|
||||
_version >= _addedVersion AND
|
||||
(_removedVersion IS NULL OR _version < _removedVersion)
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
-- Domain for unsigned small integers
|
||||
CREATE DOMAIN unsignedsmallint AS SMALLINT
|
||||
CHECK (VALUE >= 0);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Migration 002: Clients (DOWN)
|
||||
|
||||
DROP TABLE IF EXISTS clientCanSync;
|
||||
DROP TABLE IF EXISTS clients;
|
||||
@@ -0,0 +1,15 @@
|
||||
-- Migration 002: Clients
|
||||
-- Core client tables for customer configuration
|
||||
|
||||
CREATE TABLE clients (
|
||||
clientId varchar(255) PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
UNIQUE (name)
|
||||
);
|
||||
|
||||
CREATE TABLE clientCanSync (
|
||||
syncId uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
canSync boolean NOT NULL,
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId)
|
||||
);
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Migration 003: Collectors (DOWN)
|
||||
|
||||
DROP TRIGGER IF EXISTS removeMinCleanVersionTrigger ON collectorMinCleanVersions;
|
||||
DROP FUNCTION IF EXISTS removeMinCleanVersion;
|
||||
DROP TABLE IF EXISTS collectorMinCleanVersions;
|
||||
DROP TABLE IF EXISTS collectorActiveVersions;
|
||||
DROP TRIGGER IF EXISTS setCollectorVersionNumberTrigger ON collectorVersions;
|
||||
DROP FUNCTION IF EXISTS setCollectorVersionNumber;
|
||||
DROP TABLE IF EXISTS collectorVersions;
|
||||
@@ -0,0 +1,62 @@
|
||||
-- Migration 003: Collectors (Simplified)
|
||||
-- Collector versioning and configuration
|
||||
-- NOTE: collectorMinTextVersions and collectorQueries removed (no longer used)
|
||||
|
||||
CREATE TABLE collectorVersions (
|
||||
clientId varchar(255) NOT NULL,
|
||||
id int NOT NULL,
|
||||
addedAt timestamp NOT NULL DEFAULT current_timestamp,
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
PRIMARY KEY (id, clientId)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION setCollectorVersionNumber()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
SELECT COALESCE(MAX(id), 0) + 1
|
||||
INTO NEW.id
|
||||
FROM collectorVersions
|
||||
WHERE clientId = NEW.clientId;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER setCollectorVersionNumberTrigger
|
||||
BEFORE INSERT ON collectorVersions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION setCollectorVersionNumber();
|
||||
|
||||
CREATE TABLE collectorActiveVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
versionId int NOT NULL,
|
||||
FOREIGN KEY (clientId, versionId) REFERENCES collectorVersions(clientId, id)
|
||||
);
|
||||
|
||||
CREATE TABLE collectorMinCleanVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
versionId bigint NOT NULL,
|
||||
addedVersion int NOT NULL,
|
||||
removedVersion int,
|
||||
FOREIGN KEY (clientId, addedVersion) REFERENCES collectorVersions(clientId, id),
|
||||
FOREIGN KEY (clientId, removedVersion) REFERENCES collectorVersions(clientId, id),
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId)
|
||||
);
|
||||
|
||||
CREATE OR REPLACE FUNCTION removeMinCleanVersion()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
UPDATE collectorMinCleanVersions
|
||||
SET removedVersion = NEW.addedVersion
|
||||
WHERE clientId = NEW.clientId AND removedVersion IS NULL;
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER removeMinCleanVersionTrigger
|
||||
BEFORE INSERT ON collectorMinCleanVersions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION removeMinCleanVersion();
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 004: Batch Uploads (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_batch_uploads_status;
|
||||
DROP INDEX IF EXISTS idx_batch_uploads_client_id;
|
||||
DROP TABLE IF EXISTS batch_uploads;
|
||||
DROP TYPE IF EXISTS batch_status;
|
||||
@@ -0,0 +1,52 @@
|
||||
-- Migration 004: Batch Uploads
|
||||
-- Batch upload tracking for ZIP file uploads
|
||||
-- Merges original migrations 103 + 104
|
||||
|
||||
-- Create batch status ENUM type
|
||||
CREATE TYPE batch_status AS ENUM (
|
||||
'processing', -- Currently processing
|
||||
'completed', -- All documents processed (may have individual failures)
|
||||
'failed', -- Batch-level failure (ZIP corrupt, etc.)
|
||||
'cancelled' -- Cancelled by user
|
||||
);
|
||||
|
||||
-- Create batch uploads table with all columns
|
||||
CREATE TABLE batch_uploads (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
client_id varchar(255) NOT NULL,
|
||||
original_filename text NOT NULL,
|
||||
|
||||
-- Essential batch metrics
|
||||
total_documents integer NOT NULL,
|
||||
processed_documents integer NOT NULL DEFAULT 0,
|
||||
failed_documents integer NOT NULL DEFAULT 0,
|
||||
invalid_type_documents integer NOT NULL DEFAULT 0,
|
||||
|
||||
-- Status and progress
|
||||
status batch_status NOT NULL DEFAULT 'processing',
|
||||
progress_percent integer NOT NULL DEFAULT 0,
|
||||
|
||||
-- Failed document filenames (JSON array)
|
||||
failed_filenames jsonb DEFAULT '[]'::jsonb,
|
||||
|
||||
-- Timestamps
|
||||
created_at timestamp NOT NULL DEFAULT NOW(),
|
||||
completed_at timestamp,
|
||||
|
||||
-- Storage metadata (from migration 104)
|
||||
archive_bucket text,
|
||||
archive_key text,
|
||||
file_size_bytes bigint,
|
||||
|
||||
FOREIGN KEY (client_id) REFERENCES clients(clientId),
|
||||
|
||||
-- Constraint: storage metadata consistency
|
||||
CONSTRAINT batch_storage_consistent CHECK (
|
||||
(archive_bucket IS NULL AND archive_key IS NULL AND file_size_bytes IS NULL) OR
|
||||
(archive_bucket IS NOT NULL AND archive_key IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Create indexes for efficient queries
|
||||
CREATE INDEX idx_batch_uploads_client_id ON batch_uploads(client_id);
|
||||
CREATE INDEX idx_batch_uploads_status ON batch_uploads(status);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 005: Folders (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_folders_path;
|
||||
DROP INDEX IF EXISTS idx_folders_clientid;
|
||||
DROP INDEX IF EXISTS idx_folders_parentid;
|
||||
DROP TABLE IF EXISTS folders;
|
||||
@@ -0,0 +1,26 @@
|
||||
-- Migration 005: Folders
|
||||
-- Virtual folder hierarchy for document organization
|
||||
|
||||
CREATE TABLE folders (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
path text NOT NULL,
|
||||
parentId uuid,
|
||||
clientId varchar(255) NOT NULL,
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
createdBy varchar(255) NOT NULL,
|
||||
FOREIGN KEY (parentId) REFERENCES folders(id),
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
UNIQUE(clientId, path),
|
||||
CHECK (id != parentId)
|
||||
);
|
||||
|
||||
-- Indexes for efficient folder queries
|
||||
CREATE INDEX idx_folders_parentid ON folders(parentId);
|
||||
CREATE INDEX idx_folders_clientid ON folders(clientId);
|
||||
CREATE INDEX idx_folders_path ON folders(path);
|
||||
|
||||
-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
|
||||
COMMENT ON TABLE folders IS
|
||||
'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
|
||||
COMMENT ON COLUMN folders.path IS
|
||||
'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
|
||||
@@ -0,0 +1,31 @@
|
||||
-- Migration 006: Documents (DOWN)
|
||||
|
||||
-- Drop document clean entries first (depends on documentCleans)
|
||||
DROP TABLE IF EXISTS documentCleanEntries;
|
||||
|
||||
-- Drop document cleans
|
||||
DROP TABLE IF EXISTS documentCleans;
|
||||
|
||||
-- Drop document entries
|
||||
DROP TABLE IF EXISTS documentEntries;
|
||||
|
||||
-- Drop documentUploads indexes
|
||||
DROP INDEX IF EXISTS idx_documentuploads_folder_id;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_clientid;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_batch_id;
|
||||
DROP INDEX IF EXISTS idx_documentuploads_filename;
|
||||
|
||||
-- Drop documentUploads table
|
||||
DROP TABLE IF EXISTS documentUploads;
|
||||
|
||||
-- Drop documents indexes
|
||||
DROP INDEX IF EXISTS idx_documents_folderid;
|
||||
DROP INDEX IF EXISTS idx_documents_filename;
|
||||
DROP INDEX IF EXISTS idx_documents_batch_id;
|
||||
|
||||
-- Drop documents table
|
||||
DROP TABLE IF EXISTS documents;
|
||||
|
||||
-- Drop enums
|
||||
DROP TYPE IF EXISTS cleanMimeType;
|
||||
DROP TYPE IF EXISTS cleanFailType;
|
||||
@@ -0,0 +1,125 @@
|
||||
-- Migration 006: Documents (Consolidated)
|
||||
-- Document storage and processing tables
|
||||
-- Merges: 005_documents + 103 + 105 + 106 + 107 + 108 + 111 + 116 + 118
|
||||
-- NOTE: documentTextExtractions and documentTextExtractionEntries removed (no longer used)
|
||||
|
||||
-- Enum for document clean failure types
|
||||
CREATE TYPE cleanFailType AS ENUM (
|
||||
'invalid_mimetype',
|
||||
'invalid_read',
|
||||
'invalid_read_pages',
|
||||
'zero_page_count',
|
||||
'large_file',
|
||||
'small_dimensions',
|
||||
'large_dimensions',
|
||||
'small_dpi',
|
||||
'large_dpi'
|
||||
);
|
||||
|
||||
-- Enum for clean document mime types
|
||||
CREATE TYPE cleanMimeType AS ENUM ('application/pdf');
|
||||
|
||||
-- Main documents table with all columns
|
||||
CREATE TABLE documents (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
clientId varchar(255) NOT NULL,
|
||||
hash text NOT NULL,
|
||||
|
||||
-- Batch reference (from migration 103)
|
||||
batch_id uuid,
|
||||
|
||||
-- Filename (from migration 105)
|
||||
filename TEXT,
|
||||
|
||||
-- Folder organization (from migration 111)
|
||||
folderId uuid,
|
||||
originalPath text,
|
||||
|
||||
-- File size (from migration 118)
|
||||
file_size_bytes BIGINT,
|
||||
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
FOREIGN KEY (batch_id) REFERENCES batch_uploads(id),
|
||||
FOREIGN KEY (folderId) REFERENCES folders(id),
|
||||
UNIQUE(clientId, hash)
|
||||
);
|
||||
|
||||
-- Indexes for documents table
|
||||
CREATE INDEX idx_documents_batch_id ON documents(batch_id);
|
||||
CREATE INDEX idx_documents_filename ON documents(filename);
|
||||
CREATE INDEX idx_documents_folderid ON documents(folderId);
|
||||
|
||||
-- Comments for immutable fields (from migration 111)
|
||||
COMMENT ON COLUMN documents.originalPath IS
|
||||
'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
|
||||
|
||||
-- Document uploads table (staging area for incoming documents)
|
||||
CREATE TABLE documentUploads (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
bucket text NOT NULL,
|
||||
key text NOT NULL,
|
||||
clientId varchar(255) NOT NULL,
|
||||
part unsignedsmallint NOT NULL,
|
||||
createdAt timestamp NOT NULL,
|
||||
|
||||
-- Filename (from migration 106)
|
||||
filename TEXT,
|
||||
|
||||
-- Batch reference (from migration 107)
|
||||
batch_id uuid,
|
||||
|
||||
-- Folder reference (from migration 116)
|
||||
folder_id uuid,
|
||||
|
||||
FOREIGN KEY (clientId) REFERENCES clients(clientId),
|
||||
FOREIGN KEY (folder_id) REFERENCES folders(id)
|
||||
);
|
||||
|
||||
-- Indexes for documentUploads
|
||||
CREATE INDEX idx_documentuploads_filename ON documentUploads(filename);
|
||||
CREATE INDEX idx_documentuploads_batch_id ON documentUploads(batch_id);
|
||||
CREATE INDEX idx_documentuploads_clientid ON documentUploads(clientId);
|
||||
CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);
|
||||
|
||||
-- Document entries (S3 storage records)
|
||||
CREATE TABLE documentEntries (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
bucket text NOT NULL,
|
||||
key text NOT NULL,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Comments for immutable fields (from migration 111)
|
||||
COMMENT ON COLUMN documentEntries.key IS
|
||||
'S3 object key. IMMUTABLE after creation - never modify this value.';
|
||||
COMMENT ON COLUMN documentEntries.bucket IS
|
||||
'S3 bucket name. IMMUTABLE after creation - never modify this value.';
|
||||
|
||||
-- Document cleans (processed/cleaned documents)
|
||||
CREATE TABLE documentCleans (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
bucket text,
|
||||
key text,
|
||||
hash text,
|
||||
mimetype cleanMimeType,
|
||||
fail cleanFailType,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id),
|
||||
CONSTRAINT bucket_and_key_together CHECK (
|
||||
(bucket IS NULL) = (key IS NULL) AND
|
||||
(bucket IS NULL) = (mimetype IS NULL) AND
|
||||
(bucket IS NULL) = (hash IS NULL)
|
||||
),
|
||||
CONSTRAINT location_xor_fail CHECK (
|
||||
(bucket IS NOT NULL) != (fail IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
-- Document clean entries (version tracking for cleans)
|
||||
CREATE TABLE documentCleanEntries (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
cleanId uuid NOT NULL,
|
||||
version bigint NOT NULL,
|
||||
FOREIGN KEY (cleanId) REFERENCES documentCleans(id)
|
||||
);
|
||||
@@ -0,0 +1,8 @@
|
||||
-- Migration 007: Labels (DOWN)
|
||||
|
||||
DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_appliedat;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_label;
|
||||
DROP INDEX IF EXISTS idx_documentlabels_documentid;
|
||||
DROP TABLE IF EXISTS documentLabels;
|
||||
DROP TABLE IF EXISTS labels;
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Migration 007: Labels
|
||||
-- Document labeling system for tracking processing status
|
||||
|
||||
-- Create labels lookup table
|
||||
CREATE TABLE labels (
|
||||
label text PRIMARY KEY,
|
||||
description text NOT NULL
|
||||
);
|
||||
|
||||
-- Seed initial label values
|
||||
INSERT INTO labels (label, description) VALUES
|
||||
('Ingested', 'Document has been ingested into the system'),
|
||||
('OCR_Processed', 'OCR text extraction completed'),
|
||||
('GenAI_Processed', 'GenAI processing completed'),
|
||||
('Doczy_AI_Completed', 'Doczy.AI processing completed'),
|
||||
('Dashboard_Ready', 'Document ready for dashboard display');
|
||||
|
||||
-- Create documentLabels junction table
|
||||
CREATE TABLE documentLabels (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
label text NOT NULL,
|
||||
appliedAt timestamp NOT NULL DEFAULT NOW(),
|
||||
appliedBy varchar(255) NOT NULL,
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (label) REFERENCES labels(label)
|
||||
);
|
||||
|
||||
-- Indexes for efficient label queries
|
||||
CREATE INDEX idx_documentlabels_documentid ON documentLabels(documentId);
|
||||
CREATE INDEX idx_documentlabels_label ON documentLabels(label);
|
||||
CREATE INDEX idx_documentlabels_appliedat ON documentLabels(appliedAt);
|
||||
-- Composite index for finding most recent label application
|
||||
CREATE INDEX idx_documentlabels_document_label_time ON documentLabels(documentId, label, appliedAt DESC);
|
||||
@@ -0,0 +1,19 @@
|
||||
-- Migration 008: Field Extractions (DOWN)
|
||||
|
||||
-- Drop array fields table and indexes
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
|
||||
|
||||
-- Drop versions table and indexes
|
||||
DROP INDEX IF EXISTS idx_unique_version_per_document;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractionVersions;
|
||||
|
||||
-- Drop main field extractions table and indexes
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
|
||||
DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
|
||||
DROP TABLE IF EXISTS documentFieldExtractions;
|
||||
@@ -0,0 +1,254 @@
|
||||
-- Migration 008: Field Extractions (Consolidated)
|
||||
-- Document field extraction storage for GenAI-extracted data
|
||||
-- Merges: 112 + 113 + 114 + 117
|
||||
|
||||
-- Create documentFieldExtractions table for single-value fields
|
||||
CREATE TABLE documentFieldExtractions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
documentId uuid NOT NULL,
|
||||
|
||||
-- Single-value fields (19 fields total - 1:1 relationship)
|
||||
-- Document identification
|
||||
fileName text,
|
||||
contractTitle text,
|
||||
aareteDerivedAmendmentNum int,
|
||||
|
||||
-- Party information
|
||||
clientName text,
|
||||
payerName text,
|
||||
payerState text,
|
||||
providerState text,
|
||||
|
||||
-- Tax identification numbers (stored as text to preserve leading zeros)
|
||||
filenameTin text,
|
||||
provGroupTin text,
|
||||
provGroupNpi text,
|
||||
provGroupNameFull text,
|
||||
provOtherTin text,
|
||||
provOtherNpi text,
|
||||
provOtherNameFull text,
|
||||
|
||||
-- Contract dates
|
||||
aareteDerivedEffectiveDt date,
|
||||
aareteDerivedTerminationDt date,
|
||||
|
||||
-- Renewal information
|
||||
autoRenewalInd boolean,
|
||||
autoRenewalTerm text,
|
||||
|
||||
-- Metadata
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
createdBy varchar(255) NOT NULL,
|
||||
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Indexes for efficient queries
|
||||
CREATE INDEX idx_documentfieldextractions_documentid
|
||||
ON documentFieldExtractions(documentId);
|
||||
CREATE INDEX idx_documentfieldextractions_createdby
|
||||
ON documentFieldExtractions(createdBy);
|
||||
CREATE INDEX idx_documentfieldextractions_filename
|
||||
ON documentFieldExtractions(fileName);
|
||||
|
||||
-- Create documentFieldExtractionVersions table for version tracking
|
||||
-- Includes documentId column from migration 117 for unique constraint
|
||||
CREATE TABLE documentFieldExtractionVersions (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
fieldExtractionId uuid NOT NULL,
|
||||
version bigint NOT NULL,
|
||||
createdBy varchar(255) NOT NULL,
|
||||
createdAt timestamp NOT NULL DEFAULT NOW(),
|
||||
|
||||
-- documentId for unique version constraint (from migration 117)
|
||||
documentId uuid NOT NULL,
|
||||
|
||||
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
|
||||
FOREIGN KEY (documentId) REFERENCES documents(id)
|
||||
);
|
||||
|
||||
-- Indexes for version queries
|
||||
CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
|
||||
ON documentFieldExtractionVersions(fieldExtractionId);
|
||||
CREATE INDEX idx_documentfieldextractionversions_version
|
||||
ON documentFieldExtractionVersions(version);
|
||||
CREATE INDEX idx_documentfieldextractionversions_documentid
|
||||
ON documentFieldExtractionVersions(documentId);
|
||||
|
||||
-- Unique constraint to prevent duplicate versions per document (from migration 117)
|
||||
CREATE UNIQUE INDEX idx_unique_version_per_document
|
||||
ON documentFieldExtractionVersions(documentId, version);
|
||||
|
||||
-- Create documentFieldExtractionArrayFields table for 1:N array fields
|
||||
CREATE TABLE documentFieldExtractionArrayFields (
|
||||
id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
|
||||
fieldExtractionId uuid NOT NULL,
|
||||
arrayIndex smallint NOT NULL,
|
||||
|
||||
-- Array-value fields (112 fields total - 1:N relationship)
|
||||
-- All fields at same arrayIndex form one "row" of the array
|
||||
|
||||
-- Exhibit information
|
||||
exhibitTitle text,
|
||||
exhibitPage text,
|
||||
|
||||
-- Reimbursement provider information
|
||||
reimbProvTin text,
|
||||
reimbProvNpi text,
|
||||
reimbProvName text,
|
||||
reimbEffectiveDt date,
|
||||
reimbTerminationDt date,
|
||||
|
||||
-- Claim and product codes
|
||||
aareteDerivedClaimTypeCd text,
|
||||
aareteDerivedProduct text,
|
||||
aareteDerivedLob text,
|
||||
aareteDerivedProgram text,
|
||||
aareteDerivedNetwork text,
|
||||
aareteDerivedProvType text,
|
||||
|
||||
-- Provider taxonomy and specialty
|
||||
provTaxonomyCd text,
|
||||
provTaxonomyCdDesc text,
|
||||
provSpecialtyCd text,
|
||||
provSpecialtyCdDesc text,
|
||||
|
||||
-- Service location
|
||||
placeOfServiceCd text,
|
||||
placeOfServiceCdDesc text,
|
||||
billTypeCd text,
|
||||
billTypeCdDesc text,
|
||||
|
||||
-- Patient demographics
|
||||
patientAgeMin text,
|
||||
patientAgeMax text,
|
||||
|
||||
-- Reimbursement terms
|
||||
reimbTerm text,
|
||||
lobProgramRelationship text,
|
||||
lobProductRelationship text,
|
||||
|
||||
-- Carveout and payment logic
|
||||
carveoutInd boolean,
|
||||
carveoutCd text,
|
||||
lesserOfInd boolean,
|
||||
greaterOfInd boolean,
|
||||
aareteDerivedReimbMethod text,
|
||||
unitOfMeasure text,
|
||||
|
||||
-- Reimbursement rates
|
||||
reimbPctRate numeric(10,4),
|
||||
reimbFeeRate numeric(12,2),
|
||||
reimbConversionFactor numeric(12,4),
|
||||
triggerCapThresholdAmt numeric(12,2),
|
||||
triggerBaseThreshold numeric(12,2),
|
||||
|
||||
-- Default and addition
|
||||
defaultInd boolean,
|
||||
additionDesc text,
|
||||
additionMaxFeeRateInc numeric(12,2),
|
||||
additionMaxPctRateInc numeric(10,4),
|
||||
aareteDerivedAdditionRateChangeTimeline text,
|
||||
|
||||
-- Fee schedule
|
||||
aareteDerivedFeeSchedule text,
|
||||
aareteDerivedFeeScheduleVersion text,
|
||||
|
||||
-- Service codes
|
||||
serviceTerm text,
|
||||
cpt4ProcCd text,
|
||||
cpt4ProcCdDesc text,
|
||||
cpt4ProcMod text,
|
||||
cpt4ProcModDesc text,
|
||||
revenueCd text,
|
||||
revenueCdDesc text,
|
||||
diagCd text,
|
||||
diagCdDesc text,
|
||||
ndcCd text,
|
||||
ndcCdDesc text,
|
||||
|
||||
-- Claim admit and status
|
||||
claimAdmitTypeCd text,
|
||||
authAdmitTypeDesc text,
|
||||
claimStatusCd text,
|
||||
claimStatusCdDesc text,
|
||||
|
||||
-- Grouper information
|
||||
grouperType text,
|
||||
grouperCd text,
|
||||
grouperCdDesc text,
|
||||
grouperPctRate numeric(10,4),
|
||||
grouperBaseRate numeric(12,2),
|
||||
aareteDerivedGrouperVersion text,
|
||||
grouperAlternativeLevelOfCare text,
|
||||
grouperSeverityInd boolean,
|
||||
grouperSeverity text,
|
||||
grouperRiskOfMortalitySubclass text,
|
||||
grouperTransferInd boolean,
|
||||
grouperReadmissionsInd boolean,
|
||||
grouperHacInd boolean,
|
||||
|
||||
-- Outlier terms
|
||||
outlierTerm text,
|
||||
outlierFirstDollarInd boolean,
|
||||
rangeNbrDays text,
|
||||
outlierFixedLossNbrDaysThreshold numeric(10,2),
|
||||
outlierFixedLossThreshold numeric(12,2),
|
||||
outlierMaximum numeric(12,2),
|
||||
outlierMaximumFrequency numeric(10,2),
|
||||
outlierPctRate numeric(10,4),
|
||||
outlierExclusionCd text,
|
||||
outlierExclusionCdDesc text,
|
||||
|
||||
-- Facility adjustments
|
||||
facilityAdjustmentTerm text,
|
||||
dshInd boolean,
|
||||
dshPctRate numeric(10,4),
|
||||
dshFeeRate numeric(12,2),
|
||||
imeInd boolean,
|
||||
imePctRate numeric(10,4),
|
||||
imeFeeRate numeric(12,2),
|
||||
ntapInd boolean,
|
||||
ntapPctRate numeric(10,4),
|
||||
ntapFeeRate numeric(12,2),
|
||||
ucInd boolean,
|
||||
ucPctRate numeric(10,4),
|
||||
ucFeeRate numeric(12,2),
|
||||
gmeInd boolean,
|
||||
gmePctRate numeric(10,4),
|
||||
gmeFeeRate numeric(12,2),
|
||||
|
||||
-- Rate escalator
|
||||
rateEscalatorInd boolean,
|
||||
rateEscalatorDesc text,
|
||||
rateEscalatorMaxRateIncPct numeric(10,4),
|
||||
rateEscalatorRateChangeTimeline numeric(10,2),
|
||||
|
||||
-- Stop loss
|
||||
stopLossTerm text,
|
||||
stopLossFirstDollarInd boolean,
|
||||
stopLossRangeNbrDays numeric(10,2),
|
||||
stopLossFixedLossThreshold numeric(12,2),
|
||||
stopLossMaximum numeric(12,2),
|
||||
stopLossMaximumFrequency numeric(10,2),
|
||||
stopLossDailyMaxRate numeric(12,2),
|
||||
stopLossPctRateOnExcessCharges numeric(10,4),
|
||||
stopLossExclusionCd text,
|
||||
stopLossExclusionDesc text,
|
||||
|
||||
FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
|
||||
|
||||
-- Ensure unique index per extraction
|
||||
UNIQUE (fieldExtractionId, arrayIndex),
|
||||
|
||||
-- Ensure arrayIndex is non-negative
|
||||
CHECK (arrayIndex >= 0)
|
||||
);
|
||||
|
||||
-- Indexes for efficient array field queries
|
||||
CREATE INDEX idx_documentfieldextractionarrayfields_fieldextractionid
|
||||
ON documentFieldExtractionArrayFields(fieldExtractionId);
|
||||
|
||||
-- Composite index for efficient ordered retrieval
|
||||
CREATE INDEX idx_documentfieldextractionarrayfields_id_index
|
||||
ON documentFieldExtractionArrayFields(fieldExtractionId, arrayIndex);
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration 010: Collector Views (DOWN)
|
||||
|
||||
DROP VIEW IF EXISTS fullActiveCollectors;
|
||||
DROP VIEW IF EXISTS currentCollectorMinCleanVersions;
|
||||
DROP VIEW IF EXISTS collectorLatestVersions;
|
||||
DROP VIEW IF EXISTS collectorCurrentActiveVersions;
|
||||
@@ -0,0 +1,43 @@
|
||||
-- Migration 010: Collector Views (Simplified)
|
||||
-- Views for collector configuration
|
||||
-- NOTE: Query-related views and text version views removed (no longer used)
|
||||
|
||||
CREATE VIEW collectorCurrentActiveVersions AS
|
||||
SELECT
|
||||
c.clientId,
|
||||
COALESCE(v.versionId, 0)::int AS activeVersion
|
||||
FROM
|
||||
clients c
|
||||
LEFT JOIN (
|
||||
SELECT DISTINCT ON (clientId)
|
||||
clientId,
|
||||
versionId
|
||||
FROM
|
||||
collectorActiveVersions
|
||||
ORDER BY clientId, id DESC
|
||||
) v ON c.clientId = v.clientId;
|
||||
|
||||
CREATE VIEW collectorLatestVersions AS
|
||||
SELECT
|
||||
j.clientId,
|
||||
COALESCE(MAX(v.id), 0)::int AS latestVersion
|
||||
FROM clients AS j
|
||||
LEFT JOIN collectorVersions AS v ON v.clientId = j.clientId
|
||||
GROUP BY j.clientId;
|
||||
|
||||
CREATE VIEW currentCollectorMinCleanVersions AS
|
||||
SELECT DISTINCT
|
||||
av.clientId,
|
||||
COALESCE(ctv.versionId, 0) AS minCleanVersion
|
||||
FROM collectorCurrentActiveVersions AS av
|
||||
LEFT JOIN collectorMinCleanVersions AS ctv ON av.clientId = ctv.clientId
|
||||
AND isInVersion(av.activeVersion, ctv.addedVersion, ctv.removedVersion);
|
||||
|
||||
-- Final fullActiveCollectors view (simplified without minTextVersion or fields)
|
||||
CREATE VIEW fullActiveCollectors AS
|
||||
SELECT DISTINCT av.clientId,
|
||||
ccv.minCleanVersion,
|
||||
av.activeVersion, lv.latestVersion
|
||||
FROM collectorCurrentActiveVersions AS av
|
||||
JOIN collectorLatestVersions AS lv ON lv.clientId = av.clientId
|
||||
JOIN currentCollectorMinCleanVersions AS ccv ON av.clientId = ccv.clientId;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Migration 011: Document Views (DOWN)
|
||||
|
||||
DROP FUNCTION IF EXISTS listValidDocumentResults;
|
||||
DROP VIEW IF EXISTS fullClients;
|
||||
DROP VIEW IF EXISTS currentClientCanSync;
|
||||
DROP VIEW IF EXISTS currentCleanEntries;
|
||||
DROP FUNCTION IF EXISTS listDocumentIDs;
|
||||
@@ -0,0 +1,190 @@
|
||||
-- Migration 011: Document Views (Simplified)
|
||||
-- Views and functions for document queries
|
||||
-- NOTE: currentTextEntries and listValidDocumentResults removed (no longer used)
|
||||
|
||||
CREATE OR REPLACE FUNCTION listDocumentIDs(
|
||||
_clientId varchar(255),
|
||||
_batchSize INTEGER,
|
||||
_offset INTEGER
|
||||
)
|
||||
RETURNS TABLE (id uuid, totalCount BIGINT) AS $$
|
||||
DECLARE
|
||||
totalCount BIGINT := 0;
|
||||
BEGIN
|
||||
SELECT COUNT(*)::BIGINT INTO totalCount FROM documents WHERE clientId = _clientId;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT d.id, totalCount
|
||||
FROM documents AS d
|
||||
WHERE d.clientId = _clientId
|
||||
ORDER BY d.id
|
||||
LIMIT _batchSize
|
||||
OFFSET _offset;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE VIEW currentCleanEntries AS
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
dc.id,
|
||||
dc.documentId,
|
||||
dc.bucket,
|
||||
dc.key,
|
||||
dc.mimetype,
|
||||
dc.fail,
|
||||
dc.hash,
|
||||
dce.version,
|
||||
d.clientId,
|
||||
ROW_NUMBER() OVER (PARTITION BY dc.documentId ORDER BY dc.id DESC, dce.id DESC) AS row_num
|
||||
FROM documents d
|
||||
JOIN currentCollectorMinCleanVersions ccv ON d.clientId = ccv.clientId
|
||||
JOIN documentCleans dc ON dc.documentId = d.id
|
||||
JOIN documentCleanEntries dce ON dc.id = dce.cleanId
|
||||
AND dce.version >= ccv.minCleanVersion
|
||||
)
|
||||
SELECT
|
||||
re.id,
|
||||
re.documentId,
|
||||
re.bucket,
|
||||
re.key,
|
||||
re.version,
|
||||
re.hash,
|
||||
re.mimetype,
|
||||
re.fail,
|
||||
re.clientId
|
||||
FROM RankedExtractions re
|
||||
WHERE row_num = 1;
|
||||
|
||||
CREATE VIEW currentClientCanSync AS
|
||||
WITH RankedExtractions AS (
|
||||
SELECT
|
||||
ccs.clientId,
|
||||
ccs.canSync,
|
||||
ROW_NUMBER() OVER (PARTITION BY ccs.clientId ORDER BY ccs.syncId DESC) AS row_num
|
||||
FROM clients c
|
||||
JOIN clientCanSync ccs ON c.clientId = ccs.clientId
|
||||
)
|
||||
SELECT
|
||||
c.clientId,
|
||||
COALESCE(r.canSync, false) AS canSync
|
||||
FROM clients c
|
||||
LEFT JOIN RankedExtractions r ON r.clientId = c.clientId AND r.row_num = 1;
|
||||
|
||||
CREATE VIEW fullClients AS
|
||||
SELECT c.clientId, c.name, cs.canSync
|
||||
FROM clients AS c
|
||||
JOIN currentClientCanSync AS cs ON cs.clientId = c.clientId;
|
||||
|
||||
-- NOTE: This function exists for schema compatibility with original migrations.
|
||||
-- It references tables/functions that no longer exist (results, resultDependencies,
|
||||
-- collectorQueryDependencyTreeByClient, currentTextEntries) and will fail if called.
|
||||
-- It was created in original migration 102 but never explicitly dropped.
|
||||
CREATE OR REPLACE FUNCTION listValidDocumentResults(doc_id uuid)
|
||||
RETURNS TABLE (documentId uuid, queryId uuid, resultId uuid, value text) AS $$
|
||||
DECLARE
|
||||
count_before INT;
|
||||
count_after INT;
|
||||
iterations INT := 0;
|
||||
BEGIN
|
||||
CREATE TEMPORARY TABLE allResults AS
|
||||
WITH
|
||||
docs AS (
|
||||
SELECT id as documentId, clientId
|
||||
FROM documents
|
||||
WHERE id = doc_id
|
||||
),
|
||||
tree as (
|
||||
select q.queryId, q.requiredIds, q.clientId, q.queryVersion
|
||||
from docs,
|
||||
LATERAL collectorQueryDependencyTreeByClient(docs.clientId) AS q
|
||||
),
|
||||
query_dependency_tree AS (
|
||||
WITH RECURSIVE query_deps AS (
|
||||
SELECT
|
||||
q.queryId,
|
||||
unnest(q.requiredIds) as requiredId
|
||||
FROM tree AS q
|
||||
WHERE array_length(q.requiredIds, 1) > 0
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
qd.queryId,
|
||||
unnest(q.requiredIds) as requiredId
|
||||
FROM query_deps qd
|
||||
JOIN tree q ON qd.requiredId = q.queryId
|
||||
WHERE array_length(q.requiredIds, 1) > 0
|
||||
)
|
||||
SELECT
|
||||
query_deps.queryId,
|
||||
array_agg(DISTINCT requiredId) AS all_required_ids
|
||||
FROM query_deps
|
||||
GROUP BY query_deps.queryId
|
||||
),
|
||||
docResults AS (
|
||||
SELECT
|
||||
d.documentId,
|
||||
q.queryId,
|
||||
r.id AS resultId,
|
||||
r.value,
|
||||
q.requiredIds as directRequiredIds,
|
||||
COALESCE(qdt.all_required_ids, ARRAY[]::uuid[]) AS allRequiredIds
|
||||
FROM docs AS d
|
||||
JOIN tree AS q ON q.clientId = d.clientId
|
||||
LEFT JOIN query_dependency_tree qdt ON q.queryId = qdt.queryId
|
||||
LEFT JOIN currentTextEntries AS tte ON tte.documentId = d.documentId
|
||||
LEFT JOIN results AS r
|
||||
ON q.queryId = r.queryId
|
||||
AND r.queryVersion = q.queryVersion
|
||||
AND tte.id = r.textEntryId
|
||||
WHERE r.value is not null
|
||||
)
|
||||
SELECT * FROM docResults dr;
|
||||
|
||||
CREATE TEMPORARY TABLE finalResults AS
|
||||
SELECT * from allResults where array_length(directRequiredIds, 1) is null;
|
||||
|
||||
SELECT COUNT(*) INTO count_before FROM finalResults;
|
||||
RAISE NOTICE 'Starting with % rows', count_before;
|
||||
|
||||
LOOP
|
||||
iterations := iterations + 1;
|
||||
|
||||
INSERT INTO finalResults
|
||||
SELECT dr.*
|
||||
FROM allResults dr
|
||||
WHERE
|
||||
dr.directRequiredIds IS NOT NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM unnest(dr.directRequiredIds) AS req_id(queryId)
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM allResults req_dr
|
||||
WHERE req_dr.queryId = req_id.queryId
|
||||
AND req_dr.resultId IN (SELECT fr.resultId FROM finalResults fr)
|
||||
AND exists (
|
||||
select 1 from resultDependencies rrd
|
||||
where dr.resultId = rrd.resultId
|
||||
and req_dr.resultId = rrd.requiredResultId
|
||||
)
|
||||
)
|
||||
)
|
||||
and dr.resultId NOT IN (SELECT fr.resultId FROM finalResults fr);
|
||||
|
||||
GET DIAGNOSTICS count_after = ROW_COUNT;
|
||||
RAISE NOTICE 'Iteration %: Additional Entries %',
|
||||
iterations, count_after;
|
||||
|
||||
IF count_after = 0 THEN
|
||||
EXIT;
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
RETURN QUERY SELECT tr.documentId, tr.queryId, tr.resultId, tr.value
|
||||
FROM finalResults tr;
|
||||
|
||||
DROP TABLE finalResults;
|
||||
DROP TABLE allResults;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Migration 012: Field Extraction Views (DOWN)
|
||||
|
||||
DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount;
|
||||
DROP VIEW IF EXISTS currentFieldExtractions;
|
||||
@@ -0,0 +1,47 @@
|
||||
-- Migration 012: Field Extraction Views
|
||||
-- Views for querying current field extractions
|
||||
|
||||
-- Create view for current (newest) field extraction per document
|
||||
CREATE VIEW currentFieldExtractions AS
|
||||
SELECT DISTINCT ON (dfe.documentId)
|
||||
dfe.id,
|
||||
dfe.documentId,
|
||||
dfe.fileName,
|
||||
dfe.contractTitle,
|
||||
dfe.aareteDerivedAmendmentNum,
|
||||
dfe.clientName,
|
||||
dfe.payerName,
|
||||
dfe.payerState,
|
||||
dfe.providerState,
|
||||
dfe.filenameTin,
|
||||
dfe.provGroupTin,
|
||||
dfe.provGroupNpi,
|
||||
dfe.provGroupNameFull,
|
||||
dfe.provOtherTin,
|
||||
dfe.provOtherNpi,
|
||||
dfe.provOtherNameFull,
|
||||
dfe.aareteDerivedEffectiveDt,
|
||||
dfe.aareteDerivedTerminationDt,
|
||||
dfe.autoRenewalInd,
|
||||
dfe.autoRenewalTerm,
|
||||
dfev.version,
|
||||
dfev.createdBy,
|
||||
dfev.createdAt
|
||||
FROM documentFieldExtractions dfe
|
||||
JOIN documentFieldExtractionVersions dfev
|
||||
ON dfev.fieldExtractionId = dfe.id
|
||||
ORDER BY dfe.documentId, dfev.id DESC;
|
||||
|
||||
-- Create extended view with array field count
|
||||
CREATE VIEW currentFieldExtractionsWithArrayCount AS
|
||||
SELECT
|
||||
cfe.*,
|
||||
COALESCE(array_counts.arraySize, 0) AS arraySize
|
||||
FROM currentFieldExtractions cfe
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
fieldExtractionId,
|
||||
COUNT(*) AS arraySize
|
||||
FROM documentFieldExtractionArrayFields
|
||||
GROUP BY fieldExtractionId
|
||||
) array_counts ON array_counts.fieldExtractionId = cfe.id;
|
||||
@@ -15,9 +15,9 @@ UPDATE clients SET name = $1 WHERE clientId = $2;
|
||||
INSERT INTO clientCanSync (canSync, clientId) VALUES ($1, $2);
|
||||
|
||||
-- name: IsClientSynced :one
|
||||
-- Query functionality has been removed. A client is considered synced when
|
||||
-- all documents have completed text extraction (or failed with explicit failure).
|
||||
-- See remove_query_plan.md for details.
|
||||
-- Text extraction has been removed. A client is considered synced when
|
||||
-- all documents have completed cleaning (clean entry exists OR clean failed).
|
||||
-- See remove_texttract_and_mocks_plan.md for details.
|
||||
WITH
|
||||
docs AS (
|
||||
-- Get all documents for this client
|
||||
@@ -33,18 +33,6 @@ doc_clean_entries as (
|
||||
FROM
|
||||
docs d
|
||||
LEFT JOIN currentCleanEntries cte ON cte.documentId = d.id
|
||||
),
|
||||
doc_text_entries AS (
|
||||
-- Documents with their current text entries
|
||||
SELECT
|
||||
d.document_id,
|
||||
cte.id AS text_entry_id,
|
||||
d.clean_entry_id,
|
||||
d.clean_fail,
|
||||
d.client_id
|
||||
FROM
|
||||
doc_clean_entries d
|
||||
LEFT JOIN currentTextEntries cte ON cte.cleanId = d.clean_entry_id
|
||||
)
|
||||
SELECT (
|
||||
-- No documents means client is synced
|
||||
@@ -52,11 +40,9 @@ SELECT (
|
||||
|
||||
OR
|
||||
|
||||
-- All documents have completed processing (clean entry exists and either
|
||||
-- text entry exists or clean failed)
|
||||
-- All documents have completed processing (clean entry exists OR clean failed)
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM doc_text_entries
|
||||
where clean_entry_id is null or
|
||||
(clean_fail is null and text_entry_id is null)
|
||||
SELECT 1 FROM doc_clean_entries
|
||||
WHERE clean_entry_id IS NULL
|
||||
)
|
||||
)::bool as is_synced;
|
||||
|
||||
@@ -9,6 +9,3 @@ INSERT INTO collectorActiveVersions (clientId, versionId) VALUES ($1, $2);
|
||||
|
||||
-- name: SetCollectorCleanVersion :exec
|
||||
INSERT INTO collectorMinCleanVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
-- name: SetCollectorTextVersion :exec
|
||||
INSERT INTO collectorMinTextVersions (clientId, addedVersion, versionId) VALUES ($1, $2, $3);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user