Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support * in progress * stage 8 complete * phase 9 completed * phase 9 complete * ongoing - s3 path fix * working * optimize ci build * e2e tests * missing test
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
@@ -0,0 +1,5 @@
+-- Rollback: Drop folders table and its indexes
+DROP INDEX IF EXISTS idx_folders_path;
+DROP INDEX IF EXISTS idx_folders_clientid;
+DROP INDEX IF EXISTS idx_folders_parentid;
+DROP TABLE IF EXISTS folders;
@@ -0,0 +1,18 @@
+-- Create folders table for hierarchical document organization
+CREATE TABLE folders (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    path text NOT NULL,
+    parentId uuid,
+    clientId varchar(255) NOT NULL,
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+    createdBy varchar(255) NOT NULL,
+    FOREIGN KEY (parentId) REFERENCES folders(id),
+    FOREIGN KEY (clientId) REFERENCES clients(clientId),
+    UNIQUE(clientId, path),
+    CHECK (id != parentId)
+);
+
+-- Indexes for efficient folder queries
+CREATE INDEX idx_folders_parentid ON folders(parentId);
+CREATE INDEX idx_folders_clientid ON folders(clientId);
+CREATE INDEX idx_folders_path ON folders(path);
@@ -0,0 +1,7 @@
+-- Rollback: Drop documentLabels and labels tables with indexes
+DROP INDEX IF EXISTS idx_documentlabels_document_label_time;
+DROP INDEX IF EXISTS idx_documentlabels_appliedat;
+DROP INDEX IF EXISTS idx_documentlabels_label;
+DROP INDEX IF EXISTS idx_documentlabels_documentid;
+DROP TABLE IF EXISTS documentLabels;
+DROP TABLE IF EXISTS labels;
@@ -0,0 +1,31 @@
+-- Create labels lookup table
+CREATE TABLE labels (
+    label text PRIMARY KEY,
+    description text NOT NULL
+);
+
+-- Seed initial label values
+INSERT INTO labels (label, description) VALUES
+    ('Ingested', 'Document has been ingested into the system'),
+    ('OCR_Processed', 'OCR text extraction completed'),
+    ('GenAI_Processed', 'GenAI processing completed'),
+    ('Doczy_AI_Completed', 'Doczy.AI processing completed'),
+    ('Dashboard_Ready', 'Document ready for dashboard display');
+
+-- Create documentLabels junction table
+CREATE TABLE documentLabels (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+    label text NOT NULL,
+    appliedAt timestamp NOT NULL DEFAULT NOW(),
+    appliedBy varchar(255) NOT NULL,
+    FOREIGN KEY (documentId) REFERENCES documents(id) ON DELETE CASCADE,
+    FOREIGN KEY (label) REFERENCES labels(label)
+);
+
+-- Indexes for efficient label queries
+CREATE INDEX idx_documentlabels_documentid ON documentLabels(documentId);
+CREATE INDEX idx_documentlabels_label ON documentLabels(label);
+CREATE INDEX idx_documentlabels_appliedat ON documentLabels(appliedAt);
+-- Composite index for finding most recent label application
+CREATE INDEX idx_documentlabels_document_label_time ON documentLabels(documentId, label, appliedAt DESC);
@@ -0,0 +1,18 @@
+-- Rollback: Remove folderId, originalPath columns and related constraints/indexes from documents table
+
+-- Remove comments from folders table
+COMMENT ON COLUMN folders.path IS NULL;
+COMMENT ON TABLE folders IS NULL;
+
+-- Remove comments from documentEntries table
+COMMENT ON COLUMN documentEntries.bucket IS NULL;
+COMMENT ON COLUMN documentEntries.key IS NULL;
+
+-- Remove originalPath column
+COMMENT ON COLUMN documents.originalPath IS NULL;
+ALTER TABLE documents DROP COLUMN IF EXISTS originalPath;
+
+-- Remove folderId column
+DROP INDEX IF EXISTS idx_documents_folderid;
+ALTER TABLE documents DROP CONSTRAINT IF EXISTS fk_documents_folderid;
+ALTER TABLE documents DROP COLUMN IF EXISTS folderId;
@@ -0,0 +1,29 @@
+-- Add folderId column to documents table for folder organization
+-- Note: filename column already exists from migration 105
+ALTER TABLE documents ADD COLUMN folderId uuid;
+
+-- Add foreign key constraint to folders table
+ALTER TABLE documents ADD CONSTRAINT fk_documents_folderid
+    FOREIGN KEY (folderId) REFERENCES folders(id);
+
+-- Add index for efficient folder-based document queries
+CREATE INDEX idx_documents_folderid ON documents(folderId);
+
+-- Add originalPath column to store the exact path provided during upload
+-- This value is IMMUTABLE after creation - it preserves the original upload path
+-- even if the document is later moved to a different virtual folder
+ALTER TABLE documents ADD COLUMN originalPath text;
+COMMENT ON COLUMN documents.originalPath IS
+    'Original path provided during upload. IMMUTABLE after creation - never modify this value.';
+
+-- Add comments to clarify immutability of S3 storage fields
+COMMENT ON COLUMN documentEntries.key IS
+    'S3 object key. IMMUTABLE after creation - never modify this value.';
+COMMENT ON COLUMN documentEntries.bucket IS
+    'S3 bucket name. IMMUTABLE after creation - never modify this value.';
+
+-- Add comment to clarify that folders are virtual (database-only, no S3 relationship)
+COMMENT ON TABLE folders IS
+    'Virtual folder hierarchy for document organization. Database-only - has no direct relationship to S3 storage locations.';
+COMMENT ON COLUMN folders.path IS
+    'Virtual folder path (e.g., "/contracts/2025"). Can be renamed without affecting S3 storage.';
@@ -0,0 +1,5 @@
+-- Rollback: Drop documentFieldExtractions table and its indexes
+DROP INDEX IF EXISTS idx_documentfieldextractions_filename;
+DROP INDEX IF EXISTS idx_documentfieldextractions_createdby;
+DROP INDEX IF EXISTS idx_documentfieldextractions_documentid;
+DROP TABLE IF EXISTS documentFieldExtractions;
@@ -0,0 +1,48 @@
+-- Create documentFieldExtractions table for single-value fields
+CREATE TABLE documentFieldExtractions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    documentId uuid NOT NULL,
+
+    -- Single-value fields (19 fields total - 1:1 relationship)
+    -- Document identification
+    fileName text,
+    contractTitle text,
+    aareteDerivedAmendmentNum int,
+
+    -- Party information
+    clientName text,
+    payerName text,
+    payerState text,
+    providerState text,
+
+    -- Tax identification numbers (stored as text to preserve leading zeros)
+    filenameTin text,
+    provGroupTin text,
+    provGroupNpi text,
+    provGroupNameFull text,
+    provOtherTin text,
+    provOtherNpi text,
+    provOtherNameFull text,
+
+    -- Contract dates
+    aareteDerivedEffectiveDt date,
+    aareteDerivedTerminationDt date,
+
+    -- Renewal information
+    autoRenewalInd boolean,
+    autoRenewalTerm text,
+
+    -- Metadata
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+    createdBy varchar(255) NOT NULL,
+
+    FOREIGN KEY (documentId) REFERENCES documents(id)
+);
+
+-- Indexes for efficient queries
+CREATE INDEX idx_documentfieldextractions_documentid
+    ON documentFieldExtractions(documentId);
+CREATE INDEX idx_documentfieldextractions_createdby
+    ON documentFieldExtractions(createdBy);
+CREATE INDEX idx_documentfieldextractions_filename
+    ON documentFieldExtractions(fileName);
@@ -0,0 +1,4 @@
+-- Rollback: Drop documentFieldExtractionVersions table and its indexes
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_version;
+DROP INDEX IF EXISTS idx_documentfieldextractionversions_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionVersions;
@@ -0,0 +1,16 @@
+-- Create documentFieldExtractionVersions table for version tracking
+CREATE TABLE documentFieldExtractionVersions (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    fieldExtractionId uuid NOT NULL,
+    version bigint NOT NULL,
+    createdBy varchar(255) NOT NULL,
+    createdAt timestamp NOT NULL DEFAULT NOW(),
+
+    FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id)
+);
+
+-- Indexes for version queries
+CREATE INDEX idx_documentfieldextractionversions_fieldextractionid
+    ON documentFieldExtractionVersions(fieldExtractionId);
+CREATE INDEX idx_documentfieldextractionversions_version
+    ON documentFieldExtractionVersions(version);
@@ -0,0 +1,4 @@
+-- Rollback: Drop documentFieldExtractionArrayFields table and its indexes
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_id_index;
+DROP INDEX IF EXISTS idx_documentfieldextractionarrayfields_fieldextractionid;
+DROP TABLE IF EXISTS documentFieldExtractionArrayFields;
@@ -0,0 +1,173 @@
+-- Create documentFieldExtractionArrayFields table for 1:N array fields
+CREATE TABLE documentFieldExtractionArrayFields (
+    id uuid PRIMARY KEY DEFAULT uuid_generate_v7(),
+    fieldExtractionId uuid NOT NULL,
+    arrayIndex smallint NOT NULL,
+
+    -- Array-value fields (112 fields total - 1:N relationship)
+    -- All fields at same arrayIndex form one "row" of the array
+
+    -- Exhibit information
+    exhibitTitle text,
+    exhibitPage text,
+
+    -- Reimbursement provider information
+    reimbProvTin text,
+    reimbProvNpi text,
+    reimbProvName text,
+    reimbEffectiveDt date,
+    reimbTerminationDt date,
+
+    -- Claim and product codes
+    aareteDerivedClaimTypeCd text,
+    aareteDerivedProduct text,
+    aareteDerivedLob text,
+    aareteDerivedProgram text,
+    aareteDerivedNetwork text,
+    aareteDerivedProvType text,
+
+    -- Provider taxonomy and specialty
+    provTaxonomyCd text,
+    provTaxonomyCdDesc text,
+    provSpecialtyCd text,
+    provSpecialtyCdDesc text,
+
+    -- Service location
+    placeOfServiceCd text,
+    placeOfServiceCdDesc text,
+    billTypeCd text,
+    billTypeCdDesc text,
+
+    -- Patient demographics
+    patientAgeMin text,
+    patientAgeMax text,
+
+    -- Reimbursement terms
+    reimbTerm text,
+    lobProgramRelationship text,
+    lobProductRelationship text,
+
+    -- Carveout and payment logic
+    carveoutInd boolean,
+    carveoutCd text,
+    lesserOfInd boolean,
+    greaterOfInd boolean,
+    aareteDerivedReimbMethod text,
+    unitOfMeasure text,
+
+    -- Reimbursement rates
+    reimbPctRate numeric(10,4),
+    reimbFeeRate numeric(12,2),
+    reimbConversionFactor numeric(12,4),
+    triggerCapThresholdAmt numeric(12,2),
+    triggerBaseThreshold numeric(12,2),
+
+    -- Default and addition
+    defaultInd boolean,
+    additionDesc text,
+    additionMaxFeeRateInc numeric(12,2),
+    additionMaxPctRateInc numeric(10,4),
+    aareteDerivedAdditionRateChangeTimeline text,
+
+    -- Fee schedule
+    aareteDerivedFeeSchedule text,
+    aareteDerivedFeeScheduleVersion text,
+
+    -- Service codes
+    serviceTerm text,
+    cpt4ProcCd text,
+    cpt4ProcCdDesc text,
+    cpt4ProcMod text,
+    cpt4ProcModDesc text,
+    revenueCd text,
+    revenueCdDesc text,
+    diagCd text,
+    diagCdDesc text,
+    ndcCd text,
+    ndcCdDesc text,
+
+    -- Claim admit and status
+    claimAdmitTypeCd text,
+    authAdmitTypeDesc text,
+    claimStatusCd text,
+    claimStatusCdDesc text,
+
+    -- Grouper information
+    grouperType text,
+    grouperCd text,
+    grouperCdDesc text,
+    grouperPctRate numeric(10,4),
+    grouperBaseRate numeric(12,2),
+    aareteDerivedGrouperVersion text,
+    grouperAlternativeLevelOfCare text,
+    grouperSeverityInd boolean,
+    grouperSeverity text,
+    grouperRiskOfMortalitySubclass text,
+    grouperTransferInd boolean,
+    grouperReadmissionsInd boolean,
+    grouperHacInd boolean,
+
+    -- Outlier terms
+    outlierTerm text,
+    outlierFirstDollarInd boolean,
+    rangeNbrDays text,
+    outlierFixedLossNbrDaysThreshold numeric(10,2),
+    outlierFixedLossThreshold numeric(12,2),
+    outlierMaximum numeric(12,2),
+    outlierMaximumFrequency numeric(10,2),
+    outlierPctRate numeric(10,4),
+    outlierExclusionCd text,
+    outlierExclusionCdDesc text,
+
+    -- Facility adjustments
+    facilityAdjustmentTerm text,
+    dshInd boolean,
+    dshPctRate numeric(10,4),
+    dshFeeRate numeric(12,2),
+    imeInd boolean,
+    imePctRate numeric(10,4),
+    imeFeeRate numeric(12,2),
+    ntapInd boolean,
+    ntapPctRate numeric(10,4),
+    ntapFeeRate numeric(12,2),
+    ucInd boolean,
+    ucPctRate numeric(10,4),
+    ucFeeRate numeric(12,2),
+    gmeInd boolean,
+    gmePctRate numeric(10,4),
+    gmeFeeRate numeric(12,2),
+
+    -- Rate escalator
+    rateEscalatorInd boolean,
+    rateEscalatorDesc text,
+    rateEscalatorMaxRateIncPct numeric(10,4),
+    rateEscalatorRateChangeTimeline numeric(10,2),
+
+    -- Stop loss
+    stopLossTerm text,
+    stopLossFirstDollarInd boolean,
+    stopLossRangeNbrDays numeric(10,2),
+    stopLossFixedLossThreshold numeric(12,2),
+    stopLossMaximum numeric(12,2),
+    stopLossMaximumFrequency numeric(10,2),
+    stopLossDailyMaxRate numeric(12,2),
+    stopLossPctRateOnExcessCharges numeric(10,4),
+    stopLossExclusionCd text,
+    stopLossExclusionDesc text,
+
+    FOREIGN KEY (fieldExtractionId) REFERENCES documentFieldExtractions(id),
+
+    -- Ensure unique index per extraction
+    UNIQUE (fieldExtractionId, arrayIndex),
+
+    -- Ensure arrayIndex is non-negative
+    CHECK (arrayIndex >= 0)
+);
+
+-- Indexes for efficient array field queries
+CREATE INDEX idx_documentfieldextractionarrayfields_fieldextractionid
+    ON documentFieldExtractionArrayFields(fieldExtractionId);
+
+-- Composite index for efficient ordered retrieval
+CREATE INDEX idx_documentfieldextractionarrayfields_id_index
+    ON documentFieldExtractionArrayFields(fieldExtractionId, arrayIndex);
@@ -0,0 +1,3 @@
+-- Rollback: Drop field extraction views
+DROP VIEW IF EXISTS currentFieldExtractionsWithArrayCount;
+DROP VIEW IF EXISTS currentFieldExtractions;
@@ -0,0 +1,44 @@
+-- Create view for current (newest) field extraction per document
+CREATE VIEW currentFieldExtractions AS
+SELECT DISTINCT ON (dfe.documentId)
+    dfe.id,
+    dfe.documentId,
+    dfe.fileName,
+    dfe.contractTitle,
+    dfe.aareteDerivedAmendmentNum,
+    dfe.clientName,
+    dfe.payerName,
+    dfe.payerState,
+    dfe.providerState,
+    dfe.filenameTin,
+    dfe.provGroupTin,
+    dfe.provGroupNpi,
+    dfe.provGroupNameFull,
+    dfe.provOtherTin,
+    dfe.provOtherNpi,
+    dfe.provOtherNameFull,
+    dfe.aareteDerivedEffectiveDt,
+    dfe.aareteDerivedTerminationDt,
+    dfe.autoRenewalInd,
+    dfe.autoRenewalTerm,
+    dfev.version,
+    dfev.createdBy,
+    dfev.createdAt
+FROM documentFieldExtractions dfe
+JOIN documentFieldExtractionVersions dfev
+    ON dfev.fieldExtractionId = dfe.id
+ORDER BY dfe.documentId, dfev.id DESC;
+
+-- Create extended view with array field count
+CREATE VIEW currentFieldExtractionsWithArrayCount AS
+SELECT
+    cfe.*,
+    COALESCE(array_counts.arraySize, 0) as arraySize
+FROM currentFieldExtractions cfe
+LEFT JOIN (
+    SELECT
+        fieldExtractionId,
+        COUNT(*) as arraySize
+    FROM documentFieldExtractionArrayFields
+    GROUP BY fieldExtractionId
+) array_counts ON array_counts.fieldExtractionId = cfe.id;
@@ -0,0 +1,4 @@
+-- Remove folder_id column from documentUploads table
+DROP INDEX IF EXISTS idx_documentuploads_folder_id;
+ALTER TABLE documentUploads DROP CONSTRAINT IF EXISTS fk_documentuploads_folder_id;
+ALTER TABLE documentUploads DROP COLUMN IF EXISTS folder_id;
@@ -0,0 +1,6 @@
+-- Add folder_id column to documentUploads table to persist folder association during batch uploads
+ALTER TABLE documentUploads ADD COLUMN folder_id uuid;
+-- Add foreign key constraint referencing folders table
+ALTER TABLE documentUploads ADD CONSTRAINT fk_documentuploads_folder_id FOREIGN KEY (folder_id) REFERENCES folders(id);
+-- Add index for efficient lookup by folder
+CREATE INDEX idx_documentuploads_folder_id ON documentUploads(folder_id);