Merged in feature/postprocessing (pull request #114)

Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests

internal/database/migrations/00000000000005_documents.up.sql

@@ -19,7 +19,6 @@ CREATE TYPE cleanFailType AS ENUM (
     'invalid_read',
     'invalid_read_pages',
     'zero_page_count',
-    'large_page_count',
     'large_file',
     'small_dimensions',
     'large_dimensions',
@@ -50,30 +49,20 @@ CREATE TABLE documentCleanEntries (
     foreign key (cleanId) references documentCleans(id)
 );
 
-CREATE TABLE documentTextTriggerExtractions (
-    id uuid primary key DEFAULT uuid_generate_v7(),
-    cleanId uuid not null,
-    version bigint not null,
-    createdAt timestamp not null,
-    part unsignedsmallint not null,
-    textractJobId text,
-    foreign key (cleanId) references documentCleans(id)
-);
-
 CREATE TABLE documentTextExtractions (
     id uuid primary key DEFAULT uuid_generate_v7(),
+    cleanId uuid not null,
     bucket text not null,
     key text not null,
     hash text not null,
     createdAt timestamp not null,
-    part unsignedsmallint not null
+    part unsignedsmallint not null,
+    foreign key (cleanId) references documentCleans(id)
 );
 
 CREATE TABLE documentTextExtractionEntries (
     id uuid primary key DEFAULT uuid_generate_v7(),
     textId uuid not null,
-    triggerId uuid not null,
     version bigint not null,
-    foreign key (triggerId) references documentTextTriggerExtractions(id),
     foreign key (textId) references documentTextExtractions(id)
 );

internal/database/migrations/00000000000102_document_views.up.sql

@@ -54,25 +54,20 @@ WHERE row_num = 1;
 CREATE VIEW currentTextEntries as
 WITH RankedExtractions AS (
     SELECT
-        dte.id,
+        extract.id,
         cc.documentId,
-        dte.bucket,
-        dte.key,
-        dte.hash,
-        trigger.id as triggerId,
-        trigger.cleanId,
-        trigger.textractJobId,
-        trigger.version as triggerVersion,
+        extract.bucket,
+        extract.key,
+        extract.hash,
+        extract.cleanId,
         dtee.version as extractionVersion,
-        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num
+        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
     FROM documents d
     JOIN currentCleanEntries cc on cc.documentId = d.id
     JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
-    JOIN documentTextTriggerExtractions trigger ON trigger.cleanId = cc.id
-        AND trigger.version >= ctv.minTextVersion
-    JOIN documentTextExtractionEntries dtee on dtee.triggerId = trigger.id
+    JOIN documentTextExtractions extract ON extract.cleanId = cc.id
+    JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
         AND dtee.version >= ctv.minTextVersion
-    JOIN documentTextExtractions dte ON dte.id = dtee.textId
 )
 SELECT
     id,
@@ -81,9 +76,6 @@ SELECT
     key,
     hash,
     cleanId,
-    triggerId,
-    textractJobId,
-    triggerVersion,
     extractionVersion
 FROM RankedExtractions
 WHERE row_num = 1;