Merged in feature/postprocessing (pull request #114)

Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests

api/clientSyncRunner/runner.go

@@ -5,8 +5,6 @@ import (
 	"log/slog"
 
 	clientsync "queryorchestration/internal/client/sync"
-
-	"github.com/go-playground/validator/v10"
 )
 
 const Name = "clientSyncRunner"
@@ -16,14 +14,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/clientSyncRunner/runner_test.go

@@ -13,7 +13,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -41,7 +40,7 @@ func TestQueryRunner(t *testing.T) {
 
 	svc := clientsync.New(cfg)
 
-	runner := clientsyncrunner.New(validator.New(), &clientsyncrunner.Services{
+	runner := clientsyncrunner.New(&clientsyncrunner.Services{
 		ClientSync: svc,
 	})
 	assert.NotNil(t, runner)

api/docCleanRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	documentclean "queryorchestration/internal/document/clean"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/docCleanRunner/runner_test.go

@@ -20,7 +20,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -49,7 +48,7 @@ func TestDocCleanRunner(t *testing.T) {
 	cfg.QueueClient = mockSQS
 	cfg.DocumentTextTriggerURL = "/i/am/here"
 
-	runner := doccleanrunner.New(validator.New(), &doccleanrunner.Services{
+	runner := doccleanrunner.New(&doccleanrunner.Services{
 		Clean: documentclean.New(cfg),
 	})
 	assert.NotNil(t, runner)

api/docInitRunner/runner.go

@@ -6,8 +6,6 @@ import (
 
 	documentinit "queryorchestration/internal/document/init"
 	"queryorchestration/internal/serviceconfig/objectstore"
-
-	"github.com/go-playground/validator/v10"
 )
 
 const Name = "docInitRunner"
@@ -17,14 +15,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/docInitRunner/runner_test.go

@@ -16,7 +16,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -41,7 +40,7 @@ func TestDocInitRunner(t *testing.T) {
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
 
-	runner := docinitrunner.New(validator.New(), &docinitrunner.Services{
+	runner := docinitrunner.New(&docinitrunner.Services{
 		Document: documentinit.New(cfg),
 	})
 	assert.NotNil(t, runner)

api/docSyncRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	documentsync "queryorchestration/internal/document/sync"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/docSyncRunner/runner_test.go

@@ -15,7 +15,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -40,7 +39,7 @@ func TestDocInitRunner(t *testing.T) {
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
 
-	runner := docsyncrunner.New(validator.New(), &docsyncrunner.Services{
+	runner := docsyncrunner.New(&docsyncrunner.Services{
 		Document: documentsync.New(cfg, &documentsync.Services{
 			Document: document.New(cfg),
 			Client:   client.New(cfg),

api/docTextProcessRunner/runner.go

@@ -1,55 +0,0 @@
-package doctextprocessrunner
-
-import (
-	"context"
-	"log/slog"
-
-	documenttext "queryorchestration/internal/document/text"
-	"queryorchestration/internal/serviceconfig/objectstore"
-
-	"github.com/go-playground/validator/v10"
-)
-
-const Name = "docTextProcessRunner"
-
-type Services struct {
-	Text *documenttext.Service
-}
-
-type Runner struct {
-	validator *validator.Validate
-	svc       *Services
-}
-
-func New(validator *validator.Validate, svc *Services) Runner {
-	return Runner{
-		validator: validator,
-		svc:       svc,
-	}
-}
-
-type Body struct {
-	Bucket string `json:"bucket" validate:"required"`
-	Key    string `json:"key" validate:"required"`
-	Hash   string `json:"hash" validate:"required"`
-}
-
-func (s Runner) Process(ctx context.Context, body Body) bool {
-	key, err := objectstore.ParseBucketKey(body.Key)
-	if err != nil {
-		slog.Error("unable to parse key", "key", body.Key)
-		return true
-	}
-
-	err = s.svc.Text.Process(ctx, &documenttext.ProcessParams{
-		Bucket: body.Bucket,
-		Key:    key,
-		Hash:   body.Hash,
-	})
-	if err != nil {
-		slog.Error("unable to process", "bucket", body.Bucket, "key", body.Key, "hash", body.Hash, "error", err)
-		return false
-	}
-
-	return true
-}

api/docTextProcessRunner/runner_test.go

@@ -1,155 +0,0 @@
-package doctextprocessrunner
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/document"
-	documenttext "queryorchestration/internal/document/text"
-	"queryorchestration/internal/server/runner"
-	"queryorchestration/internal/serviceconfig/aws"
-	"queryorchestration/internal/serviceconfig/objectstore"
-	"queryorchestration/internal/serviceconfig/queue/querysync"
-	"queryorchestration/internal/serviceconfig/textract"
-	objectstoremock "queryorchestration/mocks/objectstore"
-	queuemock "queryorchestration/mocks/queue"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
-	"github.com/google/uuid"
-	"github.com/pashagolub/pgxmock/v3"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-type DocInitConfig struct {
-	runner.BaseConfig[Body]
-	aws.AWSConfig
-	textract.TextractConfig
-	querysync.QuerySyncConfig
-	objectstore.ObjectStoreConfig
-}
-
-func TestDocTextProcessRunner(t *testing.T) {
-	ctx := context.Background()
-
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocInitConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
-
-	runner := New(validator.New(), &Services{
-		Text: documenttext.New(cfg),
-	})
-	assert.NotNil(t, runner)
-
-	t.Run("pass through", func(t *testing.T) {
-		triggerId := uuid.New()
-		docId := uuid.New()
-		textractId := "heeloo"
-		textId := uuid.New()
-		cleanId := uuid.New()
-		bucketName := "bucketName"
-		docinfo := document.DocumentSummary{
-			ID:       uuid.New(),
-			ClientID: "hello",
-			Hash:     `"5d41402abc4b2a76b9719d911017c592"`,
-		}
-		key := objectstore.BucketKey{
-			ClientID:  docinfo.ClientID,
-			Location:  objectstore.TextTextract,
-			EntityID:  triggerId,
-			CreatedAt: time.Now().UTC(),
-		}
-		doc := Body{
-			Bucket: bucketName,
-			Key:    key.String(),
-			Hash:   docinfo.Hash,
-		}
-
-		pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).WillReturnRows(
-			pgxmock.NewRows([]string{"id", "cleanId", "version", "textractJobId", "documentId"}).
-				AddRow(triggerId, cleanId, int64(123), &textractId, docId),
-		)
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, docinfo.Hash).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).
-				AddRow(textId),
-		)
-		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		body := io.NopCloser(strings.NewReader("hello"))
-		mockS3.EXPECT().
-			GetObject(
-				mock.Anything,
-				mock.MatchedBy(func(in *s3.GetObjectInput) bool {
-					return *in.IfMatch == `"5d41402abc4b2a76b9719d911017c592"`
-				}),
-				mock.Anything,
-			).
-			Return(&s3.GetObjectOutput{
-				Body: body,
-			}, nil)
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		assert.True(t, runner.Process(ctx, doc))
-	})
-	t.Run("invalid key", func(t *testing.T) {
-		key := objectstore.BucketKey{}
-		doc := Body{
-			Key: key.String(),
-		}
-
-		assert.True(t, runner.Process(ctx, doc))
-	})
-	t.Run("unable to process", func(t *testing.T) {
-		triggerId := uuid.New()
-		bucketName := "bucketName"
-		docinfo := document.DocumentSummary{
-			ID:       uuid.New(),
-			ClientID: "hello",
-			Hash:     `"5d41402abc4b2a76b9719d911017c592"`,
-		}
-		key := objectstore.BucketKey{
-			ClientID:  docinfo.ClientID,
-			Location:  objectstore.TextTextract,
-			EntityID:  triggerId,
-			CreatedAt: time.Now().UTC(),
-		}
-		doc := Body{
-			Bucket: bucketName,
-			Key:    key.String(),
-			Hash:   docinfo.Hash,
-		}
-
-		pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).
-			WillReturnError(errors.New("db fail"))
-
-		assert.False(t, runner.Process(ctx, doc))
-	})
-}

api/docTextRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	documenttext "queryorchestration/internal/document/text"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/docTextRunner/runner_test.go

@@ -2,6 +2,11 @@ package doctextrunner_test
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,8 +22,9 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 	textractmock "queryorchestration/mocks/textract"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -36,84 +42,169 @@ type DocTextConfig struct {
 
 func TestDocCleanRunner(t *testing.T) {
 	ctx := context.Background()
-
 	pool, err := pgxmock.NewPool()
 	require.NoError(t, err)
 
 	cfg := &DocTextConfig{}
 	cfg.DBPool = pool
 	cfg.DBQueries = repository.New(pool)
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	cfg.QuerySyncURL = "/i/am/here"
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
 	mockTextract := textractmock.NewMockTextractClient(t)
 	cfg.TextractClient = mockTextract
+	mockQueue := queuemock.NewMockSQSClient(t)
+	cfg.QueueClient = mockQueue
+	cfg.QuerySyncURL = "ex"
 
-	runner := doctextrunner.New(validator.New(), &doctextrunner.Services{
+	runner := doctextrunner.New(&doctextrunner.Services{
 		Text: documenttext.New(cfg),
 	})
 	assert.NotNil(t, runner)
+	doc := doctextrunner.Body{
+		DocumentID: uuid.New(),
+	}
 
-	t.Run("valid", func(t *testing.T) {
-		doc := doctextrunner.Body{
-			DocumentID: uuid.New(),
-		}
+	cleanId := uuid.New()
+	bucket := "bucket"
+	hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
+	textId := uuid.New()
+	key := objectstore.BucketKey{
+		ClientID:  "aa",
+		Location:  objectstore.Import,
+		CreatedAt: time.Now().UTC(),
+		EntityID:  cleanId,
+	}
+	keyStr := key.String()
+	filename := "../../assets/sampleGeneration/generated/helloWorld.gen"
+	file, err := os.Open(filename)
+	require.NoError(t, err)
+	defer file.Close()
 
-		cleanId := uuid.New()
-		bucket := "hello"
-		key := objectstore.BucketKey{
-			CreatedAt: time.Now().UTC(),
-			ClientID:  "hi",
-			EntityID:  uuid.New(),
-			Location:  objectstore.Import,
-		}
-		keyStr := key.String()
-		hash := "example"
+	decoder := json.NewDecoder(file)
+	var textractBaseOut map[string][]awstextract.AnalyzeDocumentOutput
+	err = decoder.Decode(&textractBaseOut)
+	require.NoError(t, err)
 
-		pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
-			pgxmock.NewRows([]string{"isextracted"}).
-				AddRow(false),
+	mimetype := repository.NullCleanmimetype{
+		Valid:         true,
+		Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+	}
+	pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
+		pgxmock.NewRows([]string{"isextracted"}).
+			AddRow(false),
+	)
+	pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
+				AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
 		)
-		mimeType := "application/pdf"
-		dbmimetype := repository.NullCleanmimetype{
-			Valid:         true,
-			Cleanmimetype: repository.Cleanmimetype(mimeType),
-		}
-		pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).WillReturnRows(
-			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
-				AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, key.ClientID),
-		)
-		jobId := "hello"
-		triggerId := uuid.New()
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&key.ClientID, pgxmock.AnyArg()).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"part", "count"}).
-					AddRow(0, 2),
-			)
-
-		pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
-		)
-
-		mockTextract.EXPECT().
-			StartDocumentTextDetection(
-				mock.Anything,
-				mock.MatchedBy(func(in *awstextract.StartDocumentTextDetectionInput) bool {
+	bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
+	mockStore.EXPECT().
+		GetObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == keyStr
+			}),
+			mock.Anything,
+		).
+		Return(&s3.GetObjectOutput{
+			Body: bodyMsg,
+		}, nil)
+	sent := 0
+	mockTextract.EXPECT().
+		AnalyzeDocument(
+			mock.Anything,
+			mock.MatchedBy(func(in *awstextract.AnalyzeDocumentInput) bool {
+				if sent == 0 {
+					sent++
 					return true
-				}),
-				mock.Anything,
-			).
-			Return(&awstextract.StartDocumentTextDetectionOutput{
-				JobId: &jobId,
-			}, nil)
+				}
+				return false
+			}),
+			mock.Anything,
+		).
+		Return(&textractBaseOut["base"][0], nil).Once()
 
-		pool.ExpectExec("name: AddDocumentTextTriggerJobId :exec").WithArgs(&jobId, triggerId).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}).
+			AddRow(textId),
+	)
+	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
 
-		assert.True(t, runner.Process(ctx, doc))
-	})
+	mockQueue.EXPECT().
+		SendMessage(
+			mock.Anything,
+			mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+				return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.DocumentID.String())
+			}),
+			mock.Anything,
+		).
+		Return(&sqs.SendMessageOutput{}, nil)
+
+	assert.True(t, runner.Process(ctx, doc))
 }
+
+const pdfHelloWorld = `%PDF-1.4
+%����
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+  /Version /1.4
+>>
+endobj
+2 0 obj
+<<
+  /Type /Pages
+  /Kids [3 0 R]
+  /Count 1
+>>
+endobj
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /MediaBox [0 0 612 792]
+  /Resources <<
+    /Font <<
+      /F1 <<
+        /Type /Font
+        /Subtype /Type1
+        /BaseFont /Helvetica
+      >>
+    >>
+  >>
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj
+<<
+  /Length
+  44
+>>
+stream
+BT
+/F1 24 Tf
+100 700 Td
+(Hello World!) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000015 00000 n
+0000000086 00000 n
+0000000151 00000 n
+0000000376 00000 n
+trailer
+<<
+  /Size 5
+  /Root 1 0 R
+>>
+startxref
+472
+%%EOF`

api/queryRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	resultset "queryorchestration/internal/query/result/set"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/queryRunner/runner_test.go

@@ -17,7 +17,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -44,7 +43,7 @@ func TestQueryRunner(t *testing.T) {
 	cfg.QueryURL = "/i/am/here"
 
 	que := query.New(cfg)
-	runner := queryrunner.New(validator.New(), &queryrunner.Services{
+	runner := queryrunner.New(&queryrunner.Services{
 		ResultSet: resultset.New(cfg, &resultset.Services{
 			Query: que,
 			Result: result.New(cfg, &result.Services{
@@ -76,8 +75,8 @@ func TestQueryRunner(t *testing.T) {
 		textEntryId := uuid.New()
 		pool.ExpectQuery("name: GetTextEntryByDocId :one").WithArgs(params.DocumentID).
 			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "hash", "cleanId", "triggerId", "textractJobId", "triggerVersion", "extractionVersion"}).
-					AddRow(textEntryId, params.DocumentID, "buket", "/i/am/here", "example", uuid.New(), uuid.New(), "jobId", int64(123), int64(543)),
+				pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "hash", "cleanId", "extractionVersion"}).
+					AddRow(textEntryId, params.DocumentID, "buket", "/i/am/here", "example", uuid.New(), int64(543)),
 			)
 		pool.ExpectQuery("name: GetQuery :one").WithArgs(query.ID).WillReturnRows(
 			pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).

api/querySyncRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	querysync "queryorchestration/internal/query/sync"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/querySyncRunner/runner_test.go

@@ -14,7 +14,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -44,7 +43,7 @@ func TestQueryRunner(t *testing.T) {
 		ResultSync: resultsync.New(cfg),
 	})
 
-	runner := querysyncrunner.New(validator.New(), &querysyncrunner.Services{
+	runner := querysyncrunner.New(&querysyncrunner.Services{
 		QuerySync: svc,
 	})
 	assert.NotNil(t, runner)

api/queryVersionSyncRunner/runner.go

@@ -6,7 +6,6 @@ import (
 
 	queryversionsync "queryorchestration/internal/query/versionsync"
 
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 )
 
@@ -17,14 +16,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/queryVersionSyncRunner/runner_test.go

@@ -13,7 +13,6 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -41,7 +40,7 @@ func TestQueryRunner(t *testing.T) {
 
 	svc := queryversionsync.New(cfg)
 
-	runner := queryversionsyncrunner.New(validator.New(), &queryversionsyncrunner.Services{
+	runner := queryversionsyncrunner.New(&queryversionsyncrunner.Services{
 		Sync: svc,
 	})
 	assert.NotNil(t, runner)

api/storeEventRunner/runner.go

@@ -5,8 +5,6 @@ import (
 	"log/slog"
 
 	documentstore "queryorchestration/internal/document/store"
-
-	"github.com/go-playground/validator/v10"
 )
 
 const Name = "storeEventRunner"
@@ -16,14 +14,12 @@ type Services struct {
 }
 
 type Runner struct {
-	validator *validator.Validate
-	svc       *Services
+	svc *Services
 }
 
-func New(validator *validator.Validate, svc *Services) Runner {
+func New(svc *Services) Runner {
 	return Runner{
-		validator: validator,
-		svc:       svc,
+		svc: svc,
 	}
 }
 

api/storeEventRunner/runner_test.go

@@ -11,11 +11,9 @@ import (
 	"queryorchestration/internal/server/runner"
 	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/documentinit"
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/assert"
@@ -26,7 +24,6 @@ import (
 type DocInitConfig struct {
 	runner.BaseConfig[S3EventNotification]
 	documentinit.DocInitConfig
-	documenttextprocess.DocTextProcessConfig
 }
 
 func TestDocInitRunner(t *testing.T) {
@@ -41,9 +38,8 @@ func TestDocInitRunner(t *testing.T) {
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
 	cfg.DocInitURL = "hi"
-	cfg.DocumentTextProcessURL = "hi"
 
-	runner := New(validator.New(), &Services{
+	runner := New(&Services{
 		documentstore.New(cfg),
 	})
 	assert.NotNil(t, runner)

assets/sampleGeneration/generated/helloWorld.gen

@@ -1,192 +1,248 @@
 {
-	"Blocks": [
+	"base": [
 		{
-			"BlockType": "PAGE",
-			"ColumnIndex": null,
-			"ColumnSpan": null,
-			"Confidence": null,
-			"EntityTypes": null,
-			"Geometry": {
-				"BoundingBox": {
-					"Height": 1,
-					"Left": 0,
-					"Top": 0,
-					"Width": 1
-				},
-				"Polygon": [
-					{
-						"X": 0.0000010227642,
-						"Y": 0
-					},
-					{
-						"X": 1,
-						"Y": 0.0001285361
-					},
-					{
-						"X": 1,
-						"Y": 1
-					},
-					{
-						"X": 0,
-						"Y": 1
-					}
-				]
-			},
-			"Id": "cf623e5d-d3f1-4aa8-9255-970b0a178924",
-			"Page": null,
-			"Query": null,
-			"Relationships": [
+			"AnalyzeDocumentModelVersion": "1.0",
+			"Blocks": [
 				{
-					"Ids": [
-						"44a179f8-88e8-4cdd-aebf-96be77e681e0"
+					"BlockType": "PAGE",
+					"ColumnIndex": null,
+					"ColumnSpan": null,
+					"Confidence": null,
+					"EntityTypes": null,
+					"Geometry": {
+						"BoundingBox": {
+							"Height": 1,
+							"Left": 0,
+							"Top": 0,
+							"Width": 1
+						},
+						"Polygon": [
+							{
+								"X": 0.0000011078263,
+								"Y": 0
+							},
+							{
+								"X": 1,
+								"Y": 0.00008426185
+							},
+							{
+								"X": 1,
+								"Y": 1
+							},
+							{
+								"X": 0,
+								"Y": 1
+							}
+						]
+					},
+					"Id": "c811b895-fdd3-45b2-aaea-9b03ac08e6c7",
+					"Page": null,
+					"Query": null,
+					"Relationships": [
+						{
+							"Ids": [
+								"fb57ceba-2d5a-4334-bc58-5ae5c7cd1a82",
+								"fdc70cec-80dc-4833-81ac-ca919e28981e"
+							],
+							"Type": "CHILD"
+						}
 					],
-					"Type": "CHILD"
+					"RowIndex": null,
+					"RowSpan": null,
+					"SelectionStatus": "",
+					"Text": null,
+					"TextType": ""
+				},
+				{
+					"BlockType": "LINE",
+					"ColumnIndex": null,
+					"ColumnSpan": null,
+					"Confidence": 99.73413,
+					"EntityTypes": null,
+					"Geometry": {
+						"BoundingBox": {
+							"Height": 0.025445992,
+							"Left": 0.16460413,
+							"Top": 0.092252895,
+							"Width": 0.210229
+						},
+						"Polygon": [
+							{
+								"X": 0.16462424,
+								"Y": 0.092252895
+							},
+							{
+								"X": 0.3748331,
+								"Y": 0.092557475
+							},
+							{
+								"X": 0.37481487,
+								"Y": 0.117698886
+							},
+							{
+								"X": 0.16460413,
+								"Y": 0.11739321
+							}
+						]
+					},
+					"Id": "fb57ceba-2d5a-4334-bc58-5ae5c7cd1a82",
+					"Page": null,
+					"Query": null,
+					"Relationships": [
+						{
+							"Ids": [
+								"979fd840-9770-44c6-92ce-0414e9de1ce3",
+								"007d447f-af28-4535-8416-b8268650e578"
+							],
+							"Type": "CHILD"
+						}
+					],
+					"RowIndex": null,
+					"RowSpan": null,
+					"SelectionStatus": "",
+					"Text": "Hello World!",
+					"TextType": ""
+				},
+				{
+					"BlockType": "WORD",
+					"ColumnIndex": null,
+					"ColumnSpan": null,
+					"Confidence": 99.91676,
+					"EntityTypes": null,
+					"Geometry": {
+						"BoundingBox": {
+							"Height": 0.025081681,
+							"Left": 0.16460413,
+							"Top": 0.0924382,
+							"Width": 0.087125756
+						},
+						"Polygon": [
+							{
+								"X": 0.1646241,
+								"Y": 0.0924382
+							},
+							{
+								"X": 0.25172988,
+								"Y": 0.09256441
+							},
+							{
+								"X": 0.25171068,
+								"Y": 0.11751988
+							},
+							{
+								"X": 0.16460413,
+								"Y": 0.11739321
+							}
+						]
+					},
+					"Id": "979fd840-9770-44c6-92ce-0414e9de1ce3",
+					"Page": null,
+					"Query": null,
+					"Relationships": null,
+					"RowIndex": null,
+					"RowSpan": null,
+					"SelectionStatus": "",
+					"Text": "Hello",
+					"TextType": "PRINTED"
+				},
+				{
+					"BlockType": "WORD",
+					"ColumnIndex": null,
+					"ColumnSpan": null,
+					"Confidence": 99.55151,
+					"EntityTypes": null,
+					"Geometry": {
+						"BoundingBox": {
+							"Height": 0.024770401,
+							"Left": 0.26289847,
+							"Top": 0.09239531,
+							"Width": 0.11193464
+						},
+						"Polygon": [
+							{
+								"X": 0.2629173,
+								"Y": 0.09239531
+							},
+							{
+								"X": 0.3748331,
+								"Y": 0.092557475
+							},
+							{
+								"X": 0.37481526,
+								"Y": 0.117165715
+							},
+							{
+								"X": 0.26289847,
+								"Y": 0.11700299
+							}
+						]
+					},
+					"Id": "007d447f-af28-4535-8416-b8268650e578",
+					"Page": null,
+					"Query": null,
+					"Relationships": null,
+					"RowIndex": null,
+					"RowSpan": null,
+					"SelectionStatus": "",
+					"Text": "World!",
+					"TextType": "PRINTED"
+				},
+				{
+					"BlockType": "LAYOUT_TEXT",
+					"ColumnIndex": null,
+					"ColumnSpan": null,
+					"Confidence": 48.242188,
+					"EntityTypes": null,
+					"Geometry": {
+						"BoundingBox": {
+							"Height": 0.025445992,
+							"Left": 0.16460413,
+							"Top": 0.092252895,
+							"Width": 0.210229
+						},
+						"Polygon": [
+							{
+								"X": 0.16462424,
+								"Y": 0.092252895
+							},
+							{
+								"X": 0.3748331,
+								"Y": 0.092557475
+							},
+							{
+								"X": 0.37481487,
+								"Y": 0.117698886
+							},
+							{
+								"X": 0.16460413,
+								"Y": 0.11739321
+							}
+						]
+					},
+					"Id": "fdc70cec-80dc-4833-81ac-ca919e28981e",
+					"Page": null,
+					"Query": null,
+					"Relationships": [
+						{
+							"Ids": [
+								"fb57ceba-2d5a-4334-bc58-5ae5c7cd1a82"
+							],
+							"Type": "CHILD"
+						}
+					],
+					"RowIndex": null,
+					"RowSpan": null,
+					"SelectionStatus": "",
+					"Text": null,
+					"TextType": ""
 				}
 			],
-			"RowIndex": null,
-			"RowSpan": null,
-			"SelectionStatus": "",
-			"Text": null,
-			"TextType": ""
-		},
-		{
-			"BlockType": "LINE",
-			"ColumnIndex": null,
-			"ColumnSpan": null,
-			"Confidence": 99.89616,
-			"EntityTypes": null,
-			"Geometry": {
-				"BoundingBox": {
-					"Height": 0.024213256,
-					"Left": 0.16538842,
-					"Top": 0.092992365,
-					"Width": 0.20915127
-				},
-				"Polygon": [
-					{
-						"X": 0.165407,
-						"Y": 0.092992365
-					},
-					{
-						"X": 0.3745397,
-						"Y": 0.09330509
-					},
-					{
-						"X": 0.37452286,
-						"Y": 0.11720562
-					},
-					{
-						"X": 0.16538842,
-						"Y": 0.11689187
-					}
-				]
+			"DocumentMetadata": {
+				"Pages": 1
 			},
-			"Id": "44a179f8-88e8-4cdd-aebf-96be77e681e0",
-			"Page": null,
-			"Query": null,
-			"Relationships": [
-				{
-					"Ids": [
-						"b2eadea5-a686-48fe-9363-a774a0a146b4",
-						"b5939fe7-0f0d-4acf-8582-d45edc0669f8"
-					],
-					"Type": "CHILD"
-				}
-			],
-			"RowIndex": null,
-			"RowSpan": null,
-			"SelectionStatus": "",
-			"Text": "Hello World!",
-			"TextType": ""
-		},
-		{
-			"BlockType": "WORD",
-			"ColumnIndex": null,
-			"ColumnSpan": null,
-			"Confidence": 99.915146,
-			"EntityTypes": null,
-			"Geometry": {
-				"BoundingBox": {
-					"Height": 0.023926819,
-					"Left": 0.16538842,
-					"Top": 0.093094304,
-					"Width": 0.086175606
-				},
-				"Polygon": [
-					{
-						"X": 0.16540693,
-						"Y": 0.093094304
-					},
-					{
-						"X": 0.25156403,
-						"Y": 0.09322314
-					},
-					{
-						"X": 0.25154623,
-						"Y": 0.11702112
-					},
-					{
-						"X": 0.16538842,
-						"Y": 0.11689187
-					}
-				]
-			},
-			"Id": "b2eadea5-a686-48fe-9363-a774a0a146b4",
-			"Page": null,
-			"Query": null,
-			"Relationships": null,
-			"RowIndex": null,
-			"RowSpan": null,
-			"SelectionStatus": "",
-			"Text": "Hello",
-			"TextType": "PRINTED"
-		},
-		{
-			"BlockType": "WORD",
-			"ColumnIndex": null,
-			"ColumnSpan": null,
-			"Confidence": 99.87717,
-			"EntityTypes": null,
-			"Geometry": {
-				"BoundingBox": {
-					"Height": 0.02385673,
-					"Left": 0.26209888,
-					"Top": 0.09313698,
-					"Width": 0.1124408
-				},
-				"Polygon": [
-					{
-						"X": 0.2621165,
-						"Y": 0.09313698
-					},
-					{
-						"X": 0.3745397,
-						"Y": 0.09330509
-					},
-					{
-						"X": 0.374523,
-						"Y": 0.1169937
-					},
-					{
-						"X": 0.26209888,
-						"Y": 0.116825044
-					}
-				]
-			},
-			"Id": "b5939fe7-0f0d-4acf-8582-d45edc0669f8",
-			"Page": null,
-			"Query": null,
-			"Relationships": null,
-			"RowIndex": null,
-			"RowSpan": null,
-			"SelectionStatus": "",
-			"Text": "World!",
-			"TextType": "PRINTED"
+			"HumanLoopActivationOutput": null,
+			"ResultMetadata": {}
 		}
 	],
-	"DetectDocumentTextModelVersion": "1.0",
-	"DocumentMetadata": {
-		"Pages": 1
-	},
-	"ResultMetadata": {}
+	"table": []
 }

assets/sampleOne/cnc_01-06.txt

@@ -1,134 +1,130 @@
-Document Index
-AMENDMENT NUMBER ONE   1PARTICIPATING PROVIDER AGREEMENT   1
-Attachment A: Medicaid   3
-EXHIBIT 2   3COMPENSATION SCHEDULE   3PROFESSIONAL SERVICES   3
-Additional Provisions:   3
-Definitions:   4
-
-
-
 Start of Page No. = 1
+
 AMENDMENT NUMBER ONE
 PARTICIPATING PROVIDER AGREEMENT
 This Amendment Number One ("Amendment") is entered into as of December 1. 2019 by and between Dummy
-HealthCare Inc. ("Health Plan") and Picture, LLC Novelty Pharmacy
-("Provider"), collectively reterred to
+HealthCare Inc. ("Health Plan") and Picture. LLC Novelty Pharmacy
+("Provider"). collectively referred to
 herein as the "Parties".
 WHEREAS, Health Plan and Provider have previously entered into a Participating Provider Agreement (the
-"Agreement") effective as of January 1. 2017 (defined in the Agreement as the "Effective Date"); and
-WITEREAS, the Parties desire to amend the Agreement;
-NOW THEREFORE, in consideration of the promises and mutual covenants herein contained. the Parties agree as
+"Agreement") effective as of January 1. 2017 (defined in the Agreement as the "Effective Date"). and
+WHEREAS. the Parties desire to amend the Agreement:
+NOW THEREFORE in consideration of the promises and mutual covenants herein contained the Parties agree as
 follows:
-1.
-Attachment A: Medicaid Exhibit 2 Compensation Schedule Professional Services shall be added
-and
-incorporated into the Agreement as shown by the attached.
-2. All other terms and conditions of the Agreement and any amendments thereto, if any, shall remain in full
-force and effect. If the terms of this Amendment conflict with any of the terms of the Agreement. the terms
+I. Attachment A: Medicaid Exhibit 2 Compensation Schedule Professional Services shall be added and
+incorporated into the Agreement as shown by the attached
+2. All other terms and conditions of the Agreement and any amendments thereto. if any. shall remain in full
+force and effect If the terms of this Amendment conflict with any of the terms of the Agreement. the terms
 of this Amendment shall prevail.
-PPA (NE) - All Products 03/30/16
-Page I of 4
+PPA (NF All Products 03/30
+Page of 4
+
 
 Start of Page No. = 2
-IN WITNESS WHEREOF. the Parties hereto have executed and delivered this
+
+IN WITNESS WHEREOF the Parties hereto have executed and delivered this
 Amendment as of the date first set forth above.
 HEALTH PLAN:
-PROVIDER:
 Dummy HealthCare Inc.
+Authorization
+Benth
+PROVIDER
 LLC Novelty Pharmacy
 Authorized Signature
-Authorized Signature
-Bank
-Mark Chappman
 Jake Parelta
-Printed Name:
-Printed Name:
-Title: PRocedent
+"Printed Name:
 Title: RPinCharge
-Date: 11/19/9
 Date: 11/15/19
-ECM #: 112233
-Tax ID Number: 12-3456789
+Tax ID Number 12-3456789
 State Medicaid Number: 0123456789
 National Provider Identifier: 123456789
+Mark Chappman
+Printed Name:
+Title: PRocedent
+Date: 11/19/19
+ECM #: 112233
 PPA (NE) - All Products 03/30/16
-Page 2 of 4
+Page of 4
 
 This page has 2 signature.
 
+
 Start of Page No. = 3
+
 Attachment A: Medicaid
 EXHIBIT 2
 COMPENSATION SCHEDULE
 PROFESSIONAL SERVICES
-Picture,
-LLC Novelty Pharmacy
+Picture, LLC Novelty Pharmacy
 This compensation schedule ("Compensation Schedule") sets forth the maximum reimbursement amounts for
-Covered Services provided by Contracted Providers to Covered Persons enrolled in a Medicaid Product. Where the
+Covered Services provided by Contracted Providers to Covered Persons enrolled in a Medicaid Product Where the
 Contracted Provider's tax identification number ("TIN") has been designated by the Payor as subject to this
-Compensation Schedule. Payor shall pay or arrange for payment of a Clean Claim for Covered Services rendered by
+Compensation Schedule Payor shall pay or arrange for payment of a Clean Claim for Covered Services rendered by
 the Contracted Provider according to the terms of, and subject to the requirements set forth in, the Agreement and
-this Compensation Schedule. Payment under this Compensation Schedule shall consist of the Allowed Amount as set
-forth herein less all applicable Cost-Sharing Amounts. All capitalized terms used in this Compensation Schedule
-shall have the meanings set forth in the Agreement, the applicable Product Attachment. or the Definitions section set
+this Compensation Schedule Payment under this Compensation Schedule shall consist of the Allowed Amount as set
+forth herein less all applicable Cost-Sharing Amounts All capitalized terms used in this Compensation Schedule
+shall have the meanings set forth in the Agreement. the applicable Product Attachment or the Definitions section set
 forth at the end of this Compensation Schedule.
 The maximum compensation for professional Covered Services rendered to a Covered Person shall be the "Allowed
 Amount." Except as otherwise provided in this Compensation Schedule. the Allowed Amount for professional
 Covered Services is the lesser of: (i) Allowable Charges; or (ii) one hundred percent (100%) of the Payor's Medicaid
-fee schedule.
+fee schedule
 Additional Provisions:
 1.
-Code Change Updates. Payor utilizes nationally recognized coding structures (including. without limitation,
-revenue codes, CPT codes, HCPCS codes, ICD codes, national drug codes, ASA relative values, etc., or their
-successors) for basic coding and descriptions of the services rendered. Updates to billing-related codes shall
+Code Change Updates Payor utilizes nationally recognized coding structures (including without limitation,
+revenue codes CPT codes. HCPCS codes, ICD codes. national drug codes. ASA relative values etc., or their
+successors) for basic coding and descriptions of the services rendered Updates to billing-related codes shall
 become effective on the date ("Code Change Effective Date") that is the later of: (i) the first day of the month
 following sixty (60) days after publication by the governmental agency having authority over the applicable
-Product of such governmental agency's acceptance of such code updates, (ii) the effective date of such code
+Product of such governmental agency's acceptance of such code updates. (ii) the effective date of such code
 updates as determined by such governmental agency or (iii) if a date is not established by such governmental
 agency or the applicable Product is not regulated by such governmental agency. the date that changes are made
 to nationally recognized codes. Such updates may include changes to service groupings. Claims processed prior
 to the Code Change Effective Date shall not be reprocessed to reflect any such code updates.
 2.
-Fee Change Updates. Updates to the fee schedule shall become effective on the effective date of such fee
+Fee Change Updates Updates to the fee schedule shall become effective on the effective date of such fee
 schedule updates. as determined by the Payor ("Fee Change Effective Date"). The date of implementation of any
 fee schedule updates. i.e. the date on which such fee change is first used for reimbursement ("Fee Change
 Implementation Date"). shall be the later of: (i) the first date on which Payor is reasonably able to implement the
-update in the claims payment system; or (ii) the Fee Change Effective Date. Claims processed prior to the Fee
-Change implementation Date shall not be reprocessed to reflect any updates to such fee schedule. even if service
+update in the claims payment system; or (ii) the Fee Change Effective Date Claims processed prior to the Fee
+Change Implementation Date shall not be reprocessed to reflect any updates to such fee schedule. even if service
 was provided after the Fee Change Effective Date.
-3.
-Modifier, Unless specifically indicated otherwise, fee amounts listed in the fee schedule represent global fees
-and may be subject to reductions based on appropriate Modifier (for example, professional and technical
-modifiers). As used in the previous sentence, "global fees" refers to services billed without a Modifier, for which
+3. Modifier Unless specifically indicated otherwise fee amounts listed in the fee schedule represent global fees
+and may be subject to reductions based on appropriate Modifier (for example. professional and technical
+modifiers) As used in the previous contence. "global fees" refers to services billed without a Modifier. for which
 the fee amount includes both the professional component and the technical component. Any Cost-Sharing
 Amounts that the Covered Person is responsible to pay under the Coverage Agreement will be subtracted from
 the Allowed Amount in determining the amount to be paid.
-4.
-Anesthesia Modifier Pricing Rules. The dollar amount that will be used in the calculation of time-based and non-
+4. Anesthesia Modifier Pricing Rules, The dollar amount that will be used in the calculation of time-based and non-
 time based anesthesia management fees in accordance with the anesthesia payment policy. Unless specifically
-PPA (NE) - All Products 03/30/16
-Page 3 of 4
+PPA (NE) All Products 03/30/16
+Page of 4
+
 
 Start of Page No. = 4
-stated otherwise, the anesthesia conversion factor indicated is fixed and will not change. The anesthesia
-conversion factor is based on an anesthesia time unit value of 15 minutes.
-5. Payment for Multiple Procedures. Where multiple outpatient surgical or scope procedures performed on a
-Covered Person during a single occasion of surgery, reimbursement will be as follows: i) the procedure for which
+
+stated otherwise. the anesthesia conversion factor indicated is fixed and will not change. The anesthesia
+conversion factor is based on an anesthesia time unit value of 15 minutes
+5. Payment for Multiple Procedures Where multiple outpatient surgical or scope procedures performed on a
+Covered Person during a single occasion of surgery. reimbursement will be as follows: i) the procedure for which
 the Allowed Amount under this Compensation Schedule is greatest will be reimbursed at one hundred percent
-(100%) of such Allowed Amount; and ii) the other procedures under this Compensation Schedule will each be
-reimbursed at fifty percent (50%) of such Allowed Amounts..
-6. Place of Service Pricing Rules. This fee schedule follows CMS guidelines for determining when services are
+(100%) of such Allowed Amount: and ii) the other procedures under this Compensation Schedule will each be
+reimbursed at fifty percent (50%) of such Allowed Amounts.
+6
+Place of Service Pricing Rules This fee schedule follows CMS guidelines for determining when services are
 priced at the facility or non-facility fee schedule (with the exception of services performed at Ambulatory Surgery
-Centers. POS 24, which will be priced at the facility fee schedule).
-7. Payment under this Compensation Schedule. All payments under this Compensation Schedule are subject to the
-terms and conditions set forth in the Agreement. the Provider Manual and any applicable billing manual.
+Centers. POS 24. which will be priced at the facility fee schedule).
+7. Payment under this Compensation Schedule All payments under this Compensation Schedule are subject to the
+terms and conditions set forth in the Agreement. the Provider Manual and any applicable billing manual
 Definitions:
 1. Allowed Amount means the amount designated in this Compensation Schedule as the maximum amount payable
 to a Contracted Provider for any particular Covered Service provided to any particular Covered Person, pursuant
-to this Agreement or its Attachments.
-2. Allowable Charges means a Contracted Provider's billed charges for services that qualify as Covered Services.
-3. Cost-Sharing Amounts means any amounts payable by a Covered Person. such as copayments. cost-sharing,
-coinsurance, deductibles or other amounts that are the Covered Person's financial responsibility under the
-applicable Coverage Agreement. if applicable.
-PPA (NE) - All Products 03/30/16
-Page 4 of 4
+to this Agreement or its Attachments
+2. Allowable Charges means a Contracted Provider's billed charges for services that qualify as Covered Services
+3. Cost-Sharing Amounts means any amounts payable by a Covered Person. such as copayments. cost-sharing
+coinsurance deductibles or other amounts that are the Covered Person's financial responsibility under the
+applicable Coverage Agreement. if applicable
+PPA (NE All Products 03/30/16
+Page of 4
+
+

assets/sampleOne/hn_0109.txt

@@ -1,46 +1,35 @@
-Document Index
-EXHIBIT A-1   2
-COMMERCIAL BENEFIT PROGRAMS   2
-DIRECT NETWORK FEE-FOR-SERVICE   2RATE EXHIBIT   2
-MEDICARE ADVANTAGE PROGRAM   3DIRECT NETWORK FEE-FOR-SERVICE   3RATE EXHIBIT   3
-I. Payment Rates   3
-MEDI-CAL BENEFIT PROGRAM   4DIRECT NETWORK FEE-FOR-SERVICE   4RATE EXHIBIT   4
-
-
-
 Start of Page No. = 1
-7.14
-Status as Independent Entities. None of the provisions of this Agreement is intended to create,
-nor
-shall be deemed or construed to create any relationship between Provider and All Health or a Payor other than
+
+7.14 Status as Independent Entities. None of the provisions of this Agreement is intended to create,
+nor shall be deemed or construed to create any relationship between Provider and All Health or a Payor other than
 that of independent entities contracting with each other solely for the purpose of effecting the provisions of this
-Agreement. Neither Provider nor All Health /Payor, nor any of their respective agents, employees or representatives
-shall be construed to be the agent, employee or representative of the other.
+Agreement Neither Provider nor All Health /Payor. nor any of their respective agents, employees or representatives
+shall be construed to be the agent employee or representative of the other
 7.15
-Addenda. Each Addendum to this Agreement is made a part of this Agreement as though set
-forth fully herein. Any provision of an Addendum that is in conflict with any provision of this Agreement shall take
+Addenda Each Addendum to this Agreement is made a part of this Agreement as though set
+forth fully herein Any provision of an Addendum that is in conflict with any provision of this Agreement shall take
 precedence and supersede the conflicting provision of this Agreement with respect to the subject matter of the
-Addendum.
-7.16
-Calculation of Time. The parties agree that for purposes of calculating time under this
-Agreement, any time period of less than ten (10) days shall be deemed to refer to business days and any time period
+Addendum
+7.16 Calculation of Time. The parties agree that for purposes of calculating time under this
+Agreement any time period of less than ten (10) days shall be deemed to refer to business days and any time period
 of ten (10) days or more shall be deemed to refer to calendar days unless the term "business" precedes the term
-"days".
+"days"
 7.17
-Waiver of Breach. The waiver of any breach of this Agreement by either party shall not
+Waiver of Breach The waiver of any breach of this Agreement by either party shall not
 constitute a continuing waiver of any subsequent breach of either the same or any other provision(s) of this
-Agreement. Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict
-compliance in the future and to exercise any right or remedy related thereto.
+Agreement Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict
+compliance in the future and to exercise any right or remedy related thereto
 THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED
-BY THE PARTIES.
-IN WITNESS WHEREOF, the parties have executed this Agreement.
+BY THE PARTIES
+IN WITNESS WHEREOF the parties have executed this Agreement
 PROVIDER
 ALL HEALTH
-Smoke
+Smith
 Signature
 Health Net Signature
+Peanth
 Harvey
-DUONG M.S.
+DUONG MS
 Steve Carrol
 Print Name
 Print Name
@@ -48,7 +37,7 @@ MEDICAL DIRECTOR
 VP Provider Network Management & Strategy
 Title
 Title
-Dummy Group Name, Inc.
+Dummy Group Name Inc.
 Group Name (If Applicable)
 Date
 3/28/2011
@@ -59,34 +48,34 @@ Effective Date
 2/2/2011
 Date
 California Provider Participation Agreement
-18
-Fee-For-Service Direct Network Template
+For-Service Direct Network Template
 HN-DN-PPA-11-03-2010
+18
 
 This page has 2 signature.
 
+
 Start of Page No. = 2
+
 EXHIBIT A-1
 COMMERCIAL BENEFIT PROGRAMS
 DIRECT NETWORK FEE-FOR-SERVICE
 RATE EXHIBIT
-Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,
+Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E.
 All Health or Payor shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary
-Covered Services delivered under commercial Benefit Programs pursuant to this Addendum, the lesser of: (i) the rates
-listed below, or (ii) 100% of Provider's billed charges.
+Covered Services delivered under commercial Benefit Programs pursuant to this Addendum, the lesser of (i) the rates
+listed below. or (ii) 100% of Provider's billed charges.
+-------Table Start--------
+[["Category of Service","Compensation"],["Covered Services delivered or arranged by Provider. excluding Laboratory services","95% of CMS Allowable"],["Anesthesia Services when provided by an Anesthesiologist or Certified Registered Nurse Anesthetist (American Society of Anesthesiology (ASA) unit scale)","$39 ASA unit"],["Medical/Surgical Services by an Anesthesiologist or Certified Registered Nurse Anesthetist","95% of CMS Allowable"],["Laboratory Services performed in Provider or Professional Provider office","95% of CMS Allowable"],["Pharmaceuticals","95% of the Average Wholesale Price (AWP)"],["OB Services - CPT 59400: Global Obstetric care with vaginal delivery - CPT 59510: Global Obstetric care with cesarean delivery - CPT 59610: Vaginal Delivery after previous cesarean delivery - CPT 59618: Attempted vaginal delivery, resulting in cesarean","$1,700.00 $1,700.00 $1,700.00 $1,700.00"],["Immunizations","95% of the Average Wholesale Price (AWP) as determined by Health Net."],["General Health Panel - CPT 80050: General Health Panel CPT 80055: Obstetric Panel","$20.00 $15.00"],["By Report (BR) Procedures, Procedures not Listed and Procedures with Relativities not Established","75% of billed charges for Covered Services"]]
+-------Table End--------
 California Provider Participation Agreement
-22
 Fee-For-Service Direct Network Template
 HN-DN-PPA-11-03-2010
+22
 
 
--------Table Start--------
-7436be95-25d1-41ad-9abb-1d9e85b85440
-[['Category of Service', 'Compensation'], ['Covered Services delivered or arranged by Provider, excluding Laboratory services', '95% of CMS Allowable'], ['Anesthesia Services when provided by an Anesthesiologist or Certified Registered Nurse Anesthetist (American Society of Anesthesiology (ASA) unit scale)', '$39 / ASA unit'], ['Medical/Surgical Services by an Anesthesiologist or Certified Registered Nurse Anesthetist', '95% of CMS Allowable'], ['Laboratory Services performed in Provider or Professional Provider office', '95% of CMS Allowable'], ['Pharmaceuticals', '95% of the Average Wholesale Price (AWP)'], ['OB Services - CPT 59400: Global Obstetric care with vaginal delivery - CPT 59510: Global Obstetric care with cesarean delivery - CPT 59610: Vaginal Delivery after previous cesarean delivery - CPT 59618: Attempted vaginal delivery, resulting in cesarean', '$1,700.00 $1,700.00 $1,700.00 $1,700.00'], ['Immunizations', '95% of the Average Wholesale Price (AWP) as determined by Health Net.'], ['General Health Panel - CPT 80050: General Health Panel - CPT 80055: Obstetric Panel', '$20.00 $15.00'], ['By Report (BR) Procedures, Procedures not Listed and Procedures with Relativities not Established', '75% of billed charges for Covered Services']]
- DIRECT NETWORK FEE-FOR-SERVICE RATE EXHIBIT
--------Table End--------
-
 Start of Page No. = 3
+
 EXHIBIT B-1
 MEDICARE ADVANTAGE PROGRAM
 DIRECT NETWORK FEE-FOR-SERVICE
@@ -95,20 +84,18 @@ I. Payment Rates
 Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,
 All Health shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered
 Services delivered under Medicare Advantage Benefit Programs pursuant to this Addendum, the lesser of: (i) the rates
-listed below, or (ii) 100% of Provider's billed charges.
-California Provider Participation Agreement
-27
-Fee-For-Service Direct Network Template
-HN-DN-PPA-11-03-2010
-
-
+listed below, or (ii) 100% of Provider's billed charges
 -------Table Start--------
-
-[['Category of Service', 'Compensation'], ['Covered Services delivered or arranged by Provider', '100% of CMS Allowable'], ['General Health Panel - CPT 80050: General Health Panel - CPT 80055: Obstetric Panel', '$20.00 $15.00'], ['By Report (BR) Procedures, Procedures not Listed and Procedures with Relativities not Established', '75% of billed charges for Covered Services']]
-None
+[["Category of Service","Compensation"],["Covered Services delivered or arranged by Provider","100% of CMS Allowable"],["General Health Panel",""],["- CPT 80050 General Health Panel","$20.00"],["- CPT 80055: Obstetric Panel","$15.00"],["By Report (BR) Procedures. Procedures not Listed and Procedures with Relativities not Established","75% of billed charges for Covered Services"]]
 -------Table End--------
+California Provider Participation Agreement
+Fee-Fer-Service Direct Network Template
+HN-DN-PPA-11-03-2010
+27
+
 
 Start of Page No. = 4
+
 EXHIBIT C-1
 MEDI-CAL BENEFIT PROGRAM
 DIRECT NETWORK FEE-FOR-SERVICE
@@ -118,8 +105,10 @@ All Health shall pay and Provider shall accept as payment in full for non-capita
 Services delivered pursuant to this Addendum, the lesser of: 100% of the State of California Medi-Cal Fee Schedule
 rates in effect at the time of service, subject to any adjustments made by the State of California under the applicable
 Medi-Cal Fee-For-Service Program; (ii) Fee-for-service rates for the commercial Benefit Program set forth in
-Addendum A, Exhibit A-1; or (iii) Provider's billed charges.
+Addendum A. Exhibit A-1: or (iii) Provider's billed charges.
 California Provider Participation Agreement
-34
 Fee-For-Service Direct Network Template
-HN-DN-PPA-11-03-2010
+HN-DN-PPA-11-03-2010
+34
+
+

build/Dockerfile

@@ -6,18 +6,30 @@ FROM golang:${GO_VERSION} AS build
 
 WORKDIR /app
 
-# Install Git if not already present
-RUN apt-get update && apt-get install -y --no-install-recommends git=1:2.* && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git=1:2.* musl-tools=1.2.* \
+    && apt-get clean && \
+    \rm -rf /var/lib/apt-get/lists/*
 
-
-# Copy the entire repository including .git directory
 COPY . .
 
+RUN go mod verify
+
 ARG TARGETARCH
 
-SHELL ["/bin/bash", "-c"]
+ENV CGO_ENABLED=1
+ENV GOARCH=${TARGETARCH}
+ENV CC=musl-gcc
+
+# Must be same a github.com/gen2brain/go-fitz in go.mod
+# ENV FZ_VERSION=1.24.14
+
 RUN --mount=type=cache,target=/go/pkg/mod/ \
-    CGO_ENABLED=0 GOARCH=$TARGETARCH go build -o /bin ./cmd/...
+    go build \
+    -mod vendor \
+    -tags musl \
+    -ldflags "-linkmode external -extldflags -static" \
+    -o /bin ./cmd/...
 
 FROM scratch AS final
 

cmd/clientSyncRunner/main.go

@@ -31,7 +31,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[clientsyncrunner.Body] {
 		svc := clientsync.New(cfg)
 
-		c := clientsyncrunner.New(cfg.GetValidator(), &clientsyncrunner.Services{
+		c := clientsyncrunner.New(&clientsyncrunner.Services{
 			ClientSync: svc,
 		})
 

cmd/docCleanRunner/main.go

@@ -35,7 +35,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[doccleanrunner.Body] {
 		clean := documentclean.New(cfg)
 
-		return doccleanrunner.New(cfg.GetValidator(), &doccleanrunner.Services{
+		return doccleanrunner.New(&doccleanrunner.Services{
 			Clean: clean,
 		})
 	}

cmd/docInitRunner/main.go

@@ -41,7 +41,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[docinitrunner.Body] {
 		docinit := documentinit.New(cfg)
 
-		return docinitrunner.New(cfg.GetValidator(), &docinitrunner.Services{
+		return docinitrunner.New(&docinitrunner.Services{
 			Document: docinit,
 		})
 	}

cmd/docSyncRunner/main.go

@@ -42,7 +42,7 @@ func main() {
 			Client:   cli,
 		})
 
-		return docsyncrunner.New(cfg.GetValidator(), &docsyncrunner.Services{
+		return docsyncrunner.New(&docsyncrunner.Services{
 			Document: docsync,
 		})
 	}

cmd/docTextProcessRunner/main.go

@@ -1,66 +0,0 @@
-// **Listens For**: Body Containing Textract Result Document Metadata
-//
-// **Action**.
-//
-// Process textract output.
-//
-// An event with the document id is pushed to the QUERYSYNC queue.
-package main
-
-import (
-	"context"
-	"log/slog"
-	"os"
-
-	doctextprocessrunner "queryorchestration/api/docTextProcessRunner"
-	documenttext "queryorchestration/internal/document/text"
-	"queryorchestration/internal/server/runner"
-	"queryorchestration/internal/serviceconfig/aws"
-	"queryorchestration/internal/serviceconfig/objectstore"
-	"queryorchestration/internal/serviceconfig/queue/querysync"
-	"queryorchestration/internal/serviceconfig/textract"
-
-	_ "github.com/lib/pq"
-)
-
-type DocTextProcessConfig struct {
-	runner.BaseConfig[doctextprocessrunner.Body]
-	aws.AWSConfig
-	textract.TextractConfig
-	querysync.QuerySyncConfig
-	objectstore.ObjectStoreConfig
-}
-
-func main() {
-	ctx := context.Background()
-
-	cfg := &DocTextProcessConfig{}
-
-	cfg.ControllerFunc = func() runner.Controller[doctextprocessrunner.Body] {
-		text := documenttext.New(cfg)
-
-		return doctextprocessrunner.New(cfg.GetValidator(), &doctextprocessrunner.Services{
-			Text: text,
-		})
-	}
-
-	server, err := runner.New(ctx, cfg)
-	if err != nil {
-		slog.Error(err.Error())
-		os.Exit(1)
-	}
-
-	err = cfg.SetStoreClient(ctx)
-	if err != nil {
-		slog.Error(err.Error())
-		os.Exit(1)
-	}
-
-	err = cfg.SetTextractClient(ctx)
-	if err != nil {
-		slog.Error(err.Error())
-		os.Exit(1)
-	}
-
-	server.Listen(ctx)
-}

cmd/docTextRunner/main.go

@@ -37,7 +37,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[doctextrunner.Body] {
 		text := documenttext.New(cfg)
 
-		return doctextrunner.New(cfg.GetValidator(), &doctextrunner.Services{
+		return doctextrunner.New(&doctextrunner.Services{
 			Text: text,
 		})
 	}
@@ -48,6 +48,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	err = cfg.SetStoreClient(ctx)
+	if err != nil {
+		slog.Error(err.Error())
+		os.Exit(1)
+	}
+
 	err = cfg.SetTextractClient(ctx)
 	if err != nil {
 		slog.Error(err.Error())

cmd/queryRunner/main.go

@@ -45,7 +45,7 @@ func main() {
 			Sync:   sync,
 		})
 
-		c := queryrunner.New(cfg.GetValidator(), &queryrunner.Services{
+		c := queryrunner.New(&queryrunner.Services{
 			ResultSet: resset,
 		})
 

cmd/querySyncRunner/main.go

@@ -45,7 +45,7 @@ func main() {
 			ResultSync: sync,
 		})
 
-		c := querysyncrunner.New(cfg.GetValidator(), &querysyncrunner.Services{
+		c := querysyncrunner.New(&querysyncrunner.Services{
 			QuerySync: svc,
 		})
 

cmd/queryVersionSyncRunner/main.go

@@ -31,7 +31,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[queryversionsyncrunner.Body] {
 		svc := queryversionsync.New(cfg)
 
-		c := queryversionsyncrunner.New(cfg.GetValidator(), &queryversionsyncrunner.Services{
+		c := queryversionsyncrunner.New(&queryversionsyncrunner.Services{
 			Sync: svc,
 		})
 

cmd/storeEventRunner/main.go

@@ -4,7 +4,7 @@
 //
 // It will identify the type of event and the location of said event, and forward the task accordingly.
 //
-// It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT and DOCTEXTPROCESS queues.
+// It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT queue.
 package main
 
 import (
@@ -16,7 +16,6 @@ import (
 	documentstore "queryorchestration/internal/document/store"
 	"queryorchestration/internal/server/runner"
 	"queryorchestration/internal/serviceconfig/queue/documentinit"
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
 
 	_ "github.com/lib/pq"
 )
@@ -24,7 +23,6 @@ import (
 type StoreEventConfig struct {
 	runner.BaseConfig[storeeventrunner.S3EventNotification]
 	documentinit.DocInitConfig
-	documenttextprocess.DocTextProcessConfig
 }
 
 func main() {
@@ -35,7 +33,7 @@ func main() {
 	cfg.ControllerFunc = func() runner.Controller[storeeventrunner.S3EventNotification] {
 		store := documentstore.New(cfg)
 
-		return storeeventrunner.New(cfg.GetValidator(), &storeeventrunner.Services{
+		return storeeventrunner.New(&storeeventrunner.Services{
 			Store: store,
 		})
 	}

cmd/textractGenerator_test/textdetection.go

@@ -9,17 +9,40 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	documenttext "queryorchestration/internal/document/text"
+	documenttypes "queryorchestration/internal/document/types"
+	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/serviceconfig/queue/querysync"
 	"queryorchestration/internal/serviceconfig/textract"
 
 	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
 	"github.com/aws/aws-sdk-go-v2/service/textract/types"
 )
 
+type Config struct {
+	serviceconfig.BaseConfig
+	textract.TextractConfig
+	querysync.QuerySyncConfig
+	objectstore.ObjectStoreConfig
+}
+
 func main() {
 	ctx := context.Background()
 
-	cfg := &textract.TextractConfig{}
+	if len(os.Args) != 3 {
+		fmt.Println("Error: expected at least two arguments")
+		fmt.Println("Usage:", os.Args[0], "<sample_name>", "<file_name>")
+		os.Exit(1)
+	}
+
+	sampleName := os.Args[1]
+	name := os.Args[2]
+
+	slog.Info("arguments", "sample_name", sampleName, "file_name", name)
+
+	cfg := &Config{}
 	profile := aws.Profile(os.Getenv("AWS_PROFILE"))
 	if profile == "" {
 		slog.Error("profile not available")
@@ -32,35 +55,105 @@ func main() {
 		os.Exit(1)
 	}
 
-	base := "../../assets/sampleGeneration"
-	name := "helloWorld"
-	filename := fmt.Sprintf("%s/%s.pdf", base, name)
-	content, err := os.ReadFile(filename)
+	root, err := os.OpenRoot("assets")
 	if err != nil {
-		slog.Error("error writing file", "error", err)
+		slog.Error("error opening root", "error", err)
 		os.Exit(1)
 	}
 
-	slog.Info("detecting text", "filename", filename)
-
-	res, err := cfg.GetTextractClient().DetectDocumentText(ctx, &awstextract.DetectDocumentTextInput{
-		Document: &types.Document{
-			Bytes: content,
-		},
-	})
+	root, err = root.OpenRoot(sampleName)
 	if err != nil {
-		slog.Error("error detecting text", "error", err)
+		slog.Error("error opening sample", "error", err)
 		os.Exit(1)
 	}
 
-	jsonContent, err := json.MarshalIndent(res, "", "\t")
+	pdfFile, err := root.Open(fmt.Sprintf("%s.pdf", name))
+	if err != nil {
+		slog.Error("error opening pdf file", "error", err)
+		os.Exit(1)
+	}
+
+	pdf, err := documenttypes.NewPDFFromReader(pdfFile)
+	if err != nil {
+		slog.Error("error creating pdf file", "error", err)
+		os.Exit(1)
+	}
+
+	textSvc := documenttext.New(cfg)
+
+	count, err := pdf.GetPageCount(ctx)
+	if err != nil {
+		slog.Error("error getting page count", "error", err)
+		os.Exit(1)
+	}
+
+	slog.Info("detecting text")
+
+	allResults := map[string][]awstextract.AnalyzeDocumentOutput{}
+	allResults["base"] = make([]awstextract.AnalyzeDocumentOutput, count)
+	allResults["table"] = []awstextract.AnalyzeDocumentOutput{}
+	for i := range count {
+		result, err := textSvc.GetBasePage(ctx, pdf, i)
+		if err != nil {
+			slog.Error("error detecting text", "error", err)
+			os.Exit(1)
+		}
+
+		for _, block := range result.Blocks {
+			feature := textSvc.GetBlockFeature(block)
+			if feature == documenttext.FeatureTable {
+				pageBytes, err := pdf.GetPageAsPNG(ctx, i)
+				if err != nil {
+					slog.Error("error getting page", "error", err)
+					os.Exit(1)
+				}
+
+				res, err := cfg.GetTextractClient().AnalyzeDocument(ctx, &awstextract.AnalyzeDocumentInput{
+					Document: &types.Document{
+						Bytes: pageBytes,
+					},
+					FeatureTypes: []types.FeatureType{
+						types.FeatureTypeTables,
+					},
+				})
+				if err != nil {
+					slog.Error("error detecting table", "error", err)
+					os.Exit(1)
+				}
+				allResults["table"] = append(allResults["table"], *res)
+			}
+		}
+
+		allResults["base"][i] = result
+	}
+
+	generatedBase := "generated"
+
+	if _, err := root.Stat(generatedBase); os.IsNotExist(err) {
+		err := root.Mkdir(generatedBase, 0755)
+		if err != nil {
+			slog.Error("error creating directory", "error", err)
+			os.Exit(1)
+		}
+
+		slog.Info("directory created", "directory", generatedBase)
+	}
+
+	jsonContent, err := json.MarshalIndent(allResults, "", "\t")
 	if err != nil {
 		slog.Error("error marshaling json", "error", err)
 		os.Exit(1)
 	}
 
-	outputFilename := fmt.Sprintf("%s/generated/%s.gen", base, name)
-	err = os.WriteFile(outputFilename, jsonContent, 0600)
+	outputFilename := fmt.Sprintf("%s/%s.gen", generatedBase, name)
+	file, err := root.Create(outputFilename)
+	if err != nil {
+		slog.Error("error creating file", "error", err)
+		os.Exit(1)
+	}
+	defer file.Close()
+
+	_, err = file.Write(jsonContent)
 	if err != nil {
 		slog.Error("error writing file", "error", err)
 		os.Exit(1)

deployments/compose.local.yaml

@@ -156,37 +156,6 @@ services:
     networks:
       - server-network
 
-  doc_text_process_runner:
-    image: queryorchestration:latest
-    command: ["./docTextProcessRunner"]
-    depends_on:
-      localstack:
-        condition: service_healthy
-      db:
-        condition: service_healthy
-    ports:
-      - "8086:8080"
-    expose:
-      - 8080
-    environment:
-      LOG_LEVEL: DEBUG
-      QUEUE_URL: ${DOCUMENT_TEXT_PROCESS_URL}
-      QUERY_SYNC_URL: ${QUERY_SYNC_URL}
-      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
-      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
-      AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
-      AWS_REGION: ${AWS_REGION}
-      PGUSER: ${PGUSER}
-      PGPASSWORD: ${PGPASSWORD}
-      PGHOST: db
-      PGPORT: 5432
-      PGDATABASE: ${PGDATABASE}
-      DB_NOSSL: ${DB_NOSSL}
-      AWS_ENDPOINT_URL: "http://localstack:4566"
-      AWS_S3_USE_PATH_STYLE: true
-    networks:
-      - server-network
-
   query_sync_runner:
     image: queryorchestration:latest
     command: ["./querySyncRunner"]

docs/README.md

@@ -48,16 +48,6 @@ By validating: the client can_sync parameter.
 
 If eligible, send an event to the DOCCLEAN queue.
 
-## docTextProcessRunner
-
-**Listens For**: Body Containing Textract Result Document Metadata
-
-**Action**.
-
-Process textract output.
-
-An event with the document id is pushed to the QUERYSYNC queue.
-
 ## docTextRunner
 
 **Listens For**: Body Containing Document ID
@@ -144,6 +134,6 @@ Adds an event for each client into the CLIENTSYNC queue for the given query, i.e
 
 It will identify the type of event and the location of said event, and forward the task accordingly.
 
-It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT and DOCTEXTPROCESS queues.
+It supports the document initialisation and document text processing stages. Events may be pushed to the DOCINIT queue.
 
 Generated by Doczy

go.mod

@@ -9,9 +9,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.38.1
 	github.com/aws/aws-sdk-go-v2/service/textract v1.35.1
 	github.com/caarlos0/env/v11 v11.3.1
+	github.com/gen2brain/go-fitz v1.24.14
 	github.com/getkin/kin-openapi v0.129.0
 	github.com/go-playground/validator/v10 v10.25.0
 	github.com/golang-migrate/migrate/v4 v4.18.2
+	github.com/google/go-cmp v0.7.0
 	github.com/jackc/pgx/v5 v5.7.2
 	github.com/joho/godotenv v1.5.1
 	github.com/labstack/echo-contrib v0.17.2
@@ -40,6 +42,7 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chigopher/pathlib v0.19.1 // indirect
+	github.com/ebitengine/purego v0.8.0 // indirect
 	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
@@ -64,11 +67,13 @@ require (
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
+	github.com/jupiterrider/ffi v0.2.0 // indirect
 	github.com/labstack/gommon v0.4.2 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
@@ -81,6 +86,7 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rs/zerolog v1.33.0 // indirect
 	github.com/sagikazarmark/locafero v0.7.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -140,7 +146,7 @@ require (
 	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/docker v28.0.1+incompatible // indirect
+	github.com/docker/docker v28.0.1+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect

go.sum

@@ -86,6 +86,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE=
+github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
@@ -94,6 +96,8 @@ github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/
 github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
 github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
+github.com/gen2brain/go-fitz v1.24.14 h1:09weRkjVtLYNGo7l0J7DyOwBExbwi8SJ9h8YPhw9WEo=
+github.com/gen2brain/go-fitz v1.24.14/go.mod h1:0KaZeQgASc20Yp5R/pFzyy7SmP01XcoHKNF842U2/S4=
 github.com/getkin/kin-openapi v0.129.0 h1:QGYTNcmyP5X0AtFQ2Dkou9DGBJsUETeLH9rFrJXZh30=
 github.com/getkin/kin-openapi v0.129.0/go.mod h1:gmWI+b/J45xqpyK5wJmRRZse5wefA5H0RDMK46kLUtI=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -171,6 +175,8 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE=
+github.com/jupiterrider/ffi v0.2.0 h1:tMM70PexgYNmV+WyaYhJgCvQAvtTCs3wXeILPutihnA=
+github.com/jupiterrider/ffi v0.2.0/go.mod h1:yqYqX5DdEccAsHeMn+6owkoI2llBLySVAF8dwCDZPVs=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -204,6 +210,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -261,6 +269,9 @@ github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ
 github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=

internal/database/migrations.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"log/slog"
 
-	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/database"
 
 	"github.com/golang-migrate/migrate/v4"
@@ -17,38 +16,52 @@ import (
 	_ "github.com/lib/pq"
 )
 
-func createDB(cfg database.ConfigProvider) error {
-	connStr := fmt.Sprintf("%s?%s", cfg.GetDBBaseURI(), cfg.GetDBOptsString())
+func createDB(ctx context.Context, cfg database.ConfigProvider) error {
+	slog.Debug("creating connection to admin database", "uri", cfg.GetDBAdminDBURI())
 
-	db, err := sql.Open(cfg.GetDBDriver(), connStr)
+	db, err := sql.Open(cfg.GetDBDriver(), cfg.GetDBAdminDBURI())
+	if err != nil {
+		return fmt.Errorf("error opening admin database: %v", err)
+	}
+
+	slog.Debug("pinging admin database", "uri", cfg.GetDBAdminDBURI())
+
+	err = db.PingContext(ctx)
+	if err != nil {
+		return fmt.Errorf("error pinging admin database: %v", err)
+	}
+
+	slog.Debug("creating admin database", "uri", cfg.GetDBAdminDBURI())
+
+	_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", cfg.GetDBName()))
+	if err != nil {
+		slog.Info("database not created", "name", cfg.GetDBName(), "error", err.Error())
+	} else {
+		slog.Info("database created", "name", cfg.GetDBName())
+	}
+
+	slog.Debug("creating connection to database", "uri", cfg.GetDBAdminDBURI())
+
+	db, err = sql.Open(cfg.GetDBDriver(), cfg.GetDBURI())
 	if err != nil {
 		return fmt.Errorf("error opening database: %v", err)
 	}
 
-	err = db.Ping()
+	slog.Debug("pinging database", "uri", cfg.GetDBAdminDBURI())
+
+	err = db.PingContext(ctx)
 	if err != nil {
 		return fmt.Errorf("error pinging database: %v", err)
 	}
 
-	rs, err := db.Query(fmt.Sprintf("SELECT 'CREATE DATABASE %s' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '%s')", cfg.GetDBName(), cfg.GetDBName()))
-	if err != nil {
-		return fmt.Errorf("error creating database: %v", err)
-	}
-
-	if rs.Next() {
-		slog.Info("database created", "name", cfg.GetDBName())
-	} else {
-		slog.Info("database already exists", "name", cfg.GetDBName())
-	}
-
 	return nil
 }
 
 //go:embed migrations/*.up.sql
 var migrations embed.FS
 
-func RunMigrations(ctx context.Context, cfg serviceconfig.ConfigProvider) error {
-	err := createDB(cfg)
+func RunMigrations(ctx context.Context, cfg database.ConfigProvider) error {
+	err := createDB(ctx, cfg)
 	if err != nil {
 		return err
 	}

internal/database/migrations/00000000000005_documents.up.sql

@@ -19,7 +19,6 @@ CREATE TYPE cleanFailType AS ENUM (
     'invalid_read',
     'invalid_read_pages',
     'zero_page_count',
-    'large_page_count',
     'large_file',
     'small_dimensions',
     'large_dimensions',
@@ -50,30 +49,20 @@ CREATE TABLE documentCleanEntries (
     foreign key (cleanId) references documentCleans(id)
 );
 
-CREATE TABLE documentTextTriggerExtractions (
-    id uuid primary key DEFAULT uuid_generate_v7(),
-    cleanId uuid not null,
-    version bigint not null,
-    createdAt timestamp not null,
-    part unsignedsmallint not null,
-    textractJobId text,
-    foreign key (cleanId) references documentCleans(id)
-);
-
 CREATE TABLE documentTextExtractions (
     id uuid primary key DEFAULT uuid_generate_v7(),
+    cleanId uuid not null,
     bucket text not null,
     key text not null,
     hash text not null,
     createdAt timestamp not null,
-    part unsignedsmallint not null
+    part unsignedsmallint not null,
+    foreign key (cleanId) references documentCleans(id)
 );
 
 CREATE TABLE documentTextExtractionEntries (
     id uuid primary key DEFAULT uuid_generate_v7(),
     textId uuid not null,
-    triggerId uuid not null,
     version bigint not null,
-    foreign key (triggerId) references documentTextTriggerExtractions(id),
     foreign key (textId) references documentTextExtractions(id)
 );

internal/database/migrations/00000000000102_document_views.up.sql

@@ -54,25 +54,20 @@ WHERE row_num = 1;
 CREATE VIEW currentTextEntries as
 WITH RankedExtractions AS (
     SELECT
-        dte.id,
+        extract.id,
         cc.documentId,
-        dte.bucket,
-        dte.key,
-        dte.hash,
-        trigger.id as triggerId,
-        trigger.cleanId,
-        trigger.textractJobId,
-        trigger.version as triggerVersion,
+        extract.bucket,
+        extract.key,
+        extract.hash,
+        extract.cleanId,
         dtee.version as extractionVersion,
-        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dte.id DESC) as row_num
+        ROW_NUMBER() OVER (PARTITION BY cc.documentId ORDER BY dtee.id DESC) as row_num
     FROM documents d
     JOIN currentCleanEntries cc on cc.documentId = d.id
     JOIN currentCollectorMinTextVersions ctv ON ctv.clientId = d.clientId
-    JOIN documentTextTriggerExtractions trigger ON trigger.cleanId = cc.id
-        AND trigger.version >= ctv.minTextVersion
-    JOIN documentTextExtractionEntries dtee on dtee.triggerId = trigger.id
+    JOIN documentTextExtractions extract ON extract.cleanId = cc.id
+    JOIN documentTextExtractionEntries dtee on dtee.textId = extract.id
         AND dtee.version >= ctv.minTextVersion
-    JOIN documentTextExtractions dte ON dte.id = dtee.textId
 )
 SELECT
     id,
@@ -81,9 +76,6 @@ SELECT
     key,
     hash,
     cleanId,
-    triggerId,
-    textractJobId,
-    triggerVersion,
     extractionVersion
 FROM RankedExtractions
 WHERE row_num = 1;

internal/database/migrations_test.go

@@ -1,7 +1,6 @@
 package database_test
 
 import (
-	"context"
 	"testing"
 
 	migrations "queryorchestration/internal/database"
@@ -16,23 +15,18 @@ func TestRunMigrations(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
-	ctx := context.Background()
-
+	ctx := t.Context()
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
 
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{})
-	defer cleanup()
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{})
 
 	err := migrations.RunMigrations(ctx, cfg)
 	require.NoError(t, err)
 }
 
 func TestRunMigrationsNoDB(t *testing.T) {
-	ctx := context.Background()
-
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
 	cfg.SetDBUser("invalid_user")
 	cfg.SetDBSecret("invalid_pass")
 	cfg.SetDBHost("invalid_host")
@@ -40,6 +34,6 @@ func TestRunMigrationsNoDB(t *testing.T) {
 	cfg.SetDBName("invalid_name")
 	cfg.SetDBNoSSL(true)
 
-	err := migrations.RunMigrations(ctx, cfg)
+	err := migrations.RunMigrations(t.Context(), cfg)
 	assert.Error(t, err)
 }

internal/database/migrationsprivate_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"context"
 	"testing"
 
 	"queryorchestration/internal/serviceconfig/database"
@@ -9,10 +10,11 @@ import (
 )
 
 func TestCreateDB(t *testing.T) {
+	ctx := context.Background()
 	cfg := &database.DBConfig{
 		DBHost: "invalid_value",
 	}
 
-	err := createDB(cfg)
+	err := createDB(ctx, cfg)
 	assert.Error(t, err)
 }

internal/database/queries/text.sql

@@ -6,24 +6,18 @@ SELECT EXISTS(
 -- name: GetDocumentTextExtractionByHash :one
 SELECT id FROM currentTextEntries WHERE cleanId = @cleanEntryId and hash = @hash;
 
--- name: AddDocumentTextTrigger :one
-INSERT INTO documentTextTriggerExtractions
-    (cleanId, version, createdAt, part) VALUES
-    ($1, $2, $3, $4)
-    returning id;
-
 -- name: GetTextractOutputCurrentPart :one
 WITH client as (
     select id from clients where id = @clientId
 ),
 parts as (
-    SELECT triggers.part
+    SELECT extract.part
         FROM client as c
         JOIN documents as docs on docs.clientId = c.id
         JOIN documentCleans as clean on docs.id = clean.documentId
-        JOIN documentTextTriggerExtractions as triggers
-            on triggers.cleanId = clean.id
-        WHERE date(triggers.createdAt) = date(@queryDate)
+        JOIN documentTextExtractions as extract
+            on extract.cleanId = clean.id
+        WHERE date(extract.createdAt) = date(@queryDate)
 ),
 max_part as (
     SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -40,17 +34,12 @@ WITH client as (
     select id from clients where id = @clientId
 ),
 parts as (
-    SELECT extractions.part
+    SELECT extract.part
         FROM client as c
         JOIN documents as docs on docs.clientId = c.id
         JOIN documentCleans as clean on docs.id = clean.documentId
-        JOIN documentTextTriggerExtractions as triggers
-            on triggers.cleanId = clean.id
-        JOIN documentTextExtractionEntries as textEntries
-            on textEntries.triggerId = triggers.id
-        JOIN documentTextExtractions as extractions
-            on extractions.id = textEntries.textId
-        WHERE date(extractions.createdAt) = date(@queryDate)
+        JOIN documentTextExtractions extract ON extract.cleanId = clean.id
+        WHERE date(extract.createdAt) = date(@queryDate)
 ),
 max_part as (
     SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -62,30 +51,18 @@ FROM parts p
 RIGHT JOIN max_part mp ON p.part = mp.max_part_num
 GROUP BY p.part;
 
--- name: AddDocumentTextTriggerJobId :exec
-UPDATE documentTextTriggerExtractions
-    SET textractJobId = @textractJobId
-    WHERE id = @id;
-
--- name: GetDocumentTextTrigger :one
-SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
-    from documentTextTriggerExtractions as dt
-    join documentCleans as dc on dt.cleanId = dc.id
-    where dt.id = @triggerId;
-
 -- name: AddDocumentText :one
 INSERT INTO documentTextExtractions
-    (bucket, key, hash, createdAt, part)
-    VALUES ($1, $2, $3, $4, $5)
+    (cleanId, bucket, key, hash, createdAt, part)
+    VALUES ($1, $2, $3, $4, $5, $6)
     returning id;
 
 -- name: AddDocumentTextEntry :exec
 INSERT INTO documentTextExtractionEntries
-    (textId, triggerId, version)
-    VALUES ($1, $2, $3);
+    (textId, version)
+    VALUES ($1, $2);
 
 -- name: GetTextEntryByDocId :one
-SELECT id, documentId, bucket, key, hash, cleanId,
-    triggerId, textractJobId, triggerVersion, extractionVersion
+SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
     FROM currentTextEntries
     WHERE documentId = @documentId;

internal/database/repository/clean_test.go

@@ -13,17 +13,17 @@ import (
 )
 
 func TestClean(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 

internal/database/repository/client_test.go

@@ -13,17 +13,17 @@ import (
 )
 
 func TestClient(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 

internal/database/repository/collector_test.go

@@ -15,17 +15,17 @@ import (
 )
 
 func TestCollector(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 

internal/database/repository/document_test.go

@@ -13,17 +13,17 @@ import (
 )
 
 func TestDocument(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 

internal/database/repository/models.go

@@ -19,7 +19,6 @@ const (
 	CleanfailtypeInvalidRead      Cleanfailtype = "invalid_read"
 	CleanfailtypeInvalidReadPages Cleanfailtype = "invalid_read_pages"
 	CleanfailtypeZeroPageCount    Cleanfailtype = "zero_page_count"
-	CleanfailtypeLargePageCount   Cleanfailtype = "large_page_count"
 	CleanfailtypeLargeFile        Cleanfailtype = "large_file"
 	CleanfailtypeSmallDimensions  Cleanfailtype = "small_dimensions"
 	CleanfailtypeLargeDimensions  Cleanfailtype = "large_dimensions"
@@ -68,7 +67,6 @@ func (e Cleanfailtype) Valid() bool {
 		CleanfailtypeInvalidRead,
 		CleanfailtypeInvalidReadPages,
 		CleanfailtypeZeroPageCount,
-		CleanfailtypeLargePageCount,
 		CleanfailtypeLargeFile,
 		CleanfailtypeSmallDimensions,
 		CleanfailtypeLargeDimensions,
@@ -290,9 +288,6 @@ type Currenttextentry struct {
 	Key               string    `db:"key"`
 	Hash              string    `db:"hash"`
 	Cleanid           uuid.UUID `db:"cleanid"`
-	Triggerid         uuid.UUID `db:"triggerid"`
-	Textractjobid     *string   `db:"textractjobid"`
-	Triggerversion    int64     `db:"triggerversion"`
 	Extractionversion int64     `db:"extractionversion"`
 }
 
@@ -327,6 +322,7 @@ type Documententry struct {
 
 type Documenttextextraction struct {
 	ID        uuid.UUID        `db:"id"`
+	Cleanid   uuid.UUID        `db:"cleanid"`
 	Bucket    string           `db:"bucket"`
 	Key       string           `db:"key"`
 	Hash      string           `db:"hash"`
@@ -335,19 +331,9 @@ type Documenttextextraction struct {
 }
 
 type Documenttextextractionentry struct {
-	ID        uuid.UUID `db:"id"`
-	Textid    uuid.UUID `db:"textid"`
-	Triggerid uuid.UUID `db:"triggerid"`
-	Version   int64     `db:"version"`
-}
-
-type Documenttexttriggerextraction struct {
-	ID            uuid.UUID        `db:"id"`
-	Cleanid       uuid.UUID        `db:"cleanid"`
-	Version       int64            `db:"version"`
-	Createdat     pgtype.Timestamp `db:"createdat"`
-	Part          uint16           `db:"part"`
-	Textractjobid *string          `db:"textractjobid"`
+	ID      uuid.UUID `db:"id"`
+	Textid  uuid.UUID `db:"textid"`
+	Version int64     `db:"version"`
 }
 
 type Fullactivecollector struct {

internal/database/repository/query_test.go

@@ -14,17 +14,17 @@ import (
 )
 
 func TestQueries(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -226,17 +226,17 @@ func TestQueries(t *testing.T) {
 }
 
 func TestQueryDependencyTree(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -447,17 +447,17 @@ func TestQueryDependencyTree(t *testing.T) {
 }
 
 func TestQueriesList(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -505,17 +505,17 @@ func TestQueriesList(t *testing.T) {
 }
 
 func TestListQueryClients(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 

internal/database/repository/result_test.go

@@ -16,17 +16,17 @@ import (
 )
 
 func TestResults(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -97,9 +97,11 @@ func TestResults(t *testing.T) {
 	require.NoError(t, err)
 	assert.False(t, issynced)
 
-	triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
+		Bucket:  "hi",
+		Key:     "hello",
+		Hash:    "example",
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -107,21 +109,9 @@ func TestResults(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
-	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hello",
-		Hash:   "example",
-		Part:   0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Version:   1,
-		Textid:    textId,
-		Triggerid: triggerId,
+		Version: 1,
+		Textid:  textId,
 	})
 	require.NoError(t, err)
 
@@ -167,17 +157,17 @@ func TestResults(t *testing.T) {
 }
 
 func TestResultValues(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -274,9 +264,11 @@ func TestResultValues(t *testing.T) {
 		Version: 1,
 	})
 	require.NoError(t, err)
-	triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
+		Bucket:  "hi",
+		Key:     "hello",
+		Hash:    "example",
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -284,21 +276,9 @@ func TestResultValues(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
-	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hello",
-		Hash:   "example",
-		Part:   0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Version:   1,
-		Textid:    textId,
-		Triggerid: triggerId,
+		Version: 1,
+		Textid:  textId,
 	})
 	require.NoError(t, err)
 
@@ -393,17 +373,17 @@ func TestResultValues(t *testing.T) {
 }
 
 func TestUnsyncedNoDepsQueries(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -498,9 +478,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 	require.NoError(t, err)
 	assert.Len(t, qs, 0)
 
-	triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
+		Bucket:  "hi",
+		Key:     "hello",
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -508,20 +489,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
-	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hello",
-		Part:   0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Version:   1,
-		Textid:    textId,
-		Triggerid: triggerId,
+		Version: 1,
+		Textid:  textId,
 	})
 	require.NoError(t, err)
 
@@ -588,9 +558,10 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 	require.NoError(t, err)
 	assert.Len(t, qs, 0)
 
-	triggerTwoId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleantwoid,
-		Version: 123,
+		Bucket:  "hi",
+		Key:     "hello",
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -598,20 +569,9 @@ func TestUnsyncedNoDepsQueries(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
-	textTwoId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hello",
-		Part:   0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Version:   1,
-		Textid:    textTwoId,
-		Triggerid: triggerTwoId,
+		Version: 1,
+		Textid:  textTwoId,
 	})
 	require.NoError(t, err)
 

internal/database/repository/sync_test.go

@@ -15,17 +15,17 @@ import (
 )
 
 func TestListClientDocumentIDs(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -120,17 +120,17 @@ func TestListClientDocumentIDs(t *testing.T) {
 }
 
 func TestClientSync(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	queries := cfg.GetDBQueries()
 
@@ -285,31 +285,21 @@ func TestClientSync(t *testing.T) {
 			Fields:   []byte(`{"first_key": null}`),
 		}, docExternal)
 
-		triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+		textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 			Cleanid: cleanId,
-			Version: 123,
 			Part:    0,
 			Createdat: pgtype.Timestamp{
 				Time:  time.Now().UTC(),
 				Valid: true,
 			},
-		})
-		require.NoError(t, err)
-		textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 			Bucket: "hi",
 			Key:    "hello",
 			Hash:   "example",
-			Part:   0,
-			Createdat: pgtype.Timestamp{
-				Time:  time.Now().UTC(),
-				Valid: true,
-			},
 		})
 		require.NoError(t, err)
 		err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-			Version:   1,
-			Textid:    textId,
-			Triggerid: triggerId,
+			Version: 1,
+			Textid:  textId,
 		})
 		require.NoError(t, err)
 
@@ -467,16 +457,6 @@ func TestClientSync(t *testing.T) {
 		})
 
 		t.Run("update text entry", func(t *testing.T) {
-			triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
-				Cleanid: cleanId,
-				Version: 123,
-				Part:    0,
-				Createdat: pgtype.Timestamp{
-					Time:  time.Now().UTC(),
-					Valid: true,
-				},
-			})
-			require.NoError(t, err)
 			textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 				Bucket: "hi",
 				Key:    "hello",
@@ -485,12 +465,12 @@ func TestClientSync(t *testing.T) {
 					Time:  time.Now().UTC(),
 					Valid: true,
 				},
+				Cleanid: cleanId,
 			})
 			require.NoError(t, err)
 			err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-				Version:   1,
-				Textid:    textId,
-				Triggerid: triggerId,
+				Version: 1,
+				Textid:  textId,
 			})
 			require.NoError(t, err)
 
@@ -594,16 +574,6 @@ func TestClientSync(t *testing.T) {
 				Fields:   []byte(`{"first_key": null}`),
 			}, docExternal)
 
-			triggerThreeId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
-				Cleanid: cleanthreeid,
-				Version: 123,
-				Part:    0,
-				Createdat: pgtype.Timestamp{
-					Time:  time.Now().UTC(),
-					Valid: true,
-				},
-			})
-			require.NoError(t, err)
 			textThreeId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 				Bucket: "hi",
 				Key:    "hello",
@@ -612,12 +582,12 @@ func TestClientSync(t *testing.T) {
 					Time:  time.Now().UTC(),
 					Valid: true,
 				},
+				Cleanid: cleanthreeid,
 			})
 			require.NoError(t, err)
 			err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-				Version:   1,
-				Textid:    textThreeId,
-				Triggerid: triggerThreeId,
+				Version: 1,
+				Textid:  textThreeId,
 			})
 			require.NoError(t, err)
 

internal/database/repository/text.sql.go

@@ -14,12 +14,13 @@ import (
 
 const addDocumentText = `-- name: AddDocumentText :one
 INSERT INTO documentTextExtractions
-    (bucket, key, hash, createdAt, part)
-    VALUES ($1, $2, $3, $4, $5)
+    (cleanId, bucket, key, hash, createdAt, part)
+    VALUES ($1, $2, $3, $4, $5, $6)
     returning id
 `
 
 type AddDocumentTextParams struct {
+	Cleanid   uuid.UUID        `db:"cleanid"`
 	Bucket    string           `db:"bucket"`
 	Key       string           `db:"key"`
 	Hash      string           `db:"hash"`
@@ -30,11 +31,12 @@ type AddDocumentTextParams struct {
 // AddDocumentText
 //
 //	INSERT INTO documentTextExtractions
-//	    (bucket, key, hash, createdAt, part)
-//	    VALUES ($1, $2, $3, $4, $5)
+//	    (cleanId, bucket, key, hash, createdAt, part)
+//	    VALUES ($1, $2, $3, $4, $5, $6)
 //	    returning id
 func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParams) (uuid.UUID, error) {
 	row := q.db.QueryRow(ctx, addDocumentText,
+		arg.Cleanid,
 		arg.Bucket,
 		arg.Key,
 		arg.Hash,
@@ -48,76 +50,22 @@ func (q *Queries) AddDocumentText(ctx context.Context, arg *AddDocumentTextParam
 
 const addDocumentTextEntry = `-- name: AddDocumentTextEntry :exec
 INSERT INTO documentTextExtractionEntries
-    (textId, triggerId, version)
-    VALUES ($1, $2, $3)
+    (textId, version)
+    VALUES ($1, $2)
 `
 
 type AddDocumentTextEntryParams struct {
-	Textid    uuid.UUID `db:"textid"`
-	Triggerid uuid.UUID `db:"triggerid"`
-	Version   int64     `db:"version"`
+	Textid  uuid.UUID `db:"textid"`
+	Version int64     `db:"version"`
 }
 
 // AddDocumentTextEntry
 //
 //	INSERT INTO documentTextExtractionEntries
-//	    (textId, triggerId, version)
-//	    VALUES ($1, $2, $3)
+//	    (textId, version)
+//	    VALUES ($1, $2)
 func (q *Queries) AddDocumentTextEntry(ctx context.Context, arg *AddDocumentTextEntryParams) error {
-	_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Triggerid, arg.Version)
-	return err
-}
-
-const addDocumentTextTrigger = `-- name: AddDocumentTextTrigger :one
-INSERT INTO documentTextTriggerExtractions
-    (cleanId, version, createdAt, part) VALUES
-    ($1, $2, $3, $4)
-    returning id
-`
-
-type AddDocumentTextTriggerParams struct {
-	Cleanid   uuid.UUID        `db:"cleanid"`
-	Version   int64            `db:"version"`
-	Createdat pgtype.Timestamp `db:"createdat"`
-	Part      uint16           `db:"part"`
-}
-
-// AddDocumentTextTrigger
-//
-//	INSERT INTO documentTextTriggerExtractions
-//	    (cleanId, version, createdAt, part) VALUES
-//	    ($1, $2, $3, $4)
-//	    returning id
-func (q *Queries) AddDocumentTextTrigger(ctx context.Context, arg *AddDocumentTextTriggerParams) (uuid.UUID, error) {
-	row := q.db.QueryRow(ctx, addDocumentTextTrigger,
-		arg.Cleanid,
-		arg.Version,
-		arg.Createdat,
-		arg.Part,
-	)
-	var id uuid.UUID
-	err := row.Scan(&id)
-	return id, err
-}
-
-const addDocumentTextTriggerJobId = `-- name: AddDocumentTextTriggerJobId :exec
-UPDATE documentTextTriggerExtractions
-    SET textractJobId = $1
-    WHERE id = $2
-`
-
-type AddDocumentTextTriggerJobIdParams struct {
-	Textractjobid *string   `db:"textractjobid"`
-	ID            uuid.UUID `db:"id"`
-}
-
-// AddDocumentTextTriggerJobId
-//
-//	UPDATE documentTextTriggerExtractions
-//	    SET textractJobId = $1
-//	    WHERE id = $2
-func (q *Queries) AddDocumentTextTriggerJobId(ctx context.Context, arg *AddDocumentTextTriggerJobIdParams) error {
-	_, err := q.db.Exec(ctx, addDocumentTextTriggerJobId, arg.Textractjobid, arg.ID)
+	_, err := q.db.Exec(ctx, addDocumentTextEntry, arg.Textid, arg.Version)
 	return err
 }
 
@@ -140,51 +88,15 @@ func (q *Queries) GetDocumentTextExtractionByHash(ctx context.Context, arg *GetD
 	return id, err
 }
 
-const getDocumentTextTrigger = `-- name: GetDocumentTextTrigger :one
-SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
-    from documentTextTriggerExtractions as dt
-    join documentCleans as dc on dt.cleanId = dc.id
-    where dt.id = $1
-`
-
-type GetDocumentTextTriggerRow struct {
-	ID            uuid.UUID `db:"id"`
-	Cleanid       uuid.UUID `db:"cleanid"`
-	Version       int64     `db:"version"`
-	Textractjobid *string   `db:"textractjobid"`
-	Documentid    uuid.UUID `db:"documentid"`
-}
-
-// GetDocumentTextTrigger
-//
-//	SELECT dt.id, dt.cleanId, dt.version, dt.textractJobId, dc.documentId
-//	    from documentTextTriggerExtractions as dt
-//	    join documentCleans as dc on dt.cleanId = dc.id
-//	    where dt.id = $1
-func (q *Queries) GetDocumentTextTrigger(ctx context.Context, triggerid uuid.UUID) (*GetDocumentTextTriggerRow, error) {
-	row := q.db.QueryRow(ctx, getDocumentTextTrigger, triggerid)
-	var i GetDocumentTextTriggerRow
-	err := row.Scan(
-		&i.ID,
-		&i.Cleanid,
-		&i.Version,
-		&i.Textractjobid,
-		&i.Documentid,
-	)
-	return &i, err
-}
-
 const getTextEntryByDocId = `-- name: GetTextEntryByDocId :one
-SELECT id, documentId, bucket, key, hash, cleanId,
-    triggerId, textractJobId, triggerVersion, extractionVersion
+SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
     FROM currentTextEntries
     WHERE documentId = $1
 `
 
 // GetTextEntryByDocId
 //
-//	SELECT id, documentId, bucket, key, hash, cleanId,
-//	    triggerId, textractJobId, triggerVersion, extractionVersion
+//	SELECT id, documentId, bucket, key, hash, cleanId, extractionVersion
 //	    FROM currentTextEntries
 //	    WHERE documentId = $1
 func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID) (*Currenttextentry, error) {
@@ -197,9 +109,6 @@ func (q *Queries) GetTextEntryByDocId(ctx context.Context, documentid uuid.UUID)
 		&i.Key,
 		&i.Hash,
 		&i.Cleanid,
-		&i.Triggerid,
-		&i.Textractjobid,
-		&i.Triggerversion,
 		&i.Extractionversion,
 	)
 	return &i, err
@@ -210,17 +119,12 @@ WITH client as (
     select id from clients where id = $1
 ),
 parts as (
-    SELECT extractions.part
+    SELECT extract.part
         FROM client as c
         JOIN documents as docs on docs.clientId = c.id
         JOIN documentCleans as clean on docs.id = clean.documentId
-        JOIN documentTextTriggerExtractions as triggers
-            on triggers.cleanId = clean.id
-        JOIN documentTextExtractionEntries as textEntries
-            on textEntries.triggerId = triggers.id
-        JOIN documentTextExtractions as extractions
-            on extractions.id = textEntries.textId
-        WHERE date(extractions.createdAt) = date($2)
+        JOIN documentTextExtractions extract ON extract.cleanId = clean.id
+        WHERE date(extract.createdAt) = date($2)
 ),
 max_part as (
     SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -249,17 +153,12 @@ type GetTextOutCurrentPartRow struct {
 //	    select id from clients where id = $1
 //	),
 //	parts as (
-//	    SELECT extractions.part
+//	    SELECT extract.part
 //	        FROM client as c
 //	        JOIN documents as docs on docs.clientId = c.id
 //	        JOIN documentCleans as clean on docs.id = clean.documentId
-//	        JOIN documentTextTriggerExtractions as triggers
-//	            on triggers.cleanId = clean.id
-//	        JOIN documentTextExtractionEntries as textEntries
-//	            on textEntries.triggerId = triggers.id
-//	        JOIN documentTextExtractions as extractions
-//	            on extractions.id = textEntries.textId
-//	        WHERE date(extractions.createdAt) = date($2)
+//	        JOIN documentTextExtractions extract ON extract.cleanId = clean.id
+//	        WHERE date(extract.createdAt) = date($2)
 //	),
 //	max_part as (
 //	    SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -282,13 +181,13 @@ WITH client as (
     select id from clients where id = $1
 ),
 parts as (
-    SELECT triggers.part
+    SELECT extract.part
         FROM client as c
         JOIN documents as docs on docs.clientId = c.id
         JOIN documentCleans as clean on docs.id = clean.documentId
-        JOIN documentTextTriggerExtractions as triggers
-            on triggers.cleanId = clean.id
-        WHERE date(triggers.createdAt) = date($2)
+        JOIN documentTextExtractions as extract
+            on extract.cleanId = clean.id
+        WHERE date(extract.createdAt) = date($2)
 ),
 max_part as (
     SELECT COALESCE(max(part), 0) as max_part_num FROM parts
@@ -317,13 +216,13 @@ type GetTextractOutputCurrentPartRow struct {
 //	    select id from clients where id = $1
 //	),
 //	parts as (
-//	    SELECT triggers.part
+//	    SELECT extract.part
 //	        FROM client as c
 //	        JOIN documents as docs on docs.clientId = c.id
 //	        JOIN documentCleans as clean on docs.id = clean.documentId
-//	        JOIN documentTextTriggerExtractions as triggers
-//	            on triggers.cleanId = clean.id
-//	        WHERE date(triggers.createdAt) = date($2)
+//	        JOIN documentTextExtractions as extract
+//	            on extract.cleanId = clean.id
+//	        WHERE date(extract.createdAt) = date($2)
 //	),
 //	max_part as (
 //	    SELECT COALESCE(max(part), 0) as max_part_num FROM parts

internal/database/repository/text_test.go

@@ -16,17 +16,17 @@ import (
 )
 
 func TestTextExtraction(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer textup()
 
 	queries := cfg.GetDBQueries()
 
@@ -68,73 +68,31 @@ func TestTextExtraction(t *testing.T) {
 	require.NoError(t, err)
 	assert.False(t, isextract)
 
-	triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
-		Cleanid: cleanid,
-		Version: 123,
-		Part:    0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
-
-	trigger, err := queries.GetDocumentTextTrigger(ctx, triggerId)
-	require.NoError(t, err)
-	assert.EqualExportedValues(t, &repository.GetDocumentTextTriggerRow{
-		ID:         triggerId,
-		Cleanid:    cleanid,
-		Version:    123,
-		Documentid: id,
-	}, trigger)
-
-	isextract, err = queries.IsDocumentTextExtracted(ctx, id)
-	require.NoError(t, err)
-	assert.False(t, isextract)
-
-	textractId := "hello"
-	err = queries.AddDocumentTextTriggerJobId(ctx, &repository.AddDocumentTextTriggerJobIdParams{
-		Textractjobid: &textractId,
-		ID:            triggerId,
-	})
-	require.NoError(t, err)
-
-	trigger, err = queries.GetDocumentTextTrigger(ctx, triggerId)
-	require.NoError(t, err)
-	assert.EqualExportedValues(t, &repository.GetDocumentTextTriggerRow{
-		ID:            triggerId,
-		Cleanid:       cleanid,
-		Version:       123,
-		Textractjobid: &textractId,
-		Documentid:    id,
-	}, trigger)
-
-	isextract, err = queries.IsDocumentTextExtracted(ctx, id)
-	require.NoError(t, err)
-	assert.False(t, isextract)
-
 	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: bucket,
-		Key:    key,
-		Hash:   "example",
-		Part:   0,
+		Cleanid: cleanid,
+		Part:    0,
+		Bucket:  bucket,
+		Key:     key,
+		Hash:    "example",
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
 			Valid: true,
 		},
 	})
 	require.NoError(t, err)
-	assert.NotNil(t, textId)
 	assert.NotEqual(t, uuid.UUID{}, textId)
 
 	isextract, err = queries.IsDocumentTextExtracted(ctx, id)
 	require.NoError(t, err)
 	assert.False(t, isextract)
 
+	isextract, err = queries.IsDocumentTextExtracted(ctx, id)
+	require.NoError(t, err)
+	assert.False(t, isextract)
+
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Textid:    textId,
-		Triggerid: triggerId,
-		Version:   543,
+		Textid:  textId,
+		Version: 543,
 	})
 	require.NoError(t, err)
 
@@ -150,9 +108,6 @@ func TestTextExtraction(t *testing.T) {
 	assert.Equal(t, key, text.Key)
 	assert.Equal(t, "example", text.Hash)
 	assert.Equal(t, cleanid, text.Cleanid)
-	assert.Equal(t, triggerId, text.Triggerid)
-	assert.Equal(t, textractId, *text.Textractjobid)
-	assert.Equal(t, int64(123), text.Triggerversion)
 	assert.Equal(t, int64(543), text.Extractionversion)
 	assert.Equal(t, textId, text.ID)
 
@@ -171,17 +126,17 @@ func TestTextExtraction(t *testing.T) {
 }
 
 func TestTextTextractPart(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer textup()
 
 	queries := cfg.GetDBQueries()
 
@@ -232,9 +187,8 @@ func TestTextTextractPart(t *testing.T) {
 		Part:  0,
 	}, part)
 
-	_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -256,9 +210,8 @@ func TestTextTextractPart(t *testing.T) {
 		Part:  0,
 	}, part)
 
-	_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -280,9 +233,8 @@ func TestTextTextractPart(t *testing.T) {
 		Part:  0,
 	}, part)
 
-	_, err = queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	_, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
 		Part:    1,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -306,17 +258,17 @@ func TestTextTextractPart(t *testing.T) {
 }
 
 func TestTextOutPart(t *testing.T) {
+	t.Parallel()
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
 	ctx := context.Background()
 
 	cfg := &serviceconfig.BaseConfig{}
-	test.SetCfgProvider(t, cfg)
-	_, textup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer textup()
 
 	queries := cfg.GetDBQueries()
 
@@ -354,9 +306,11 @@ func TestTextOutPart(t *testing.T) {
 	})
 	require.NoError(t, err)
 
-	triggerId, err := queries.AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
+	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
 		Cleanid: cleanid,
-		Version: 123,
+		Bucket:  "hi",
+		Key:     "hi",
+		Hash:    "hi",
 		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
@@ -364,6 +318,11 @@ func TestTextOutPart(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
+	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
+		Textid:  textId,
+		Version: 123,
+	})
+	require.NoError(t, err)
 
 	part, err := queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
 		Clientid: &clientId,
@@ -373,47 +332,17 @@ func TestTextOutPart(t *testing.T) {
 		},
 	})
 	require.NoError(t, err)
-	assert.Equal(t, &repository.GetTextOutCurrentPartRow{
-		Count: 0,
-		Part:  0,
-	}, part)
-
-	textId, err := queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hi",
-		Hash:   "hi",
-		Part:   0,
-		Createdat: pgtype.Timestamp{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
-	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Textid:    textId,
-		Triggerid: triggerId,
-		Version:   123,
-	})
-	require.NoError(t, err)
-
-	part, err = queries.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
-		Clientid: &clientId,
-		Querydate: pgtype.Timestamptz{
-			Time:  time.Now().UTC(),
-			Valid: true,
-		},
-	})
-	require.NoError(t, err)
 	assert.Equal(t, &repository.GetTextOutCurrentPartRow{
 		Count: 1,
 		Part:  0,
 	}, part)
 
 	textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hi",
-		Hash:   "hi",
-		Part:   0,
+		Cleanid: cleanid,
+		Bucket:  "hi",
+		Key:     "hi",
+		Hash:    "hi",
+		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
 			Valid: true,
@@ -421,9 +350,8 @@ func TestTextOutPart(t *testing.T) {
 	})
 	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Textid:    textId,
-		Triggerid: triggerId,
-		Version:   123,
+		Textid:  textId,
+		Version: 123,
 	})
 	require.NoError(t, err)
 
@@ -441,10 +369,11 @@ func TestTextOutPart(t *testing.T) {
 	}, part)
 
 	textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hi",
-		Hash:   "hi",
-		Part:   0,
+		Cleanid: cleanid,
+		Bucket:  "hi",
+		Key:     "hi",
+		Hash:    "hi",
+		Part:    0,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
 			Valid: true,
@@ -452,9 +381,8 @@ func TestTextOutPart(t *testing.T) {
 	})
 	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Textid:    textId,
-		Triggerid: triggerId,
-		Version:   123,
+		Textid:  textId,
+		Version: 123,
 	})
 	require.NoError(t, err)
 
@@ -472,10 +400,11 @@ func TestTextOutPart(t *testing.T) {
 	}, part)
 
 	textId, err = queries.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-		Bucket: "hi",
-		Key:    "hi",
-		Hash:   "hi",
-		Part:   1,
+		Cleanid: cleanid,
+		Bucket:  "hi",
+		Key:     "hi",
+		Hash:    "hi",
+		Part:    1,
 		Createdat: pgtype.Timestamp{
 			Time:  time.Now().UTC(),
 			Valid: true,
@@ -483,9 +412,8 @@ func TestTextOutPart(t *testing.T) {
 	})
 	require.NoError(t, err)
 	err = queries.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-		Textid:    textId,
-		Triggerid: triggerId,
-		Version:   123,
+		Textid:  textId,
+		Version: 123,
 	})
 	require.NoError(t, err)
 

internal/document/clean/clean.go

@@ -57,7 +57,12 @@ func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (*
 		}, nil
 	}
 
-	content, err := s.getContent(ctx, params, mimeType)
+	content, err := documenttypes.GetFile(ctx, s.cfg, documenttypes.GetFileParams{
+		Bucket:   params.Bucket,
+		Hash:     params.Hash,
+		Key:      params.Key,
+		Mimetype: mimeType,
+	})
 	if err != nil {
 		return nil, err
 	}

internal/document/clean/content.go

@@ -1,41 +0,0 @@
-package documentclean
-
-import (
-	"context"
-	"errors"
-
-	documenttypes "queryorchestration/internal/document/types"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-)
-
-type Content interface {
-	IsCorrupt(context.Context) *documenttypes.InvalidDocumentReason
-}
-
-func (s *Service) getContent(ctx context.Context, params *CleanParams, mimeType documenttypes.MimeType) (Content, error) {
-	key := params.Key.String()
-	resp, err := s.cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{
-		Bucket:  &params.Bucket,
-		Key:     &key,
-		IfMatch: &params.Hash,
-	})
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	var content Content
-	switch mimeType {
-	case documenttypes.MimeTypePDF:
-		pdf, err := documenttypes.NewPDFFromReadCloser(resp.Body)
-		if err != nil {
-			return nil, err
-		}
-		content = pdf
-	default:
-		return nil, errors.New("invalid mimetype")
-	}
-
-	return content, nil
-}

internal/document/clean/content_test.go

@@ -1,65 +0,0 @@
-package documentclean
-
-import (
-	"context"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	documenttypes "queryorchestration/internal/document/types"
-	"queryorchestration/internal/serviceconfig/objectstore"
-	objectstoremock "queryorchestration/mocks/objectstore"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetContent(t *testing.T) {
-	t.Run("Working", func(t *testing.T) {
-		ctx := context.Background()
-
-		cfg := &DocCleanConfig{}
-		mockS3 := objectstoremock.NewMockS3Client(t)
-		cfg.StoreClient = mockS3
-
-		svc := Service{
-			cfg: cfg,
-		}
-
-		bucket := "example_bucket"
-		key := objectstore.BucketKey{
-			ClientID:  "ddddddd",
-			CreatedAt: time.Now().UTC(),
-			EntityID:  uuid.New(),
-			Location:  objectstore.Import,
-		}
-		params := &CleanParams{
-			Hash:   "hash",
-			Bucket: bucket,
-			Key:    key,
-		}
-
-		mimeType := documenttypes.MimeTypePDF
-
-		body := io.NopCloser(strings.NewReader(pdfHelloWorld))
-		mockS3.EXPECT().
-			GetObject(
-				mock.Anything,
-				mock.MatchedBy(func(in *s3.GetObjectInput) bool {
-					return *in.Bucket == bucket && *in.Key == key.String()
-				}),
-				mock.Anything,
-			).
-			Return(&s3.GetObjectOutput{
-				Body: body,
-			}, nil)
-
-		content, err := svc.getContent(ctx, params, mimeType)
-		require.NoError(t, err)
-		assert.NotNil(t, content)
-	})
-}

internal/document/store/process.go

@@ -5,7 +5,6 @@ import (
 	"log/slog"
 
 	docinitrunner "queryorchestration/api/docInitRunner"
-	doctextprocessrunner "queryorchestration/api/docTextProcessRunner"
 	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue"
 )
@@ -44,13 +43,6 @@ func (s *Service) Process(ctx context.Context, params Params) error {
 			Key:    params.Key,
 			Hash:   params.Hash,
 		}
-	case objectstore.TextTextract:
-		queueParams.QueueURL = s.cfg.GetDocumentTextProcessURL()
-		queueParams.Body = doctextprocessrunner.Body{
-			Bucket: params.Bucket,
-			Key:    params.Key,
-			Hash:   params.Hash,
-		}
 	default:
 		slog.Info("unsupported key", "key", params.Key)
 		return nil

internal/document/store/process_test.go

@@ -8,7 +8,6 @@ import (
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/objectstore"
 	"queryorchestration/internal/serviceconfig/queue/documentinit"
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
@@ -19,7 +18,6 @@ import (
 type ProcessConfig struct {
 	serviceconfig.BaseConfig
 	documentinit.DocInitConfig
-	documenttextprocess.DocTextProcessConfig
 }
 
 func TestProcess(t *testing.T) {
@@ -29,7 +27,6 @@ func TestProcess(t *testing.T) {
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
 	cfg.DocInitURL = "docinit"
-	cfg.DocumentTextProcessURL = "doctextprocess"
 
 	svc := New(cfg)
 
@@ -68,28 +65,6 @@ func TestProcess(t *testing.T) {
 		})
 		assert.NoError(t, err)
 	})
-	t.Run("document text process", func(t *testing.T) {
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.DocumentTextProcessURL
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		location := objectstore.BucketKey{
-			ClientID:  "7db16095-9155-47d4-8004-b3b3ead93c83",
-			Location:  objectstore.TextTextract,
-			CreatedAt: time.Now().UTC(),
-		}
-		err := svc.Process(ctx, Params{
-			Event: EventS3ObjectCreatedPut,
-			Key:   location.String(),
-		})
-		assert.NoError(t, err)
-	})
 }
 
 func TestIsSupportedEvent(t *testing.T) {

internal/document/store/service.go

@@ -3,13 +3,11 @@ package documentstore
 import (
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/queue/documentinit"
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
 )
 
 type ConfigProvider interface {
 	serviceconfig.ConfigProvider
 	documentinit.ConfigProvider
-	documenttextprocess.ConfigProvider
 }
 
 type Service struct {

internal/document/store/service_test.go

@@ -5,7 +5,6 @@ import (
 
 	"queryorchestration/internal/serviceconfig"
 	"queryorchestration/internal/serviceconfig/queue/documentinit"
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -13,7 +12,6 @@ import (
 type DocInitConfig struct {
 	serviceconfig.BaseConfig
 	documentinit.DocInitConfig
-	documenttextprocess.DocTextProcessConfig
 }
 
 func TestNew(t *testing.T) {

internal/document/text/documentText.go

@@ -0,0 +1,43 @@
+package documenttext
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	documenttypes "queryorchestration/internal/document/types"
+)
+
+func (s *Service) getDocumentText(ctx context.Context, doc documenttypes.File) (io.Reader, error) {
+	pagesNum, err := doc.GetPageCount(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	textPages := make([]string, pagesNum)
+	for i := range pagesNum {
+		text, err := s.getPageText(ctx, doc, i)
+		if err != nil {
+			return nil, err
+		}
+		textPages[i] = text
+	}
+
+	text, err := s.mergeDocument(textPages)
+	if err != nil {
+		return nil, err
+	}
+
+	return text, nil
+}
+
+func (s *Service) mergeDocument(pages []string) (io.Reader, error) {
+	var builder strings.Builder
+	for i, page := range pages {
+		builder.WriteString(fmt.Sprintf("Start of Page No. = %d\n\n", i+1))
+		builder.WriteString(page)
+		builder.WriteString("\n\n")
+	}
+	return strings.NewReader(builder.String()), nil
+}

internal/document/text/execute.go

@@ -0,0 +1,48 @@
+package documenttext
+
+import (
+	"context"
+	"log/slog"
+
+	documenttypes "queryorchestration/internal/document/types"
+	"queryorchestration/internal/serviceconfig/objectstore"
+
+	"github.com/google/uuid"
+)
+
+func (s *Service) executeExtraction(ctx context.Context, documentId uuid.UUID) error {
+	clean, err := s.cfg.GetDBQueries().GetCleanEntryByDocId(ctx, documentId)
+	if err != nil {
+		return err
+	} else if clean.Fail.Valid {
+		slog.Error("document has no clean document", "id", documentId)
+		return nil
+	}
+
+	key, err := objectstore.ParseBucketKey(*clean.Key)
+	if err != nil {
+		return err
+	}
+
+	doc, err := documenttypes.GetFile(ctx, s.cfg, documenttypes.GetFileParams{
+		Bucket:   *clean.Bucket,
+		Key:      key,
+		Hash:     *clean.Hash,
+		Mimetype: documenttypes.ParseDBNullMimeType(clean.Mimetype),
+	})
+	if err != nil {
+		return err
+	}
+
+	text, err := s.getDocumentText(ctx, doc)
+	if err != nil {
+		return err
+	}
+
+	err = s.storeProcess(ctx, text, clean)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/document/text/execute_test.go

@@ -0,0 +1,102 @@
+package documenttext
+
+import (
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	objectstoremock "queryorchestration/mocks/objectstore"
+	textractmock "queryorchestration/mocks/textract"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExecuteExtract(t *testing.T) {
+	ctx := context.Background()
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+
+	cfg := &DocTextConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
+	mockTextract := textractmock.NewMockTextractClient(t)
+	cfg.TextractClient = mockTextract
+
+	svc := Service{
+		cfg: cfg,
+	}
+
+	cleanId := uuid.New()
+	docId := uuid.New()
+	bucket := "bucket"
+	hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
+	textId := uuid.New()
+	key := objectstore.BucketKey{
+		ClientID:  "aa",
+		Location:  objectstore.Import,
+		CreatedAt: time.Now().UTC(),
+		EntityID:  cleanId,
+	}
+	keyStr := key.String()
+	textractFile := "../../../assets/sampleGeneration/generated/helloWorld.gen"
+	textractBaseOut := getTextractFileResponse(t, textractFile)
+
+	mimetype := repository.NullCleanmimetype{
+		Valid:         true,
+		Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+	}
+	pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(docId).
+		WillReturnRows(
+			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
+				AddRow(cleanId, docId, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
+		)
+	bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
+	mockStore.EXPECT().
+		GetObject(
+			mock.Anything,
+			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+				return *in.Bucket == bucket && *in.Key == keyStr
+			}),
+			mock.Anything,
+		).
+		Return(&s3.GetObjectOutput{
+			Body: bodyMsg,
+		}, nil)
+	sent := 0
+	mockTextract.EXPECT().
+		AnalyzeDocument(
+			mock.Anything,
+			mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+				if sent == 0 {
+					sent++
+					return true
+				}
+				return false
+			}),
+			mock.Anything,
+		).
+		Return(&textractBaseOut["base"][0], nil).Once()
+
+	pool.ExpectBegin()
+	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
+		pgxmock.NewRows([]string{"id"}).
+			AddRow(textId),
+	)
+	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
+		WillReturnResult(pgxmock.NewResult("", 1))
+	pool.ExpectCommit()
+
+	err = svc.executeExtraction(ctx, docId)
+	require.NoError(t, err)
+}

internal/document/text/extract.go

@@ -16,24 +16,16 @@ func (s *Service) Extract(ctx context.Context, documentId uuid.UUID) error {
 	isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, documentId)
 	if err != nil {
 		return err
-	} else if isextracted {
-		return s.informExtraction(ctx, documentId)
 	}
 
-	entry, err := s.cfg.GetDBQueries().GetCleanEntryByDocId(ctx, documentId)
-	if err != nil {
-		return err
-	} else if entry.Fail.Valid {
-		slog.Error("document has no clean document", "id", documentId)
-		return nil
+	if !isextracted {
+		err = s.executeExtraction(ctx, documentId)
+		if err != nil {
+			return err
+		}
 	}
 
-	err = s.triggerExtract(ctx, entry)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return s.informExtraction(ctx, documentId)
 }
 
 func (s *Service) informExtraction(ctx context.Context, id uuid.UUID) error {

internal/document/text/extract_test.go

@@ -3,7 +3,10 @@ package documenttext
 import (
 	"context"
 	"fmt"
+	"io"
+	"strings"
 	"testing"
+	"time"
 
 	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/serviceconfig"
@@ -15,13 +18,13 @@ import (
 	queuemock "queryorchestration/mocks/queue"
 	textractmock "queryorchestration/mocks/textract"
 
-	"github.com/stretchr/testify/require"
-
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
 	awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
 	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 )
 
 type DocTextConfig struct {
@@ -37,91 +40,116 @@ func TestCreate(t *testing.T) {
 	pool, err := pgxmock.NewPool()
 	require.NoError(t, err)
 
-	mockSQS := queuemock.NewMockSQSClient(t)
-
 	cfg := &DocTextConfig{}
-	cfg.QueueClient = mockSQS
-	cfg.QuerySyncURL = "/i/am/here"
 	cfg.DBPool = pool
 	cfg.DBQueries = repository.New(pool)
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
+	mockStore := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockStore
 	mockTextract := textractmock.NewMockTextractClient(t)
 	cfg.TextractClient = mockTextract
+	mockQueue := queuemock.NewMockSQSClient(t)
+	cfg.QueueClient = mockQueue
+	cfg.QuerySyncURL = "ex"
 
 	svc := Service{
 		cfg: cfg,
 	}
 
-	t.Run("valid", func(t *testing.T) {
-		docId := uuid.New()
+	docId := uuid.New()
+	t.Run("is not extracted", func(t *testing.T) {
 		cleanId := uuid.New()
-		bucket := "bucket_name"
-		key := "/i/am/here"
-		clientId := "AAA"
-		hash := "example"
+		bucket := "bucket"
+		hash := `"a06d5e0e795adeb6d0b3732330dbcc15"`
+		textId := uuid.New()
+		key := objectstore.BucketKey{
+			ClientID:  "aa",
+			Location:  objectstore.Import,
+			CreatedAt: time.Now().UTC(),
+			EntityID:  cleanId,
+		}
+		keyStr := key.String()
+		textractFile := "../../../assets/sampleGeneration/generated/helloWorld.gen"
+		textractBaseOut := getTextractFileResponse(t, textractFile)
 
+		mimetype := repository.NullCleanmimetype{
+			Valid:         true,
+			Cleanmimetype: repository.CleanmimetypeApplicationPdf,
+		}
 		pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(docId).WillReturnRows(
 			pgxmock.NewRows([]string{"isextracted"}).
 				AddRow(false),
 		)
-		mimeType := "application/pdf"
-		dbmimetype := repository.NullCleanmimetype{
-			Valid:         true,
-			Cleanmimetype: repository.Cleanmimetype(mimeType),
-		}
-		pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(docId).WillReturnRows(
-			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
-				AddRow(cleanId, docId, &bucket, &key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, clientId),
-		)
-
-		jobId := "hello"
-		triggerId := uuid.New()
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
+		pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(docId).
 			WillReturnRows(
-				pgxmock.NewRows([]string{"part", "count"}).
-					AddRow(0, 2),
+				pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "clientId"}).
+					AddRow(cleanId, docId, &bucket, &keyStr, int64(123), &hash, mimetype, nil, key.ClientID),
 			)
-		pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
-		)
-
-		mockTextract.EXPECT().
-			StartDocumentTextDetection(
+		bodyMsg := io.NopCloser(strings.NewReader(pdfHelloWorld))
+		mockStore.EXPECT().
+			GetObject(
 				mock.Anything,
-				mock.MatchedBy(func(in *awstextract.StartDocumentTextDetectionInput) bool {
-					return true
+				mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+					return *in.Bucket == bucket && *in.Key == keyStr
 				}),
 				mock.Anything,
 			).
-			Return(&awstextract.StartDocumentTextDetectionOutput{
-				JobId: &jobId,
+			Return(&s3.GetObjectOutput{
+				Body: bodyMsg,
 			}, nil)
+		sent := 0
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *awstextract.AnalyzeDocumentInput) bool {
+					if sent == 0 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][0], nil).Once()
 
-		pool.ExpectExec("name: AddDocumentTextTriggerJobId :exec").WithArgs(&jobId, triggerId).
+		pool.ExpectBegin()
+		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(textId),
+		)
+		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
 			WillReturnResult(pgxmock.NewResult("", 1))
 		pool.ExpectCommit()
 
+		mockQueue.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
 		err = svc.Extract(ctx, docId)
 		require.NoError(t, err)
 	})
-	t.Run("invalid clean", func(t *testing.T) {
-		id := uuid.New()
-		fail := repository.NullCleanfailtype{
-			Valid:         true,
-			Cleanfailtype: repository.CleanfailtypeInvalidMimetype,
-		}
-
-		pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(id).WillReturnRows(
-			pgxmock.NewRows([]string{"isextracted"}).AddRow(false),
-		)
-		pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(id).WillReturnRows(
-			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
-				AddRow(id, uuid.New(), nil, nil, int64(1), nil, nil, fail, "AAA"),
+	t.Run("is extracted", func(t *testing.T) {
+		pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(docId).WillReturnRows(
+			pgxmock.NewRows([]string{"isextracted"}).
+				AddRow(true),
 		)
 
-		err = svc.Extract(ctx, id)
+		mockQueue.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
+		err = svc.Extract(ctx, docId)
 		require.NoError(t, err)
 	})
 }

internal/document/text/getPages.go

@@ -0,0 +1,223 @@
+package documenttext
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+
+	documenttypes "queryorchestration/internal/document/types"
+
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/aws/aws-sdk-go-v2/service/textract/types"
+	"github.com/google/uuid"
+)
+
+func (s *Service) getPageText(ctx context.Context, document documenttypes.File, index int) (string, error) {
+	res, err := s.GetBasePage(ctx, document, index)
+	if err != nil {
+		return "", err
+	}
+
+	var pageBlock *types.Block
+	for _, block := range res.Blocks {
+		if block.BlockType == types.BlockTypePage {
+			pageBlock = &block
+			break
+		}
+	}
+	if pageBlock == nil {
+		return "", fmt.Errorf("no page block found for index: %d", index)
+	}
+
+	blockMap := s.getBlockMap(res.Blocks)
+
+	lines, tables, tableBlockMap, found, err := s.getPageElements(ctx, document, index, *pageBlock, blockMap)
+	if err != nil {
+		return "", err
+	}
+
+	return s.buildPageText(ctx, lines, document, blockMap, index, tables, tableBlockMap, found)
+}
+
+func (s *Service) getPageElements(ctx context.Context, document documenttypes.File, index int, pageBlock types.Block, blockMap map[uuid.UUID]types.Block) ([]uuid.UUID, []uuid.UUID, map[uuid.UUID]types.Block, map[uuid.UUID]*bool, error) {
+	lines := []uuid.UUID{}
+	hasTable := false
+	found := map[uuid.UUID]*bool{}
+
+	for _, id := range s.getChildIds(pageBlock) {
+		if found[id] != nil && !*found[id] {
+			continue
+		}
+
+		if s.isLayout(blockMap[id].BlockType) {
+			lines = append(lines, id)
+			f := true
+			found[id] = &f
+		}
+		if blockMap[id].BlockType == types.BlockTypeLayoutTable {
+			hasTable = true
+		}
+		if blockMap[id].BlockType == types.BlockTypeLayoutTable ||
+			blockMap[id].BlockType == types.BlockTypeLayoutList {
+			for _, childId := range s.getChildIds(blockMap[id]) {
+				f := false
+				found[childId] = &f
+			}
+		}
+	}
+
+	var tables []uuid.UUID
+	var tableBlockMap map[uuid.UUID]types.Block
+	if hasTable {
+		ids, blockMap, err := s.getTables(ctx, document, index)
+		if err != nil {
+			return nil, nil, nil, nil, err
+		}
+		tables = ids
+		tableBlockMap = blockMap
+	}
+
+	return lines, tables, tableBlockMap, found, nil
+}
+
+func (s *Service) isLayout(blockType types.BlockType) bool {
+	return blockType == types.BlockTypeLayoutText ||
+		blockType == types.BlockTypeLayoutTitle ||
+		blockType == types.BlockTypeLayoutHeader ||
+		blockType == types.BlockTypeLayoutFooter ||
+		blockType == types.BlockTypeLayoutSectionHeader ||
+		blockType == types.BlockTypeLayoutPageNumber ||
+		blockType == types.BlockTypeLayoutList ||
+		blockType == types.BlockTypeLayoutFigure ||
+		blockType == types.BlockTypeLayoutTable ||
+		blockType == types.BlockTypeLayoutKeyValue ||
+		blockType == types.BlockTypeSignature
+}
+
+type buildParams struct {
+	numOfSignatures int
+	currentTable    int
+}
+
+func (s *Service) buildPageText(ctx context.Context, lines []uuid.UUID, document documenttypes.File, blockMap map[uuid.UUID]types.Block, index int, tables []uuid.UUID, tableBlockMap map[uuid.UUID]types.Block, found map[uuid.UUID]*bool) (string, error) {
+	var builder strings.Builder
+	params := buildParams{
+		numOfSignatures: 0,
+		currentTable:    0,
+	}
+
+	for _, id := range lines {
+		if found[id] == nil || !*found[id] {
+			continue
+		}
+		err := s.addComponentText(ctx, blockMap[id], blockMap, tables, tableBlockMap, &params, &builder)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if params.numOfSignatures > 0 {
+		builder.WriteString(fmt.Sprintf("\nThis page has %d signature.\n", params.numOfSignatures))
+	}
+
+	return builder.String(), nil
+}
+
+func (s *Service) addComponentText(ctx context.Context, block types.Block, blockMap map[uuid.UUID]types.Block, tables []uuid.UUID, tableBlockMap map[uuid.UUID]types.Block, params *buildParams, builder *strings.Builder) error {
+	switch block.BlockType {
+	case types.BlockTypeLayoutText, types.BlockTypeLayoutTitle, types.BlockTypeLayoutHeader, types.BlockTypeLayoutFooter, types.BlockTypeLayoutSectionHeader, types.BlockTypeLayoutPageNumber, types.BlockTypeLayoutList, types.BlockTypeLayoutFigure, types.BlockTypeLayoutKeyValue:
+		for _, id := range s.getChildIds(block) {
+			err := s.addComponentText(ctx, blockMap[id], blockMap, tables, tableBlockMap, params, builder)
+			if err != nil {
+				return err
+			}
+		}
+	case types.BlockTypeLine:
+		_, err := builder.WriteString(fmt.Sprintf("%s\n", *block.Text))
+		if err != nil {
+			slog.Error("unable to write string", "error", err)
+			return err
+		}
+	case types.BlockTypeWord:
+		_, err := builder.WriteString(fmt.Sprintf(" %s", *block.Text))
+		if err != nil {
+			slog.Error("unable to write string", "error", err)
+			return err
+		}
+	case types.BlockTypeSignature:
+		params.numOfSignatures++
+	case types.BlockTypeLayoutTable:
+		tableId := tables[params.currentTable]
+		params.currentTable++
+		tableBlock := tableBlockMap[tableId]
+		err := s.addTable(ctx, tableBlock, tableBlockMap, builder)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Service) GetBasePage(ctx context.Context, doc documenttypes.File, index int) (textract.AnalyzeDocumentOutput, error) {
+	pageBytes, err := doc.GetPageAsPNG(ctx, index)
+	if err != nil {
+		return textract.AnalyzeDocumentOutput{}, err
+	}
+
+	res, err := s.cfg.GetTextractClient().AnalyzeDocument(ctx, &textract.AnalyzeDocumentInput{
+		Document: &types.Document{
+			Bytes: pageBytes,
+		},
+		FeatureTypes: []types.FeatureType{
+			types.FeatureTypeLayout,
+			types.FeatureTypeSignatures,
+		},
+	})
+	if err != nil {
+		return textract.AnalyzeDocumentOutput{}, err
+	}
+
+	return *res, err
+}
+
+func (s *Service) getBlockMap(blocksList []types.Block) map[uuid.UUID]types.Block {
+	blocks := make(map[uuid.UUID]types.Block, len(blocksList))
+	for _, block := range blocksList {
+		if block.Id == nil {
+			continue
+		}
+		id, err := uuid.Parse(*block.Id)
+		if err != nil {
+			slog.Warn("error parsing block id", "error", err, "id", *block.Id)
+			continue
+		}
+
+		blocks[id] = block
+	}
+
+	return blocks
+}
+
+func (s *Service) getChildIds(block types.Block) []uuid.UUID {
+	childIds := []uuid.UUID{}
+	for _, relationship := range block.Relationships {
+		if relationship.Type != types.RelationshipTypeChild {
+			slog.Info("not a child element", "type", relationship.Type)
+			continue
+		}
+
+		for _, idStr := range relationship.Ids {
+			id, err := uuid.Parse(idStr)
+			if err != nil {
+				slog.Error("invalid child uuid", "id", id, "error", err)
+				continue
+			}
+
+			childIds = append(childIds, id)
+		}
+	}
+
+	return childIds
+}

internal/document/text/getPages_test.go

@@ -0,0 +1,570 @@
+package documenttext
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"math"
+	"os"
+	"strings"
+	"testing"
+
+	documenttypes "queryorchestration/internal/document/types"
+	textractmock "queryorchestration/mocks/textract"
+
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/aws/aws-sdk-go-v2/service/textract/types"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetBasePage(t *testing.T) {
+	ctx := context.Background()
+	cfg := DocTextConfig{}
+	svc := Service{
+		cfg: &cfg,
+	}
+
+	mockTextract := textractmock.NewMockTextractClient(t)
+	cfg.TextractClient = mockTextract
+	sent := 0
+
+	pdf, err := documenttypes.NewPDFFromReader(strings.NewReader(pdfHelloWorld))
+	require.NoError(t, err)
+
+	textractFile := "../../../assets/sampleGeneration/generated/helloWorld.gen"
+	textractBaseOut := getTextractFileResponse(t, textractFile)
+
+	mockTextract.EXPECT().
+		AnalyzeDocument(
+			mock.Anything,
+			mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+				if sent == 0 {
+					sent++
+					return true
+				}
+				return false
+			}),
+			mock.Anything,
+		).
+		Return(&textractBaseOut["base"][0], nil).Once()
+
+	text, err := svc.GetBasePage(ctx, pdf, 0)
+	require.NoError(t, err)
+	assert.EqualExportedValues(t, textractBaseOut["base"][0], text)
+}
+
+func TestGetDocumentText(t *testing.T) {
+	ctx := context.Background()
+	cfg := DocTextConfig{}
+	svc := Service{
+		cfg: &cfg,
+	}
+
+	t.Run("Hello World", func(t *testing.T) {
+		mockTextract := textractmock.NewMockTextractClient(t)
+		cfg.TextractClient = mockTextract
+		sent := 0
+
+		pdf, err := documenttypes.NewPDFFromReader(strings.NewReader(pdfHelloWorld))
+		require.NoError(t, err)
+
+		textractFile := "../../../assets/sampleGeneration/generated/helloWorld.gen"
+		textractBaseOut := getTextractFileResponse(t, textractFile)
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 0 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][0], nil).Once()
+
+		text, err := svc.getDocumentText(ctx, pdf)
+		require.NoError(t, err)
+		expected := `Start of Page No. = 1
+
+Hello World!
+
+
+`
+		assertStringToReader(t, expected, text)
+	})
+
+	t.Run("sampleOne - cnc_01-06", func(t *testing.T) {
+		mockTextract := textractmock.NewMockTextractClient(t)
+		cfg.TextractClient = mockTextract
+		sent := 0
+
+		textractFile := "../../../assets/sampleOne/generated/cnc_01-06.gen"
+		textractBaseOut := getTextractFileResponse(t, textractFile)
+
+		pdfFile, err := os.Open("../../../assets/sampleOne/cnc_01-06.pdf")
+		require.NoError(t, err)
+		defer pdfFile.Close()
+
+		pdf, err := documenttypes.NewPDFFromReader(pdfFile)
+		require.NoError(t, err)
+
+		txtFile, err := os.Open("../../../assets/sampleOne/cnc_01-06.txt")
+		require.NoError(t, err)
+		defer txtFile.Close()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 0 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][0], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 1 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][1], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 2 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][2], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 3 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][3], nil).Once()
+
+		text, err := svc.getDocumentText(ctx, pdf)
+		require.NoError(t, err)
+
+		assertReaders(t, txtFile, text)
+	})
+	t.Run("sampleOne - hn_0109", func(t *testing.T) {
+		mockTextract := textractmock.NewMockTextractClient(t)
+		cfg.TextractClient = mockTextract
+		sent := 0
+
+		textractFile := "../../../assets/sampleOne/generated/hn_0109.gen"
+		textractBaseOut := getTextractFileResponse(t, textractFile)
+
+		pdfFile, err := os.Open("../../../assets/sampleOne/hn_0109.pdf")
+		require.NoError(t, err)
+		defer pdfFile.Close()
+
+		pdf, err := documenttypes.NewPDFFromReader(pdfFile)
+		require.NoError(t, err)
+
+		txtFile, err := os.Open("../../../assets/sampleOne/hn_0109.txt")
+		require.NoError(t, err)
+		defer txtFile.Close()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 0 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][0], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 1 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][1], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 2 && in.FeatureTypes[0] == types.FeatureTypeTables {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["table"][0], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 3 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][2], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 4 && in.FeatureTypes[0] == types.FeatureTypeTables {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["table"][1], nil).Once()
+
+		mockTextract.EXPECT().
+			AnalyzeDocument(
+				mock.Anything,
+				mock.MatchedBy(func(in *textract.AnalyzeDocumentInput) bool {
+					if sent == 5 {
+						sent++
+						return true
+					}
+					return false
+				}),
+				mock.Anything,
+			).
+			Return(&textractBaseOut["base"][3], nil).Once()
+
+		text, err := svc.getDocumentText(ctx, pdf)
+		require.NoError(t, err)
+
+		assertReaders(t, txtFile, text)
+	})
+}
+
+func TestGetBlockMap(t *testing.T) {
+	svc := Service{}
+	t.Run("0 blocks", func(t *testing.T) {
+		blocks := []types.Block{}
+		outBlocks := svc.getBlockMap(blocks)
+		assert.Equal(t, map[uuid.UUID]types.Block{}, outBlocks)
+	})
+	t.Run("nil id blocks", func(t *testing.T) {
+		blocks := []types.Block{
+			{},
+		}
+		outBlocks := svc.getBlockMap(blocks)
+		assert.Equal(t, map[uuid.UUID]types.Block{}, outBlocks)
+	})
+	t.Run("1 block", func(t *testing.T) {
+		blockId := uuid.New()
+		blockIdStr := blockId.String()
+		blocks := []types.Block{
+			{
+				Id: &blockIdStr,
+			},
+		}
+		outBlocks := svc.getBlockMap(blocks)
+		assert.Equal(t, map[uuid.UUID]types.Block{
+			blockId: {
+				Id: &blockIdStr,
+			},
+		}, outBlocks)
+	})
+	t.Run("invalid block id", func(t *testing.T) {
+		blockIdStr := "blockId.String()"
+		blocks := []types.Block{
+			{
+				Id: &blockIdStr,
+			},
+		}
+		outBlocks := svc.getBlockMap(blocks)
+		assert.Equal(t, map[uuid.UUID]types.Block{}, outBlocks)
+	})
+	t.Run("2 blocks", func(t *testing.T) {
+		blockOneId := uuid.New()
+		blockOneIdStr := blockOneId.String()
+		blockTwoId := uuid.New()
+		blockTwoIdStr := blockTwoId.String()
+		blocks := []types.Block{
+			{
+				Id: &blockOneIdStr,
+			},
+			{
+				Id: &blockTwoIdStr,
+			},
+		}
+		outBlocks := svc.getBlockMap(blocks)
+		assert.Equal(t, map[uuid.UUID]types.Block{
+			blockOneId: {
+				Id: &blockOneIdStr,
+			},
+			blockTwoId: {
+				Id: &blockTwoIdStr,
+			},
+		}, outBlocks)
+	})
+}
+
+func TestMergeDocument(t *testing.T) {
+	svc := Service{}
+	t.Run("0 pages", func(t *testing.T) {
+		pages := []string{}
+		out, err := svc.mergeDocument(pages)
+		require.NoError(t, err)
+		outStr, err := io.ReadAll(out)
+		require.NoError(t, err)
+		assert.Equal(t, ``, string(outStr))
+	})
+	t.Run("1 page", func(t *testing.T) {
+		pages := []string{
+			"page one",
+		}
+		out, err := svc.mergeDocument(pages)
+		require.NoError(t, err)
+		outStr, err := io.ReadAll(out)
+		require.NoError(t, err)
+		assert.Equal(t, `Start of Page No. = 1
+
+page one
+
+`, string(outStr))
+	})
+	t.Run("2 pages", func(t *testing.T) {
+		pages := []string{
+			"page one",
+			"page two",
+		}
+		out, err := svc.mergeDocument(pages)
+		require.NoError(t, err)
+		outStr, err := io.ReadAll(out)
+		require.NoError(t, err)
+		assert.Equal(t, `Start of Page No. = 1
+
+page one
+
+Start of Page No. = 2
+
+page two
+
+`, string(outStr))
+	})
+	t.Run("3 pages", func(t *testing.T) {
+		pages := []string{
+			"page one",
+			"page two",
+			"page three",
+		}
+		out, err := svc.mergeDocument(pages)
+		require.NoError(t, err)
+		outStr, err := io.ReadAll(out)
+		require.NoError(t, err)
+		assert.Equal(t, `Start of Page No. = 1
+
+page one
+
+Start of Page No. = 2
+
+page two
+
+Start of Page No. = 3
+
+page three
+
+`, string(outStr))
+	})
+}
+
+func getTextractFileResponse(t *testing.T, filename string) map[string][]textract.AnalyzeDocumentOutput {
+	file, err := os.Open(filename)
+	require.NoError(t, err)
+	defer file.Close()
+
+	decoder := json.NewDecoder(file)
+	var out map[string][]textract.AnalyzeDocumentOutput
+	err = decoder.Decode(&out)
+	require.NoError(t, err)
+
+	return out
+}
+
+func assertStringToReader(t *testing.T, expected string, actual io.Reader) {
+	actualBytes, err := io.ReadAll(actual)
+	require.NoError(t, err)
+	actualStr := string(actualBytes)
+	if !assert.Equal(t, expected, actualStr) {
+		diff := cmp.Diff(expected, actualStr)
+		t.Logf("Detailed diff (-expected +actual):\n%s", diff)
+	}
+}
+
+func assertReaders(t *testing.T, expected io.Reader, actual io.Reader) {
+	actualBytes, err := io.ReadAll(actual)
+	require.NoError(t, err)
+	actualStr := string(actualBytes)
+
+	expectedBytes, err := io.ReadAll(expected)
+	require.NoError(t, err)
+	expectedStr := string(expectedBytes)
+
+	if !assert.Equal(t, expectedStr, actualStr) {
+		diff := cmp.Diff(expectedStr, actualStr)
+		t.Logf("Detailed diff (-expected +actual):\n%s", diff)
+	}
+}
+
+func levenshteinDistance(s1, s2 string) int {
+	rows, cols := len(s1)+1, len(s2)+1
+	dist := make([][]int, rows)
+	for i := range dist {
+		dist[i] = make([]int, cols)
+	}
+
+	for i := 0; i < rows; i++ {
+		dist[i][0] = i
+	}
+	for j := 0; j < cols; j++ {
+		dist[0][j] = j
+	}
+
+	for i := 1; i < rows; i++ {
+		for j := 1; j < cols; j++ {
+			cost := 1
+			if s1[i-1] == s2[j-1] {
+				cost = 0
+			}
+			dist[i][j] = min(
+				dist[i-1][j]+1,
+				dist[i][j-1]+1,
+				dist[i-1][j-1]+cost,
+			)
+		}
+	}
+
+	return dist[rows-1][cols-1]
+}
+
+func calculateSimilarityPercentage(s1, s2 string) float64 {
+	distance := levenshteinDistance(s1, s2)
+	maxLen := math.Max(float64(len(s1)), float64(len(s2)))
+
+	if maxLen == 0 {
+		return 100.0
+	}
+
+	return 100.0 * (1.0 - float64(distance)/maxLen)
+}
+
+func AreSimilar(s1, s2 string, threshold float64) bool {
+	return calculateSimilarityPercentage(s1, s2) >= threshold
+}
+
+const pdfHelloWorld = `%PDF-1.4
+%����
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+  /Version /1.4
+>>
+endobj
+2 0 obj
+<<
+  /Type /Pages
+  /Kids [3 0 R]
+  /Count 1
+>>
+endobj
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /MediaBox [0 0 612 792]
+  /Resources <<
+    /Font <<
+      /F1 <<
+        /Type /Font
+        /Subtype /Type1
+        /BaseFont /Helvetica
+      >>
+    >>
+  >>
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj
+<<
+  /Length
+  44
+>>
+stream
+BT
+/F1 24 Tf
+100 700 Td
+(Hello World!) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000015 00000 n
+0000000086 00000 n
+0000000151 00000 n
+0000000376 00000 n
+trailer
+<<
+  /Size 5
+  /Root 1 0 R
+>>
+startxref
+472
+%%EOF`

internal/document/text/processExtract_test.go

@@ -1,292 +0,0 @@
-package documenttext
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"io"
-	"strings"
-	"testing"
-	"time"
-
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/serviceconfig/objectstore"
-	objectstoremock "queryorchestration/mocks/objectstore"
-	queuemock "queryorchestration/mocks/queue"
-
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/google/uuid"
-	"github.com/pashagolub/pgxmock/v3"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-)
-
-func TestProcess(t *testing.T) {
-	ctx := context.Background()
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocTextConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	cfg.QuerySyncURL = "querysync"
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
-
-	svc := Service{
-		cfg: cfg,
-	}
-
-	triggerId := uuid.MustParse("6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d")
-	docId := uuid.MustParse("6ac21b2b-f36c-4ae7-a815-307d4a9bfe4a")
-	textractId := "hello"
-	textId := uuid.New()
-	cleanId := uuid.New()
-	hash := `"5d41402abc4b2a76b9719d911017c592"`
-
-	pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).WillReturnRows(
-		pgxmock.NewRows([]string{"id", "cleanId", "version", "textractJobId", "documentId"}).
-			AddRow(triggerId, cleanId, int64(123), &textractId, docId),
-	)
-	pool.ExpectBegin()
-	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
-		pgxmock.NewRows([]string{"id"}).
-			AddRow(textId),
-	)
-	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
-		WillReturnResult(pgxmock.NewResult("", 1))
-	pool.ExpectCommit()
-
-	body := io.NopCloser(strings.NewReader("hello"))
-	mockS3.EXPECT().
-		GetObject(
-			mock.Anything,
-			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
-				return *in.IfMatch == "hash"
-			}),
-			mock.Anything,
-		).
-		Return(&s3.GetObjectOutput{
-			Body: body,
-		}, nil)
-
-	mockSQS.EXPECT().
-		SendMessage(
-			mock.Anything,
-			mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-				return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
-			}),
-			mock.Anything,
-		).
-		Return(&sqs.SendMessageOutput{}, nil)
-
-	key := objectstore.BucketKey{
-		ClientID: "my_client",
-		Location: objectstore.TextTextract,
-		EntityID: triggerId,
-	}
-	err = svc.Process(ctx, &ProcessParams{
-		Bucket: "bucket",
-		Key:    key,
-		Hash:   "hash",
-	})
-	assert.NoError(t, err)
-}
-
-func TestStoreTrigger(t *testing.T) {
-	ctx := context.Background()
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocTextConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	cfg.QuerySyncURL = "querysync"
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
-
-	svc := Service{
-		cfg: cfg,
-	}
-
-	cleanId := uuid.New()
-	textId := uuid.New()
-	triggerId := uuid.New()
-	clientId := "hello"
-	bucket := "bucket"
-
-	t.Run("exists", func(t *testing.T) {
-		hash := `"5d41402abc4b2a76b9719d911017c592"`
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).
-				AddRow(textId),
-		)
-		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		text := strings.NewReader("hello")
-		err = svc.storeProcess(ctx, text, &repository.GetDocumentTextTriggerRow{
-			ID:      triggerId,
-			Cleanid: cleanId,
-		}, &ProcessParams{
-			Bucket: bucket,
-			Key:    objectstore.BucketKey{},
-		})
-		require.NoError(t, err)
-	})
-	t.Run("not exists", func(t *testing.T) {
-		part := uint16(0)
-		key := objectstore.BucketKey{
-			ClientID:  clientId,
-			Location:  objectstore.TextOut,
-			CreatedAt: time.Now().UTC(),
-			EntityID:  triggerId,
-			Part:      &part,
-		}
-		hash := `"09644372e99020106946045c6fd2d70b"`
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
-			WillReturnError(sql.ErrNoRows)
-		pool.ExpectQuery("name: GetTextOutCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"part", "count"}).
-					AddRow(0, 2),
-			)
-		pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash, pgxmock.AnyArg(), uint16(0)).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id"}).
-					AddRow(textId),
-			)
-
-		mockS3.EXPECT().
-			PutObject(
-				mock.Anything,
-				mock.MatchedBy(func(in *s3.PutObjectInput) bool {
-					bod, err := io.ReadAll(in.Body)
-					require.NoError(t, err)
-					return string(bod) == "byebye" && *in.Key == key.String() && *in.Bucket == bucket
-				}),
-				mock.Anything,
-			).
-			Return(&s3.PutObjectOutput{}, nil)
-
-		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		text := strings.NewReader("byebye")
-		err = svc.storeProcess(ctx, text, &repository.GetDocumentTextTriggerRow{
-			ID:      triggerId,
-			Cleanid: cleanId,
-		}, &ProcessParams{
-			Key:    key,
-			Bucket: bucket,
-		})
-		require.NoError(t, err)
-	})
-}
-
-func TestDuplicateReader(t *testing.T) {
-	original := strings.NewReader("hello")
-	readerOne, readerTwo, err := duplicateReader(original)
-	require.NoError(t, err)
-	contentOne, err := io.ReadAll(readerOne)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("hello"), contentOne)
-	contentTwo, err := io.ReadAll(readerTwo)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("hello"), contentTwo)
-}
-
-func TestProcessTrigger(t *testing.T) {
-	ctx := context.Background()
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocTextConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	cfg.QuerySyncURL = "querysync"
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
-
-	svc := Service{
-		cfg: cfg,
-	}
-
-	cleanId := uuid.New()
-	textId := uuid.New()
-	triggerId := uuid.New()
-	clientId := "hello"
-	bucket := "bucket"
-	key := objectstore.BucketKey{
-		ClientID:  clientId,
-		Location:  objectstore.TextOut,
-		EntityID:  triggerId,
-		CreatedAt: time.Now().UTC(),
-	}
-	hash := `"5d41402abc4b2a76b9719d911017c592"`
-
-	pool.ExpectBegin()
-	pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
-		WillReturnError(sql.ErrNoRows)
-	pool.ExpectQuery("name: GetTextOutCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
-		WillReturnRows(
-			pgxmock.NewRows([]string{"part", "count"}).
-				AddRow(0, 2),
-		)
-	pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash, pgxmock.AnyArg(), uint16(0)).
-		WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).
-				AddRow(textId),
-		)
-
-	body := io.NopCloser(strings.NewReader("hello"))
-	mockS3.EXPECT().
-		GetObject(
-			mock.Anything,
-			mock.MatchedBy(func(in *s3.GetObjectInput) bool {
-				return *in.Key == key.String() && *in.Bucket == bucket && *in.IfMatch == hash
-			}),
-			mock.Anything,
-		).
-		Return(&s3.GetObjectOutput{
-			Body: body,
-		}, nil)
-
-	mockS3.EXPECT().
-		PutObject(
-			mock.Anything,
-			mock.MatchedBy(func(in *s3.PutObjectInput) bool {
-				bod, err := io.ReadAll(in.Body)
-				require.NoError(t, err)
-				return string(bod) == "hello" && *in.Key == key.String() && *in.Bucket == bucket
-			}),
-			mock.Anything,
-		).
-		Return(&s3.PutObjectOutput{}, nil)
-
-	pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
-		WillReturnResult(pgxmock.NewResult("", 1))
-	pool.ExpectCommit()
-
-	err = svc.processTrigger(ctx, &repository.GetDocumentTextTriggerRow{
-		ID:      triggerId,
-		Cleanid: cleanId,
-	}, &ProcessParams{
-		Bucket: bucket,
-		Key:    key,
-		Hash:   hash,
-	})
-	require.NoError(t, err)
-}

internal/document/text/storeExtract.go

@@ -6,7 +6,6 @@ import (
 	"database/sql"
 	"errors"
 	"io"
-	"strings"
 	"time"
 
 	"queryorchestration/internal/database/repository"
@@ -18,62 +17,7 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
-type ProcessParams struct {
-	Bucket string
-	Key    objectstore.BucketKey
-	Hash   string
-}
-
-type Page string
-
-func (s *Service) getPages(baseTextract io.Reader) ([]*Page, error) {
-	// For each page
-	// Look for table, form, signature
-	// If one/many found - run appropriate tier and merge in
-	// tiers?? - analyze (for tables?)
-
-	// cleanPages
-	// linkPages
-
-	return []*Page{}, nil
-}
-
-func (s *Service) mergePages(pages []*Page) (io.Reader, error) {
-	// merge document - pages to text
-	// e.g. common tables
-	return strings.NewReader("hello"), nil
-}
-
-func (s *Service) processTrigger(ctx context.Context, trigger *repository.GetDocumentTextTriggerRow, params *ProcessParams) error {
-	keyStr := params.Key.String()
-	response, err := s.cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{
-		Bucket:  &params.Bucket,
-		Key:     &keyStr,
-		IfMatch: &params.Hash,
-	})
-	if err != nil {
-		return err
-	}
-
-	pages, err := s.getPages(response.Body)
-	if err != nil {
-		return err
-	}
-
-	text, err := s.mergePages(pages)
-	if err != nil {
-		return err
-	}
-
-	err = s.storeProcess(ctx, text, trigger, params)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *repository.GetDocumentTextTriggerRow, params *ProcessParams) error {
+func (s *Service) storeProcess(ctx context.Context, text io.Reader, clean *repository.Currentcleanentry) error {
 	version := build.GetVersionUnixTimestamp()
 
 	hashReader, storeReader, err := duplicateReader(text)
@@ -88,7 +32,7 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 
 	return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
 		existingId, err := q.GetDocumentTextExtractionByHash(ctx, &repository.GetDocumentTextExtractionByHashParams{
-			Cleanentryid: trigger.Cleanid,
+			Cleanentryid: clean.ID,
 			Hash:         hash,
 		})
 		if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -99,10 +43,10 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 		if existingId == uuid.Nil || errors.Is(err, sql.ErrNoRows) {
 			createdAt := time.Now().UTC()
 			key := objectstore.BucketKey{
-				ClientID:  params.Key.ClientID,
-				Location:  objectstore.TextOut,
+				ClientID:  clean.Clientid,
+				Location:  objectstore.Text,
 				CreatedAt: createdAt,
-				EntityID:  trigger.ID,
+				EntityID:  clean.ID,
 			}
 
 			currentPart, err := q.GetTextOutCurrentPart(ctx, &repository.GetTextOutCurrentPartParams{
@@ -122,10 +66,11 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 			keyStr := key.String()
 
 			textId, err = q.AddDocumentText(ctx, &repository.AddDocumentTextParams{
-				Bucket: params.Bucket,
-				Key:    keyStr,
-				Hash:   hash,
-				Part:   *key.Part,
+				Cleanid: clean.ID,
+				Bucket:  *clean.Bucket,
+				Key:     keyStr,
+				Hash:    hash,
+				Part:    *key.Part,
 				Createdat: pgtype.Timestamp{
 					Time:  key.CreatedAt,
 					Valid: true,
@@ -136,7 +81,7 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 			}
 
 			_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
-				Bucket: &params.Bucket,
+				Bucket: clean.Bucket,
 				Key:    &keyStr,
 				Body:   storeReader,
 			})
@@ -149,9 +94,8 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 		}
 
 		err = q.AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
-			Textid:    textId,
-			Triggerid: trigger.ID,
-			Version:   version,
+			Textid:  textId,
+			Version: version,
 		})
 		if err != nil {
 			return err
@@ -161,25 +105,6 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
 	})
 }
 
-func (s *Service) Process(ctx context.Context, params *ProcessParams) error {
-	trigger, err := s.cfg.GetDBQueries().GetDocumentTextTrigger(ctx, params.Key.EntityID)
-	if err != nil {
-		return err
-	}
-
-	err = s.processTrigger(ctx, trigger, params)
-	if err != nil {
-		return err
-	}
-
-	err = s.informExtraction(ctx, trigger.Documentid)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func duplicateReader(reader io.Reader) (io.Reader, io.Reader, error) {
 	content, err := io.ReadAll(reader)
 	if err != nil {

internal/document/text/storeExtract_test.go

@@ -0,0 +1,126 @@
+package documenttext
+
+import (
+	"context"
+	"database/sql"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	objectstoremock "queryorchestration/mocks/objectstore"
+	queuemock "queryorchestration/mocks/queue"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStoreExtract(t *testing.T) {
+	ctx := context.Background()
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+
+	cfg := &DocTextConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	cfg.QuerySyncURL = "querysync"
+	mockSQS := queuemock.NewMockSQSClient(t)
+	cfg.QueueClient = mockSQS
+	mockS3 := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockS3
+
+	svc := Service{
+		cfg: cfg,
+	}
+
+	cleanId := uuid.New()
+	textId := uuid.New()
+	clientId := "hello"
+	bucket := "bucket"
+
+	t.Run("exists", func(t *testing.T) {
+		hash := `"5d41402abc4b2a76b9719d911017c592"`
+		pool.ExpectBegin()
+		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(textId),
+		)
+		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		text := strings.NewReader("hello")
+		err = svc.storeProcess(ctx, text, &repository.Currentcleanentry{
+			ID: cleanId,
+		})
+		require.NoError(t, err)
+	})
+	t.Run("not exists", func(t *testing.T) {
+		part := uint16(0)
+		key := objectstore.BucketKey{
+			ClientID:  clientId,
+			Location:  objectstore.Text,
+			CreatedAt: time.Now().UTC(),
+			EntityID:  cleanId,
+			Part:      &part,
+		}
+		keyStr := key.String()
+		hash := `"09644372e99020106946045c6fd2d70b"`
+		pool.ExpectBegin()
+		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
+			WillReturnError(sql.ErrNoRows)
+		pool.ExpectQuery("name: GetTextOutCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
+			WillReturnRows(
+				pgxmock.NewRows([]string{"part", "count"}).
+					AddRow(0, 2),
+			)
+		pool.ExpectQuery("name: AddDocumentText :one").WithArgs(cleanId, bucket, key.String(), hash, pgxmock.AnyArg(), uint16(0)).
+			WillReturnRows(
+				pgxmock.NewRows([]string{"id"}).
+					AddRow(textId),
+			)
+
+		mockS3.EXPECT().
+			PutObject(
+				mock.Anything,
+				mock.MatchedBy(func(in *s3.PutObjectInput) bool {
+					bod, err := io.ReadAll(in.Body)
+					require.NoError(t, err)
+					return string(bod) == "byebye" && *in.Key == key.String() && *in.Bucket == bucket
+				}),
+				mock.Anything,
+			).
+			Return(&s3.PutObjectOutput{}, nil)
+
+		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		text := strings.NewReader("byebye")
+		err = svc.storeProcess(ctx, text, &repository.Currentcleanentry{
+			ID:       cleanId,
+			Clientid: clientId,
+			Bucket:   &bucket,
+			Key:      &keyStr,
+		})
+		require.NoError(t, err)
+	})
+}
+
+func TestDuplicateReader(t *testing.T) {
+	original := strings.NewReader("hello")
+	readerOne, readerTwo, err := duplicateReader(original)
+	require.NoError(t, err)
+	contentOne, err := io.ReadAll(readerOne)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("hello"), contentOne)
+	contentTwo, err := io.ReadAll(readerTwo)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("hello"), contentTwo)
+}

internal/document/text/tables.go

@@ -0,0 +1,109 @@
+package documenttext
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"strings"
+
+	documenttypes "queryorchestration/internal/document/types"
+
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/aws/aws-sdk-go-v2/service/textract/types"
+	"github.com/google/uuid"
+)
+
+type cell struct {
+	value  string
+	row    int
+	column int
+}
+
+func (s *Service) addTable(ctx context.Context, block types.Block, blockMap map[uuid.UUID]types.Block, builder *strings.Builder) error {
+	table := []cell{}
+	maxRow := 0
+	maxColumn := 0
+
+	for _, uid := range s.getChildIds(block) {
+		childBlock := blockMap[uid]
+		if childBlock.BlockType != types.BlockTypeCell {
+			continue
+		}
+
+		var str strings.Builder
+		for _, uid := range s.getChildIds(childBlock) {
+			if blockMap[uid].BlockType == types.BlockTypeWord {
+				if str.String() != "" {
+					str.WriteString(" ")
+				}
+				str.WriteString(*blockMap[uid].Text)
+			}
+			if maxRow < int(*childBlock.RowIndex) {
+				maxRow = int(*childBlock.RowIndex)
+			}
+			if maxColumn < int(*childBlock.ColumnIndex) {
+				maxColumn = int(*childBlock.ColumnIndex)
+			}
+			table = append(table, cell{
+				column: int(*childBlock.ColumnIndex) - 1,
+				row:    int(*childBlock.RowIndex) - 1,
+				value:  str.String(),
+			})
+		}
+	}
+
+	tableArr := make([][]string, maxRow)
+	for i := range tableArr {
+		tableArr[i] = make([]string, maxColumn)
+	}
+
+	for _, cell := range table {
+		tableArr[cell.row][cell.column] = cell.value
+	}
+
+	jsonData, err := json.Marshal(tableArr)
+	if err != nil {
+		return err
+	}
+
+	builder.WriteString("-------Table Start--------\n")
+	builder.WriteString(string(jsonData))
+	builder.WriteString("\n-------Table End--------\n")
+
+	return nil
+}
+
+func (s *Service) getTables(ctx context.Context, document documenttypes.File, index int) ([]uuid.UUID, map[uuid.UUID]types.Block, error) {
+	pageBytes, err := document.GetPageAsPNG(ctx, index)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	res, err := s.cfg.GetTextractClient().AnalyzeDocument(ctx, &textract.AnalyzeDocumentInput{
+		Document: &types.Document{
+			Bytes: pageBytes,
+		},
+		FeatureTypes: []types.FeatureType{
+			types.FeatureTypeTables,
+		},
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	tables := []uuid.UUID{}
+	for _, block := range res.Blocks {
+		if block.BlockType == types.BlockTypeTable {
+			id, err := uuid.Parse(*block.Id)
+			if err != nil {
+				slog.Warn("error parsing block id", "error", err, "id", *block.Id)
+				continue
+			}
+
+			tables = append(tables, id)
+		}
+	}
+	blocks := s.getBlockMap(res.Blocks)
+
+	return tables, blocks, nil
+}

internal/document/text/triggerExtract.go

@@ -1,87 +0,0 @@
-package documenttext
-
-import (
-	"context"
-	"errors"
-	"time"
-
-	"queryorchestration/internal/database/repository"
-	"queryorchestration/internal/serviceconfig/build"
-	"queryorchestration/internal/serviceconfig/objectstore"
-
-	"github.com/aws/aws-sdk-go-v2/service/textract"
-	"github.com/aws/aws-sdk-go-v2/service/textract/types"
-	"github.com/jackc/pgx/v5/pgtype"
-)
-
-func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentcleanentry) error {
-	version := build.GetVersionUnixTimestamp()
-
-	return s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
-		key := objectstore.BucketKey{
-			CreatedAt: time.Now().UTC(),
-			Location:  objectstore.TextTextract,
-			ClientID:  clean.Clientid,
-		}
-
-		currentPart, err := q.GetTextractOutputCurrentPart(ctx, &repository.GetTextractOutputCurrentPartParams{
-			Clientid: &clean.Clientid,
-			Querydate: pgtype.Timestamptz{
-				Time:  key.CreatedAt,
-				Valid: true,
-			},
-		})
-		if err != nil {
-			return err
-		}
-
-		part := s.cfg.GetDirectoryPart(currentPart.Part, currentPart.Count)
-		key.Part = &part
-
-		triggerId, err := s.cfg.GetDBQueries().AddDocumentTextTrigger(ctx, &repository.AddDocumentTextTriggerParams{
-			Cleanid: clean.ID,
-			Version: version,
-			Createdat: pgtype.Timestamp{
-				Time:  key.CreatedAt,
-				Valid: true,
-			},
-			Part: *key.Part,
-		})
-		if err != nil {
-			return err
-		}
-
-		key.EntityID = triggerId
-		keyStr := key.String()
-		jobTag := triggerId.String()
-
-		res, err := s.cfg.GetTextractClient().StartDocumentTextDetection(ctx, &textract.StartDocumentTextDetectionInput{
-			DocumentLocation: &types.DocumentLocation{
-				S3Object: &types.S3Object{
-					Bucket: clean.Bucket,
-					Name:   clean.Key,
-				},
-			},
-			JobTag: &jobTag,
-			OutputConfig: &types.OutputConfig{
-				S3Bucket: clean.Bucket,
-				S3Prefix: &keyStr,
-			},
-		})
-		if err != nil {
-			return err
-		} else if res.JobId == nil {
-			return errors.New("No job id returned from textract")
-		}
-
-		err = s.cfg.GetDBQueries().AddDocumentTextTriggerJobId(ctx, &repository.AddDocumentTextTriggerJobIdParams{
-			ID:            triggerId,
-			Textractjobid: res.JobId,
-		})
-		if err != nil {
-			return err
-		}
-
-		return nil
-	})
-}

internal/document/text/triggerExtract_test.go

@@ -1,85 +0,0 @@
-package documenttext
-
-import (
-	"context"
-	"testing"
-
-	"queryorchestration/internal/database/repository"
-	objectstoremock "queryorchestration/mocks/objectstore"
-	textractmock "queryorchestration/mocks/textract"
-
-	"github.com/aws/aws-sdk-go-v2/service/textract"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/require"
-
-	"github.com/google/uuid"
-	"github.com/pashagolub/pgxmock/v3"
-)
-
-func TestTriggerExtract(t *testing.T) {
-	ctx := context.Background()
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocTextConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	mockS3 := objectstoremock.NewMockS3Client(t)
-	cfg.StoreClient = mockS3
-	mockTextract := textractmock.NewMockTextractClient(t)
-	cfg.TextractClient = mockTextract
-
-	svc := Service{
-		cfg: cfg,
-	}
-
-	t.Run("successful clean", func(t *testing.T) {
-		cleanId := uuid.New()
-		bucket := "bucket_name"
-		key := "/i/am/here"
-		hash := "hhasshhh"
-		clientId := "AA"
-
-		jobId := "hello"
-		triggerId := uuid.New()
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: GetTextractOutputCurrentPart :one").WithArgs(&clientId, pgxmock.AnyArg()).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"part", "count"}).
-					AddRow(0, 2),
-			)
-
-		pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg(), pgxmock.AnyArg(), uint16(0)).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
-		)
-
-		mockTextract.EXPECT().
-			StartDocumentTextDetection(
-				mock.Anything,
-				mock.MatchedBy(func(in *textract.StartDocumentTextDetectionInput) bool {
-					return true
-				}),
-				mock.Anything,
-			).
-			Return(&textract.StartDocumentTextDetectionOutput{
-				JobId: &jobId,
-			}, nil)
-
-		pool.ExpectExec("name: AddDocumentTextTriggerJobId :exec").WithArgs(&jobId, triggerId).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-
-		err = svc.triggerExtract(ctx, &repository.Currentcleanentry{
-			ID:       cleanId,
-			Clientid: clientId,
-			Key:      &key,
-			Bucket:   &bucket,
-			Hash:     &hash,
-			Mimetype: repository.NullCleanmimetype{
-				Valid:         true,
-				Cleanmimetype: repository.CleanmimetypeApplicationPdf,
-			},
-		})
-		require.NoError(t, err)
-	})
-}

internal/document/types/contentType.go

@@ -1,5 +1,14 @@
 package documenttypes
 
+import (
+	"context"
+	"fmt"
+
+	"queryorchestration/internal/serviceconfig/objectstore"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
 type MimeType string
 
 const (
@@ -7,3 +16,43 @@ const (
 	MimeTypeBinaryOctetStream MimeType = "binary/octet-stream"
 	MimeTypeInvalid           MimeType = "invalid"
 )
+
+type File interface {
+	IsCorrupt(context.Context) *InvalidDocumentReason
+	GetPageAsPNG(context.Context, int) ([]byte, error)
+	GetPage(context.Context, int) ([]byte, error)
+	GetPageCount(context.Context) (int, error)
+}
+type GetFileParams struct {
+	Bucket   string
+	Key      objectstore.BucketKey
+	Hash     string
+	Mimetype MimeType
+}
+
+func GetFile(ctx context.Context, cfg objectstore.ConfigProvider, params GetFileParams) (File, error) {
+	keyStr := params.Key.String()
+	resp, err := cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{
+		Bucket:  &params.Bucket,
+		Key:     &keyStr,
+		IfMatch: &params.Hash,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var file File
+	switch params.Mimetype {
+	case MimeTypePDF:
+		pdf, err := NewPDFFromReader(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		file = pdf
+	default:
+		return nil, fmt.Errorf("invalid mimetype: %s", params.Mimetype)
+	}
+
+	return file, nil
+}

internal/document/types/invalid.go

@@ -7,7 +7,6 @@ const (
 	InvalidDocumentRead            InvalidDocumentReason = "invalid_read"
 	InvalidDocumentReadPages       InvalidDocumentReason = "invalid_read_pages"
 	InvalidDocumentZeroPageCount   InvalidDocumentReason = "zero_page_count"
-	InvalidDocumentLargePageCount  InvalidDocumentReason = "large_page_count"
 	InvalidDocumentLargeFile       InvalidDocumentReason = "large_file"
 	InvalidDocumentSmallDimensions InvalidDocumentReason = "small_dimensions"
 	InvalidDocumentLargeDimensions InvalidDocumentReason = "large_dimensions"

internal/document/types/parsetype.go

@@ -30,8 +30,6 @@ func ToDBFailType(t InvalidDocumentReason) repository.Cleanfailtype {
 		return repository.CleanfailtypeInvalidReadPages
 	case InvalidDocumentZeroPageCount:
 		return repository.CleanfailtypeZeroPageCount
-	case InvalidDocumentLargePageCount:
-		return repository.CleanfailtypeLargePageCount
 	case InvalidDocumentLargeFile:
 		return repository.CleanfailtypeLargeFile
 	case InvalidDocumentSmallDimensions:
@@ -57,8 +55,6 @@ func ParseDBFailType(t repository.Cleanfailtype) InvalidDocumentReason {
 		return InvalidDocumentReadPages
 	case repository.CleanfailtypeZeroPageCount:
 		return InvalidDocumentZeroPageCount
-	case repository.CleanfailtypeLargePageCount:
-		return InvalidDocumentLargePageCount
 	case repository.CleanfailtypeLargeFile:
 		return InvalidDocumentLargeFile
 	case repository.CleanfailtypeSmallDimensions:

internal/document/types/parsetypes_test.go

@@ -75,9 +75,6 @@ func TestParseDBFailType(t *testing.T) {
 	t.Run("zero page count", func(t *testing.T) {
 		assert.Equal(t, repository.CleanfailtypeZeroPageCount, documenttypes.ToDBFailType(documenttypes.InvalidDocumentZeroPageCount))
 	})
-	t.Run("large page count", func(t *testing.T) {
-		assert.Equal(t, repository.CleanfailtypeLargePageCount, documenttypes.ToDBFailType(documenttypes.InvalidDocumentLargePageCount))
-	})
 	t.Run("large file", func(t *testing.T) {
 		assert.Equal(t, repository.CleanfailtypeLargeFile, documenttypes.ToDBFailType(documenttypes.InvalidDocumentLargeFile))
 	})
@@ -131,9 +128,6 @@ func TestToDBFailType(t *testing.T) {
 	t.Run("zero page count", func(t *testing.T) {
 		assert.Equal(t, documenttypes.InvalidDocumentZeroPageCount, documenttypes.ParseDBFailType(repository.CleanfailtypeZeroPageCount))
 	})
-	t.Run("large page count", func(t *testing.T) {
-		assert.Equal(t, documenttypes.InvalidDocumentLargePageCount, documenttypes.ParseDBFailType(repository.CleanfailtypeLargePageCount))
-	})
 	t.Run("large file", func(t *testing.T) {
 		assert.Equal(t, documenttypes.InvalidDocumentLargeFile, documenttypes.ParseDBFailType(repository.CleanfailtypeLargeFile))
 	})

internal/document/types/pdf.go

@@ -3,8 +3,12 @@ package documenttypes
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"log/slog"
 
+	"github.com/gen2brain/go-fitz"
+	"github.com/pdfcpu/pdfcpu/pkg/api"
 	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu"
 	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/model"
 	"github.com/pdfcpu/pdfcpu/pkg/pdfcpu/types"
@@ -13,24 +17,24 @@ import (
 type PDF struct {
 	pdf          io.ReadSeeker
 	config       *model.Configuration
-	maxPages     int
 	maxBytes     int64
 	maxDimension int
 	minDimension int
 	minDPI       int
+	pngDPI       int
 }
 
 const (
-	TEXTRACT_SYNC_LIMIT    int64 = 10 * 1024 * 1024
+	TEXTRACT_SYNC_LIMIT    int64 = 5 * 1024 * 1024
 	TEXTRACT_ASYNC_LIMIT   int64 = 500 * 1024 * 1024
-	TEXTRACT_PAGE_LIMIT          = 3000
 	TEXTRACT_MAX_DIMENSION       = 10000
 	TEXTRACT_MIN_DIMENSION       = 50
 	TEXTRACT_MIN_DPI             = 72
 	PDF_DEFAULT_DPI              = 72
+	PNG_DEFAULT_DPI              = 72
 )
 
-func NewPDFFromReadCloser(buf io.ReadCloser) (*PDF, error) {
+func NewPDFFromReader(buf io.Reader) (*PDF, error) {
 	seek, err := ReaderToSeeker(buf)
 	if err != nil {
 		return nil, err
@@ -49,14 +53,69 @@ func NewPDF(buf io.ReadSeeker) *PDF {
 	return &PDF{
 		pdf:          buf,
 		config:       config,
-		maxPages:     TEXTRACT_PAGE_LIMIT,
 		maxBytes:     TEXTRACT_SYNC_LIMIT,
 		maxDimension: TEXTRACT_MAX_DIMENSION,
 		minDimension: TEXTRACT_MIN_DIMENSION,
 		minDPI:       TEXTRACT_MIN_DPI,
+		pngDPI:       PNG_DEFAULT_DPI,
 	}
 }
 
+func (s *PDF) GetPageCount(ctx context.Context) (int, error) {
+	pdfCtx, err := pdfcpu.ReadWithContext(ctx, s.pdf, s.config)
+	if err != nil {
+		return 0, err
+	}
+
+	err = pdfCtx.EnsurePageCount()
+	if err != nil {
+		return 0, err
+	}
+
+	return pdfCtx.PageCount, nil
+}
+
+func (s *PDF) GetPage(ctx context.Context, index int) ([]byte, error) {
+	pdfCtx, err := pdfcpu.ReadWithContext(ctx, s.pdf, s.config)
+	if err != nil {
+		return nil, err
+	}
+
+	err = pdfCtx.EnsurePageCount()
+	if err != nil {
+		return nil, err
+	}
+
+	extractedReader, err := api.ExtractPage(pdfCtx, index+1)
+	if err != nil {
+		return nil, err
+	}
+
+	return io.ReadAll(extractedReader)
+}
+
+func (s *PDF) GetPageAsPNG(ctx context.Context, index int) ([]byte, error) {
+	pageBytes, err := s.GetPage(ctx, index)
+	if err != nil {
+		return nil, err
+	}
+
+	reader := bytes.NewReader(pageBytes)
+
+	fitzdoc, err := fitz.NewFromReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	defer fitzdoc.Close()
+
+	img, err := fitzdoc.ImagePNG(0, float64(s.pngDPI))
+	if err != nil {
+		return nil, fmt.Errorf("failed to render page %d: %v", index, err)
+	}
+
+	return img, nil
+}
+
 func (s *PDF) IsCorrupt(ctx context.Context) *InvalidDocumentReason {
 	pdfCtx, err := pdfcpu.ReadWithContext(ctx, s.pdf, s.config)
 	if err != nil {
@@ -64,34 +123,29 @@ func (s *PDF) IsCorrupt(ctx context.Context) *InvalidDocumentReason {
 		return &reason
 	}
 
-	if int64(pdfCtx.Read.ReadFileSize()) > s.maxBytes {
-		reason := InvalidDocumentLargeFile
-		return &reason
-	}
-
-	docErr := s.isValidPageCount(pdfCtx)
-	if docErr != nil {
-		return docErr
-	}
-
-	docErr = s.isValidDimensions(pdfCtx)
-	if docErr != nil {
-		return docErr
-	}
-
-	return nil
-}
-
-func (s *PDF) isValidPageCount(pdfCtx *model.Context) *InvalidDocumentReason {
-	err := pdfCtx.EnsurePageCount()
+	count, err := s.GetPageCount(ctx)
 	if err != nil {
 		reason := InvalidDocumentRead
 		return &reason
 	}
 
-	if pdfCtx.PageCount > s.maxPages {
-		reason := InvalidDocumentLargePageCount
-		return &reason
+	for i := range count {
+		page, err := s.GetPage(ctx, i)
+		if err != nil {
+			slog.Error("unable to get page", "index", i, "error", err)
+			reason := InvalidDocumentRead
+			return &reason
+		}
+
+		if int64(len(page)) > s.maxBytes {
+			reason := InvalidDocumentLargeFile
+			return &reason
+		}
+	}
+
+	docErr := s.isValidDimensions(pdfCtx)
+	if docErr != nil {
+		return docErr
 	}
 
 	return nil
@@ -160,13 +214,11 @@ func (s *PDF) isValidDPI(mediaBox *types.Rectangle, dims types.Dim) *InvalidDocu
 	return nil
 }
 
-func ReaderToSeeker(readCloser io.ReadCloser) (io.ReadSeeker, error) {
+func ReaderToSeeker(readCloser io.Reader) (io.ReadSeeker, error) {
 	data, err := io.ReadAll(readCloser)
 	if err != nil {
 		return nil, err
 	}
 
-	readCloser.Close()
-
 	return bytes.NewReader(data), nil
 }

internal/document/types/pdf_test.go

@@ -2,7 +2,10 @@ package documenttypes
 
 import (
 	"context"
+	"io"
 	"math"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -165,27 +168,21 @@ func TestIsValidDimensions(t *testing.T) {
 	})
 }
 
-func TestIsValidPageCount(t *testing.T) {
+func TestGetPageCount(t *testing.T) {
 	ctx := context.Background()
 	t.Run("valid", func(t *testing.T) {
 		pdf := NewPDF(strings.NewReader(pdfHelloWorld))
 
-		pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
-		require.NoError(t, err)
-
-		reason := pdf.isValidPageCount(pdfCtx)
-		assert.Nil(t, reason)
+		count, err := pdf.GetPageCount(ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, count)
 	})
 	t.Run("large page count", func(t *testing.T) {
 		pdf := NewPDF(strings.NewReader(pdfTwoPages))
 
-		pdfCtx, err := pdfcpu.ReadWithContext(ctx, pdf.pdf, pdf.config)
-		require.NoError(t, err)
-
-		pdf.maxPages = 1
-
-		reason := pdf.isValidPageCount(pdfCtx)
-		assert.Equal(t, InvalidDocumentLargePageCount, *reason)
+		count, err := pdf.GetPageCount(ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, 2, count)
 	})
 }
 
@@ -217,13 +214,6 @@ func TestIsCorrupt(t *testing.T) {
 		reason := pdf.IsCorrupt(ctx)
 		assert.Equal(t, InvalidDocumentLargeDimensions, *reason)
 	})
-	t.Run("large page count", func(t *testing.T) {
-		pdf := NewPDF(strings.NewReader(pdfTwoPages))
-		pdf.maxPages = 1
-
-		reason := pdf.IsCorrupt(ctx)
-		assert.Equal(t, InvalidDocumentLargePageCount, *reason)
-	})
 	t.Run("version 1.0", func(t *testing.T) {
 		pdf := NewPDF(strings.NewReader(PDF1_0))
 
@@ -278,6 +268,62 @@ func TestIsCorrupt(t *testing.T) {
 		reason := pdf.IsCorrupt(ctx)
 		assert.Nil(t, reason)
 	})
+	t.Run("validate pdfs in assets/Generation", func(t *testing.T) {
+		var pdfFiles []string
+		dir := "../../../assets/sampleGeneration"
+
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !info.IsDir() && strings.ToLower(filepath.Ext(path)) == ".pdf" {
+				pdfFiles = append(pdfFiles, path)
+			}
+			return nil
+		})
+		require.NoError(t, err)
+
+		for _, filename := range pdfFiles {
+			file, err := os.Open(filename)
+			require.NoError(t, err)
+			defer file.Close()
+
+			readSeeker := io.ReadSeeker(file)
+
+			pdf := NewPDF(readSeeker)
+			docErr := pdf.IsCorrupt(ctx)
+			assert.Nil(t, docErr)
+		}
+	})
+	t.Run("validate pdfs in assets/sampleOne", func(t *testing.T) {
+		var pdfFiles []string
+		dir := "../../../assets/sampleOne"
+
+		err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !info.IsDir() && strings.ToLower(filepath.Ext(path)) == ".pdf" {
+				pdfFiles = append(pdfFiles, path)
+			}
+			return nil
+		})
+		require.NoError(t, err)
+
+		for _, filename := range pdfFiles {
+			file, err := os.Open(filename)
+			require.NoError(t, err)
+			defer file.Close()
+
+			readSeeker := io.ReadSeeker(file)
+
+			pdf := NewPDF(readSeeker)
+			docErr := pdf.IsCorrupt(ctx)
+			assert.Nil(t, docErr)
+		}
+	})
 }
 
 func TestNewPDF(t *testing.T) {
@@ -285,8 +331,21 @@ func TestNewPDF(t *testing.T) {
 		pdf := NewPDF(strings.NewReader(pdfHelloWorld))
 		assert.NotNil(t, pdf.pdf)
 		assert.NotNil(t, pdf.config)
-		assert.Equal(t, TEXTRACT_PAGE_LIMIT, pdf.maxPages)
 		assert.Equal(t, TEXTRACT_SYNC_LIMIT, pdf.maxBytes)
+		assert.Equal(t, TEXTRACT_MIN_DIMENSION, pdf.minDimension)
+		assert.Equal(t, TEXTRACT_MAX_DIMENSION, pdf.maxDimension)
+		assert.Equal(t, TEXTRACT_MIN_DPI, pdf.minDPI)
+		assert.Equal(t, PNG_DEFAULT_DPI, pdf.pngDPI)
+	})
+}
+
+func TestPDFGetPage(t *testing.T) {
+	ctx := context.Background()
+	t.Run("hello world", func(t *testing.T) {
+		pdf := NewPDF(strings.NewReader(PDF1_7))
+		pageZero, err := pdf.GetPage(ctx, 0)
+		assert.NoError(t, err)
+		assert.Contains(t, string(pageZero), "PDF 1.7 Example")
 	})
 }
 
@@ -413,51 +472,47 @@ startxref
 %%EOF`
 
 const PDF1_2 = `%PDF-1.2
+%¥±ë
+
 1 0 obj
-<< /Type /Catalog
-   /Pages 2 0 R
->>
+<</Type/Catalog/Pages 2 0 R>>
 endobj
 
 2 0 obj
-<< /Type /Pages
-   /Kids [3 0 R]
-   /Count 1
->>
+<</Type/Pages/Kids[3 0 R]/Count 1>>
 endobj
 
 3 0 obj
-<< /Type /Page
-   /Parent 2 0 R
-   /MediaBox [0 0 612 792]
-   /Contents 4 0 R
->>
+<</Type/Page/Parent 2 0 R/Resources<</Font<</F1 4 0 R>>>>/MediaBox[0 0 612 792]/Contents 5 0 R>>
 endobj
 
 4 0 obj
-<< /Length 65 /Filter /FlateDecode >>
+<</Type/Font/Subtype/Type1/BaseFont/Helvetica>>
+endobj
+
+5 0 obj
+<</Length 44>>
 stream
-x�s
-�2�30W0 P&� �
-�
-�
-�(X�Y��\��R!$�K��H����PS!$��RR�P�� �)�
+BT
+/F1 24 Tf
+100 700 Td
+(hello version 1.2) Tj
+ET
 endstream
 endobj
 
 xref
-0 5
+0 6
 0000000000 65535 f
-0000000009 00000 n
+0000000018 00000 n
 0000000063 00000 n
-0000000125 00000 n
-0000000210 00000 n
+0000000114 00000 n
+0000000223 00000 n
+0000000285 00000 n
 trailer
-<< /Size 5
-   /Root 1 0 R
->>
+<</Size 6/Root 1 0 R>>
 startxref
-325
+377
 %%EOF`
 
 const PDF1_3 = `%PDF-1.3

internal/query/result/set/set_test.go

@@ -67,8 +67,8 @@ func TestSet(t *testing.T) {
 	textEntryId := uuid.New()
 	pool.ExpectQuery("name: GetTextEntryByDocId :one").WithArgs(params.DocumentID).
 		WillReturnRows(
-			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "hash", "cleanId", "triggerId", "textractJobId", "triggerVersion", "extractionVersion"}).
-				AddRow(textEntryId, params.DocumentID, "buket", "/i/am/here", "example", uuid.New(), uuid.New(), "jobId", int64(123), int64(543)),
+			pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "hash", "cleanId", "extractionVersion"}).
+				AddRow(textEntryId, params.DocumentID, "buket", "/i/am/here", "example", uuid.New(), int64(543)),
 		)
 	pool.ExpectQuery("name: GetQuery :one").WithArgs(query.ID).WillReturnRows(
 		pgxmock.NewRows([]string{"id", "type", "activeVersion", "latestVersion", "config", "requiredIds"}).

internal/server/api/listener_test.go

@@ -26,8 +26,9 @@ func TestNew(t *testing.T) {
 
 	cfg := &BaseConfig{}
 	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{})
-	defer cleanup()
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{})
+	test.SetDBCfg(t, cfg)
 
 	registerHandlers := func() (*openapi3.T, error) {
 		return &openapi3.T{}, nil

internal/server/runner/listener_test.go

@@ -28,13 +28,14 @@ func TestNew(t *testing.T) {
 	ctx := context.Background()
 	cfg := &TestConfig{}
 	test.SetCfgProvider(t, cfg)
+	net := test.DepNetwork.Get(t, ctx)
 
-	_, acleanup := test.CreateAWSContainer(t, ctx, cfg, &test.CreateAWSConfig{})
+	_, acleanup := test.CreateAWSContainer(t, ctx, cfg, net)
 	defer acleanup()
 	err := cfg.SetQueueClient(ctx)
 	require.NoError(t, err)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{})
-	defer cleanup()
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{})
+	test.SetDBCfg(t, cfg)
 
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()

internal/server/server_test.go

@@ -21,9 +21,10 @@ func TestNew(t *testing.T) {
 
 	cfg := &server.BaseConfig{}
 	test.SetCfgProvider(t, cfg)
+	net := test.DepNetwork.Get(t, ctx)
 
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{})
-	defer cleanup()
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{})
+	test.SetDBCfg(t, cfg)
 
 	clean, err := server.New(ctx, cfg)
 	require.NoError(t, err)

internal/serviceconfig/database/config.go

@@ -25,6 +25,7 @@ type DBConfig struct {
 type ConfigProvider interface {
 	GetDBBaseURI() string
 	GetDBURI() string
+	GetDBAdminDBURI() string
 	GetDBOpts() map[string]string
 	GetDBOptsString() string
 	GetDBHost() string
@@ -82,6 +83,10 @@ func (b *DBConfig) GetDBBaseURI() string {
 	return fmt.Sprintf("%s://%s:%s@%s:%d/", b.GetDBDriver(), b.DBUser, b.DBSecret, b.DBHost, b.DBPort)
 }
 
+func (b *DBConfig) GetDBAdminDBURI() string {
+	return fmt.Sprintf("%s%s?%s", b.GetDBBaseURI(), b.GetDBDriver(), b.GetDBOptsString())
+}
+
 func (b *DBConfig) GetDBURI() string {
 	return fmt.Sprintf("%s%s?%s", b.GetDBBaseURI(), b.DBName, b.GetDBOptsString())
 }

internal/serviceconfig/database/config_test.go

@@ -67,6 +67,22 @@ func TestGetDBURI(t *testing.T) {
 	assert.Equal(t, "postgres://user:pass@host:123/name?sslmode=disable", uri)
 }
 
+func TestGetDBAdminDBURI(t *testing.T) {
+	cfg := database.DBConfig{}
+
+	uri := cfg.GetDBURI()
+	assert.Equal(t, "postgres://:@:0/?", uri)
+
+	cfg.DBHost = "host"
+	cfg.DBPort = 123
+	cfg.DBUser = "user"
+	cfg.DBSecret = "pass"
+	cfg.DBName = "name"
+	cfg.DBNoSSL = true
+	uri = cfg.GetDBAdminDBURI()
+	assert.Equal(t, "postgres://user:pass@host:123/postgres?sslmode=disable", uri)
+}
+
 func TestGetDBHost(t *testing.T) {
 	cfg := database.DBConfig{}
 

internal/serviceconfig/database/pool_test.go

@@ -22,10 +22,10 @@ func TestSetDBPool(t *testing.T) {
 
 	cfg := &serviceconfig.BaseConfig{}
 	test.SetCfgProvider(t, cfg)
-	_, cleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{
 		RunMigrations: true,
 	})
-	defer cleanup()
 
 	err := cfg.SetDBPool(ctx)
 	require.NoError(t, err)

internal/serviceconfig/objectstore/bucketkey.go

@@ -24,10 +24,9 @@ type BucketKey struct {
 }
 
 const (
-	Import       BucketLocation = "import"
-	TextTextract BucketLocation = "text/textract"
-	TextOut      BucketLocation = "text/out"
-	Export       BucketLocation = "export"
+	Import BucketLocation = "import"
+	Text   BucketLocation = "text"
+	Export BucketLocation = "export"
 )
 
 var PartExclusions = []BucketLocation{
@@ -133,7 +132,7 @@ func (c *BucketKey) parseClientID(name string) error {
 }
 func (c *BucketKey) parseLocation(name string) error {
 	c.Location = BucketLocation(strings.ReplaceAll(name, "-", "/"))
-	pattern := fmt.Sprintf(`^(%s|%s|%s|%s)$`, Import, TextTextract, TextOut, Export)
+	pattern := fmt.Sprintf(`^(%s|%s|%s)$`, Import, Text, Export)
 	reg := regexp.MustCompile(pattern)
 	matches := reg.FindStringSubmatch(string(c.Location))
 	if len(matches) != 2 {

internal/serviceconfig/objectstore/bucketkey_test.go

@@ -19,20 +19,16 @@ func TestGetBucketKey(t *testing.T) {
 	assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__b745910f-f529-43c5-87e1-25629d46ef40", key.String())
 	key.Location = Import
 	assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
-	key.Location = TextTextract
-	assert.Equal(t, "AAA/text/textract/20250331/0/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
-	key.Location = TextOut
-	assert.Equal(t, "AAA/text/out/20250331/0/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
+	key.Location = Text
+	assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
 	key.Location = Export
 	assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
 	part := uint16(3)
 	key.Part = &part
 	key.Location = Import
 	assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
-	key.Location = TextTextract
-	assert.Equal(t, "AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
-	key.Location = TextOut
-	assert.Equal(t, "AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
+	key.Location = Text
+	assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
 	key.Location = Export
 	assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
 	filetype := "csv"
@@ -67,24 +63,15 @@ func TestParseBucketKey(t *testing.T) {
 		CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
 	}, key)
 
-	key, err = ParseBucketKey("AAA/text/textract/20250331/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40")
+	key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
 	assert.NoError(t, err)
 	assert.EqualExportedValues(t, BucketKey{
 		ClientID:  "AAA",
-		Location:  TextTextract,
+		Location:  Text,
 		EntityID:  uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
 		CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC),
 	}, key)
 
-	key, err = ParseBucketKey("AAA/text/out/20250331/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40")
-	assert.NoError(t, err)
-	assert.EqualExportedValues(t, BucketKey{
-		ClientID:  "AAA",
-		Location:  TextOut,
-		EntityID:  uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
-		CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
-	}, key)
-
 	key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40")
 	assert.NoError(t, err)
 	assert.EqualExportedValues(t, BucketKey{
@@ -105,21 +92,11 @@ func TestParseBucketKey(t *testing.T) {
 		Part:      &part,
 	}, key)
 
-	key, err = ParseBucketKey("AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40")
+	key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
 	assert.NoError(t, err)
 	assert.EqualExportedValues(t, BucketKey{
 		ClientID:  "AAA",
-		Location:  TextTextract,
-		EntityID:  uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
-		CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
-		Part:      &part,
-	}, key)
-
-	key, err = ParseBucketKey("AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40")
-	assert.NoError(t, err)
-	assert.EqualExportedValues(t, BucketKey{
-		ClientID:  "AAA",
-		Location:  TextOut,
+		Location:  Text,
 		EntityID:  uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
 		CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
 		Part:      &part,

internal/serviceconfig/objectstore/config_test.go

@@ -94,7 +94,8 @@ func TestCalculateETag(t *testing.T) {
 
 	cfg := &StoreConfig{}
 	test.SetCfgProvider(t, cfg)
-	acfg, clean := test.CreateAWSContainer(t, ctx, cfg, &test.CreateAWSConfig{})
+	net := test.DepNetwork.Get(t, ctx)
+	acfg, clean := test.CreateAWSContainer(t, ctx, cfg, net)
 	defer clean()
 	test.SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
 	test.CreateBucket(t, ctx, cfg, test.BucketName)

internal/serviceconfig/queue/documenttextprocess/config.go

@@ -1,15 +0,0 @@
-package documenttextprocess
-
-const EnvName = "DOCUMENT_TEXT_PROCESS_URL"
-
-type DocTextProcessConfig struct {
-	DocumentTextProcessURL string `env:"DOCUMENT_TEXT_PROCESS_URL,required,notEmpty"`
-}
-
-func (c *DocTextProcessConfig) GetDocumentTextProcessURL() string {
-	return c.DocumentTextProcessURL
-}
-
-type ConfigProvider interface {
-	GetDocumentTextProcessURL() string
-}

internal/serviceconfig/queue/documenttextprocess/config_test.go

@@ -1,21 +0,0 @@
-package documenttextprocess_test
-
-import (
-	"testing"
-
-	"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestGetDocumentTextURL(t *testing.T) {
-	cfg := documenttextprocess.DocTextProcessConfig{}
-
-	name := cfg.GetDocumentTextProcessURL()
-	assert.Equal(t, "", name)
-
-	cfg.DocumentTextProcessURL = "name"
-	name = cfg.GetDocumentTextProcessURL()
-	assert.Equal(t, "name", name)
-	assert.Equal(t, cfg.DocumentTextProcessURL, name)
-}

internal/test/api.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/docker/go-connections/nat"
 	"github.com/stretchr/testify/require"
-	"github.com/testcontainers/testcontainers-go"
 )
 
 type APIName string
@@ -26,19 +25,17 @@ type API struct {
 
 type APIConfig struct {
 	API      API
-	Network  *testcontainers.DockerNetwork
 	MockHTTP string
 }
 
-func CreateAPI(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, config *APIConfig) (*Container, func()) {
+func CreateAPI(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, network string, config *APIConfig) (*Container, func()) {
 	port, err := nat.NewPort("tcp", "8080")
 	require.NoError(t, err)
 
-	container, cleanup := createContainer(t, ctx, &containerConfig{
+	container, cleanup := createContainer(t, ctx, network, &containerConfig{
 		Cfg:              cfg,
 		Name:             string(config.API.Name),
 		DownstreamQueues: config.API.DownstreamQueues,
-		Network:          config.Network,
 		MockHTTP:         config.MockHTTP,
 		WaitForMsg:       "⇨ http server started on [::]:8080",
 	})

internal/test/api_test.go

@@ -1,7 +1,6 @@
 package test_test
 
 import (
-	"context"
 	"testing"
 
 	"queryorchestration/internal/serviceconfig"
@@ -14,24 +13,18 @@ func TestCreateAPI(t *testing.T) {
 	if testing.Short() {
 		t.Skip("Skipping long test in short mode")
 	}
-	ctx := context.Background()
-
-	ncfg, ncleanup := test.CreateNetwork(t, ctx)
-	defer ncleanup()
+	ctx := t.Context()
 
 	cfg := &serviceconfig.BaseConfig{}
 	test.SetCfgProvider(t, cfg)
-	_, dbcleanup := test.CreateDB(t, ctx, cfg, &test.CreateDatabaseConfig{
-		Network: ncfg,
-	})
-	defer dbcleanup()
+	net := test.DepNetwork.Get(t, ctx)
+	_ = test.CreateDB(t, ctx, cfg, net, &test.CreateDatabaseConfig{})
 
 	acfg := &test.APIConfig{
-		API:     test.QueryAPI,
-		Network: ncfg,
+		API: test.QueryAPI,
 	}
 
-	conn, cleanup := test.CreateAPI(t, ctx, cfg, acfg)
+	conn, cleanup := test.CreateAPI(t, ctx, cfg, net, acfg)
 	assert.NotNil(t, conn)
 	assert.NotNil(t, cleanup)
 

internal/test/assertions.go

@@ -0,0 +1,36 @@
+package test
+
+import (
+	"encoding/json"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/google/go-cmp/cmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func GetTextractFileResponse(t testing.TB, filename string) map[string][]textract.DetectDocumentTextOutput {
+	file, err := os.Open(filename)
+	require.NoError(t, err)
+	defer file.Close()
+
+	decoder := json.NewDecoder(file)
+	var out map[string][]textract.DetectDocumentTextOutput
+	err = decoder.Decode(&out)
+	require.NoError(t, err)
+
+	return out
+}
+
+func AssertStringToReader(t testing.TB, expected string, actual io.Reader) {
+	actualBytes, err := io.ReadAll(actual)
+	require.NoError(t, err)
+	actualStr := string(actualBytes)
+	if !assert.Equal(t, expected, actualStr) {
+		diff := cmp.Diff(expected, actualStr)
+		t.Logf("Detailed diff (-expected +actual):\n%s", diff)
+	}
+}

internal/test/assertions_test.go

@@ -0,0 +1,29 @@
+package test_test
+
+import (
+	"strings"
+	"testing"
+
+	"queryorchestration/internal/test"
+
+	"github.com/aws/aws-sdk-go-v2/service/textract"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestAssertStringToReader(t *testing.T) {
+	t.Run("pass", func(t *testing.T) {
+		mockT := &testing.T{}
+		test.AssertStringToReader(t, "hi", strings.NewReader("hi"))
+		assert.False(t, mockT.Failed())
+	})
+	t.Run("fail", func(t *testing.T) {
+		mockT := &testing.T{}
+		test.AssertStringToReader(mockT, "bye", strings.NewReader("hi"))
+		assert.True(t, mockT.Failed())
+	})
+}
+
+func TestGetTextractFileResponse(t *testing.T) {
+	out := test.GetTextractFileResponse(t, "../../assets/sampleGeneration/generated/helloWorld.gen")
+	assert.IsType(t, map[string][]textract.DetectDocumentTextOutput{}, out)
+}

internal/test/aws.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"testing"
 	"time"
 
@@ -20,11 +21,6 @@ import (
 type AWSContainerConfig struct {
 	Container        testcontainers.Container
 	ExternalEndpoint string
-	NetworkEndpoint  string
-}
-
-type CreateAWSConfig struct {
-	Network *testcontainers.DockerNetwork
 }
 
 type AWSConfigProvider interface {
@@ -32,10 +28,15 @@ type AWSConfigProvider interface {
 	objectstore.ConfigProvider
 }
 
-func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider, acfg *CreateAWSConfig) (*AWSContainerConfig, func()) {
-	alias := "localstack"
+const (
+	awsAlias = "localstack"
+	awsPort  = 4566
+)
 
-	port, err := nat.NewPort("tcp", "4566")
+func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider, network string) (*AWSContainerConfig, func()) {
+	alias := NormaliseAlias(fmt.Sprintf("%s_%s", awsAlias, t.Name()))
+
+	port, err := nat.NewPort("tcp", strconv.FormatInt(int64(awsPort), 10))
 	require.NoError(t, err)
 
 	req := testcontainers.ContainerRequest{
@@ -69,13 +70,10 @@ func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider
 					return statusCode == http.StatusOK
 				}),
 		),
-	}
-
-	if acfg.Network != nil {
-		req.Networks = []string{acfg.Network.Name}
-		req.NetworkAliases = map[string][]string{
-			acfg.Network.Name: {alias},
-		}
+		Networks: []string{network},
+		NetworkAliases: map[string][]string{
+			network: {alias},
+		},
 	}
 
 	container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
@@ -90,25 +88,16 @@ func CreateAWSContainer(t testing.TB, ctx context.Context, cfg AWSConfigProvider
 	require.NoError(t, err)
 
 	extEndpoint := fmt.Sprintf("http://%s:%s", host, mappedPort.Port())
-	t.Setenv("AWS_ENDPOINT_URL", extEndpoint)
 	cfg.SetAWSEndpoint(extEndpoint)
-	t.Setenv("AWS_ENDPOINT_URL_SQS", extEndpoint)
 	cfg.SetSQSEndpoint(extEndpoint)
-	t.Setenv("AWS_ENDPOINT_URL_S3", extEndpoint)
 	cfg.SetS3Endpoint(extEndpoint)
-
-	var endpoint string
-	if acfg.Network != nil {
-		endpoint = fmt.Sprintf("http://%s:%s", alias, port.Port())
-		cfg.SetAWSEndpoint(endpoint)
-		cfg.SetSQSEndpoint(endpoint)
-		cfg.SetS3Endpoint(endpoint)
-	}
+	t.Setenv("AWS_ENDPOINT_URL", extEndpoint)
+	t.Setenv("AWS_ENDPOINT_URL_SQS", extEndpoint)
+	t.Setenv("AWS_ENDPOINT_URL_S3", extEndpoint)
 
 	return &AWSContainerConfig{
 			Container:        container,
 			ExternalEndpoint: extEndpoint,
-			NetworkEndpoint:  endpoint,
 		}, func() {
 			err := container.Terminate(ctx)
 			if err != nil {