jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/textextraction (pull request #110)

Browse Source 
Text Extraction

* bases

* go

* splitting

* structure

* movetoasync

* movetoasync

* settinguptrigger

* reorder

* storevent

* standardisepollingvalidation

* unittests

* fixlint

* fixlint

* awscfg

* generatesample

* followthrough

* tests

* clena

* store

* externalidcleanup

* clientid

* local

* baseunittests

* putobjecttests

* tests
This commit is contained in:
Michael McGuinness
2025-04-02 18:50:03 +00:00
parent e6a4cb3990
commit 81e7223560
226 changed files with 17283 additions and 2744 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docTextRunner/runner_test.go
+35 -48
View File
@@ -2,21 +2,22 @@ package doctextrunner_test
import (
"context"
"encoding/json"
"fmt"
"testing"
doctextrunner "queryorchestration/api/docTextRunner"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/querysync"
"queryorchestration/internal/serviceconfig/textract"
objectstoremock "queryorchestration/mocks/objectstore"
queuemock "queryorchestration/mocks/queue"
textractmock "queryorchestration/mocks/textract"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
awstextract "github.com/aws/aws-sdk-go-v2/service/textract"
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
@@ -26,9 +27,11 @@ import (
)
type DocTextConfig struct {
serviceconfig.BaseConfig
runner.BaseConfig[doctextrunner.Body]
aws.AWSConfig
querysync.QuerySyncConfig
textract.TextractConfig
objectstore.ObjectStoreConfig
}
func TestDocCleanRunner(t *testing.T) {
@@ -43,6 +46,10 @@ func TestDocCleanRunner(t *testing.T) {
mockSQS := queuemock.NewMockSQSClient(t)
cfg.QueueClient = mockSQS
cfg.QuerySyncURL = "/i/am/here"
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
mockTextract := textractmock.NewMockTextractClient(t)
cfg.TextractClient = mockTextract
runner := doctextrunner.New(validator.New(), &doctextrunner.Services{
Text: documenttext.New(cfg),
@@ -51,72 +58,52 @@ func TestDocCleanRunner(t *testing.T) {
t.Run("valid", func(t *testing.T) {
doc := doctextrunner.Body{
ID: uuid.New(),
}
bodyBytes, err := json.Marshal(doc)
require.NoError(t, err)
body := string(bodyBytes)
msg := &types.Message{
Body: &body,
DocumentID: uuid.New(),
}
cleanId := uuid.New()
inloc := document.Location{
Bucket: "bucket_name",
Key: "/i/am/here",
}
hash := "example"
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.ID).WillReturnRows(
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
pgxmock.NewRows([]string{"isextracted"}).
AddRow(false),
)
cleanId := uuid.New()
mimeType := "application/pdf"
dbmimetype := repository.NullCleanmimetype{
Valid: true,
Cleanmimetype: repository.Cleanmimetype(mimeType),
}
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.ID).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, doc.ID, &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
AddRow(cleanId, doc.DocumentID, &inloc.Bucket, &inloc.Key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, "clientId"),
)
pool.ExpectQuery("name: GetCleanEntry :one").WithArgs(cleanId).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}).
AddRow(cleanId, doc.ID, &inloc.Bucket, &inloc.Key, int64(1), dbmimetype, repository.NullCleanfailtype{}),
)
textId := uuid.New()
jobId := "hello"
triggerId := uuid.New()
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, "example").WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "cleanEntryId"}),
pool.ExpectQuery("name: AddDocumentTextTrigger :one").WithArgs(cleanId, pgxmock.AnyArg()).WillReturnRows(
pgxmock.NewRows([]string{"id"}).AddRow(triggerId),
)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(inloc.Bucket, inloc.Key, cleanId, "example").
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, pgxmock.AnyArg()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
mockSQS.EXPECT().
SendMessage(
mockTextract.EXPECT().
StartDocumentTextDetection(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
mock.MatchedBy(func(in *awstextract.StartDocumentTextDetectionInput) bool {
return true
}),
mock.Anything,
).
Return(&sqs.SendMessageOutput{}, nil)
Return(&awstextract.StartDocumentTextDetectionOutput{
JobId: &jobId,
}, nil)
assert.True(t, runner.Process(ctx, msg))
})
t.Run("invalid body", func(t *testing.T) {
body := "definitely invalid"
msgId := "id"
msg := &types.Message{
Body: &body,
MessageId: &msgId,
}
pool.ExpectExec("name: AddDocumentTextTriggerJobId :exec").WithArgs(&jobId, triggerId).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
assert.True(t, runner.Process(ctx, msg))
assert.True(t, runner.Process(ctx, doc))
})
}