Merged in feature/textextraction (pull request #110)

Text Extraction * bases * go * splitting * structure * movetoasync * movetoasync * settinguptrigger * reorder * storevent * standardisepollingvalidation * unittests * fixlint * fixlint * awscfg * generatesample * followthrough * tests * clena * store * externalidcleanup * clientid * local * baseunittests * putobjecttests * tests
2025-04-02 18:50:03 +00:00
parent e6a4cb3990
commit 81e7223560
226 changed files with 17283 additions and 2744 deletions
@@ -0,0 +1,116 @@
+package doctextprocessrunner
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	"queryorchestration/internal/database/repository"
+	"queryorchestration/internal/document"
+	documenttext "queryorchestration/internal/document/text"
+	"queryorchestration/internal/server/runner"
+	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/serviceconfig/queue/querysync"
+	"queryorchestration/internal/serviceconfig/textract"
+	objectstoremock "queryorchestration/mocks/objectstore"
+	queuemock "queryorchestration/mocks/queue"
+
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	"github.com/go-playground/validator/v10"
+	"github.com/google/uuid"
+	"github.com/pashagolub/pgxmock/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+type DocInitConfig struct {
+	runner.BaseConfig[Body]
+	aws.AWSConfig
+	textract.TextractConfig
+	querysync.QuerySyncConfig
+	objectstore.ObjectStoreConfig
+}
+
+func TestDocTextProcessRunner(t *testing.T) {
+	ctx := context.Background()
+
+	pool, err := pgxmock.NewPool()
+	require.NoError(t, err)
+
+	cfg := &DocInitConfig{}
+	cfg.DBPool = pool
+	cfg.DBQueries = repository.New(pool)
+	mockSQS := queuemock.NewMockSQSClient(t)
+	cfg.QueueClient = mockSQS
+	mockS3 := objectstoremock.NewMockS3Client(t)
+	cfg.StoreClient = mockS3
+
+	runner := New(validator.New(), &Services{
+		Text: documenttext.New(cfg),
+	})
+	assert.NotNil(t, runner)
+
+	t.Run("valid", func(t *testing.T) {
+		triggerId := uuid.New()
+		docId := uuid.New()
+		textractId := "heeloo"
+		textId := uuid.New()
+		cleanId := uuid.New()
+		bucketName := "bucketName"
+		docinfo := document.DocumentSummary{
+			ID:       uuid.New(),
+			ClientID: "hello",
+			Hash:     `"5d41402abc4b2a76b9719d911017c592"`,
+		}
+		location := fmt.Sprintf("%s/text/textract/%s", docinfo.ClientID, triggerId)
+		doc := Body{
+			Bucket:   bucketName,
+			Key:      location,
+			Hash:     docinfo.Hash,
+			ClientID: docinfo.ClientID,
+		}
+
+		pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).WillReturnRows(
+			pgxmock.NewRows([]string{"id", "cleanId", "version", "textractJobId", "documentId"}).
+				AddRow(triggerId, cleanId, int64(123), &textractId, docId),
+		)
+		pool.ExpectBegin()
+		pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, docinfo.Hash).WillReturnRows(
+			pgxmock.NewRows([]string{"id"}).
+				AddRow(textId),
+		)
+		pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(textId, triggerId, pgxmock.AnyArg()).
+			WillReturnResult(pgxmock.NewResult("", 1))
+		pool.ExpectCommit()
+
+		body := io.NopCloser(strings.NewReader("hello"))
+		mockS3.EXPECT().
+			GetObject(
+				mock.Anything,
+				mock.MatchedBy(func(in *s3.GetObjectInput) bool {
+					return *in.IfMatch == `"5d41402abc4b2a76b9719d911017c592"`
+				}),
+				mock.Anything,
+			).
+			Return(&s3.GetObjectOutput{
+				Body: body,
+			}, nil)
+
+		mockSQS.EXPECT().
+			SendMessage(
+				mock.Anything,
+				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
+					return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docId.String())
+				}),
+				mock.Anything,
+			).
+			Return(&sqs.SendMessageOutput{}, nil)
+
+		assert.True(t, runner.Process(ctx, doc))
+	})
+}