Merged in feature/textextraction (pull request #110)

Text Extraction

* bases

* go

* splitting

* structure

* movetoasync

* movetoasync

* settinguptrigger

* reorder

* storevent

* standardisepollingvalidation

* unittests

* fixlint

* fixlint

* awscfg

* generatesample

* followthrough

* tests

* clena

* store

* externalidcleanup

* clientid

* local

* baseunittests

* putobjecttests

* tests

api/docCleanRunner/runner_test.go

@@ -2,7 +2,6 @@ package doccleanrunner_test
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
@@ -12,15 +11,14 @@ import (
 	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/document"
 	documentclean "queryorchestration/internal/document/clean"
-	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/server/runner"
 	"queryorchestration/internal/serviceconfig/objectstore"
-	"queryorchestration/internal/serviceconfig/queue/documenttext"
+	"queryorchestration/internal/serviceconfig/queue/documenttexttrigger"
 	objectstoremock "queryorchestration/mocks/objectstore"
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
@@ -30,8 +28,8 @@ import (
 )
 
 type DocCleanConfig struct {
-	serviceconfig.BaseConfig
-	documenttext.DocTextConfig
+	runner.BaseConfig[doccleanrunner.Body]
+	documenttexttrigger.DocTextTriggerConfig
 	objectstore.ObjectStoreConfig
 }
 
@@ -48,7 +46,7 @@ func TestDocCleanRunner(t *testing.T) {
 	cfg.StoreClient = mockStore
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
-	cfg.DocumentTextURL = "/i/am/here"
+	cfg.DocumentTextTriggerURL = "/i/am/here"
 
 	runner := doccleanrunner.New(validator.New(), &doccleanrunner.Services{
 		Clean: documentclean.New(cfg),
@@ -57,18 +55,12 @@ func TestDocCleanRunner(t *testing.T) {
 
 	t.Run("valid", func(t *testing.T) {
 		bod := doccleanrunner.Body{
-			ID: uuid.New(),
-		}
-		bodyBytes, err := json.Marshal(bod)
-		require.NoError(t, err)
-		body := string(bodyBytes)
-		msg := &types.Message{
-			Body: &body,
+			DocumentID: uuid.New(),
 		}
 
 		doc := document.DocumentSummary{
-			ID:       bod.ID,
-			ClientID: uuid.New(),
+			ID:       bod.DocumentID,
+			ClientID: "hello",
 			Hash:     "example_hash",
 		}
 		inloc := document.Location{
@@ -99,9 +91,9 @@ func TestDocCleanRunner(t *testing.T) {
 		cleanid := uuid.New()
 		pool.ExpectQuery("name: GetMostRecentDocumentCleanEntry :one").WithArgs(dbid).
 			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "mimetype", "fail"}),
+				pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}),
 			)
-		pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, dbmimetype, repository.NullCleanfailtype{}).
+		pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}).
 			WillReturnRows(
 				pgxmock.NewRows([]string{"id"}).
 					AddRow(cleanid),
@@ -114,7 +106,7 @@ func TestDocCleanRunner(t *testing.T) {
 			SendMessage(
 				mock.Anything,
 				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.DocumentTextURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
+					return *in.QueueUrl == cfg.DocumentTextTriggerURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", doc.ID.String())
 				}),
 				mock.Anything,
 			).
@@ -145,17 +137,7 @@ func TestDocCleanRunner(t *testing.T) {
 				Body: bodyMsg,
 			}, nil)
 
-		assert.True(t, runner.Process(ctx, msg))
-	})
-	t.Run("invalid body", func(t *testing.T) {
-		body := "definitely invalid"
-		msgId := "id"
-		msg := &types.Message{
-			Body:      &body,
-			MessageId: &msgId,
-		}
-
-		assert.True(t, runner.Process(ctx, msg))
+		assert.True(t, runner.Process(ctx, bod))
 	})
 }