Merged in feature/textextraction (pull request #110)

Text Extraction

* bases

* go

* splitting

* structure

* movetoasync

* movetoasync

* settinguptrigger

* reorder

* storevent

* standardisepollingvalidation

* unittests

* fixlint

* fixlint

* awscfg

* generatesample

* followthrough

* tests

* clena

* store

* externalidcleanup

* clientid

* local

* baseunittests

* putobjecttests

* tests

api/docInitRunner/runner_test.go

@@ -1,21 +1,19 @@
-package docinitrunner
+package docinitrunner_test
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"testing"
 
-	"queryorchestration/internal/client"
+	docinitrunner "queryorchestration/api/docInitRunner"
 	"queryorchestration/internal/database/repository"
 	"queryorchestration/internal/document"
 	documentinit "queryorchestration/internal/document/init"
-	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/server/runner"
 	"queryorchestration/internal/serviceconfig/queue/documentsync"
 	queuemock "queryorchestration/mocks/queue"
 
 	"github.com/aws/aws-sdk-go-v2/service/sqs"
-	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/pashagolub/pgxmock/v3"
@@ -25,7 +23,7 @@ import (
 )
 
 type DocInitConfig struct {
-	serviceconfig.BaseConfig
+	runner.BaseConfig[docinitrunner.Body]
 	documentsync.DocSyncConfig
 }
 
@@ -41,41 +39,25 @@ func TestDocInitRunner(t *testing.T) {
 	mockSQS := queuemock.NewMockSQSClient(t)
 	cfg.QueueClient = mockSQS
 
-	runner := New(validator.New(), &Services{
+	runner := docinitrunner.New(validator.New(), &docinitrunner.Services{
 		Document: documentinit.New(cfg),
 	})
 	assert.NotNil(t, runner)
 
 	t.Run("valid", func(t *testing.T) {
-		clientId := uuid.New()
+		clientId := "clientid"
 		bucketName := "bucketName"
-		location := fmt.Sprintf("%s/aaa", clientId)
+		location := fmt.Sprintf("%s/import/aaa", clientId)
 		docinfo := document.DocumentSummary{
 			ID:       uuid.New(),
-			ClientID: clientId,
+			ClientID: "hello",
 			Hash:     "example_hash",
 		}
-		doc := S3EventNotification{
-			Records: []S3EventRecord{
-				{
-					EventName: EventS3ObjectCreatedPut,
-					S3: S3EventRecordDetails{
-						Bucket: S3Bucket{
-							Name: bucketName,
-						},
-						Object: S3Object{
-							Key:  location,
-							ETag: docinfo.Hash,
-						},
-					},
-				},
-			},
-		}
-		bodyBytes, err := json.Marshal(doc)
-		require.NoError(t, err)
-		body := string(bodyBytes)
-		msg := &types.Message{
-			Body: &body,
+		doc := docinitrunner.Body{
+			Bucket:   bucketName,
+			Key:      location,
+			Hash:     docinfo.Hash,
+			ClientID: clientId,
 		}
 
 		pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, clientId).WillReturnRows(
@@ -106,123 +88,6 @@ func TestDocInitRunner(t *testing.T) {
 			).
 			Return(&sqs.SendMessageOutput{}, nil)
 
-		assert.True(t, runner.Process(ctx, msg))
-	})
-	t.Run("invalid body", func(t *testing.T) {
-		body := "definitely invalid"
-		msgId := "id"
-		msg := &types.Message{
-			Body:      &body,
-			MessageId: &msgId,
-		}
-
-		assert.True(t, runner.Process(ctx, msg))
-	})
-}
-
-func TestProcessRecord(t *testing.T) {
-	ctx := context.Background()
-
-	pool, err := pgxmock.NewPool()
-	require.NoError(t, err)
-
-	cfg := &DocInitConfig{}
-	cfg.DBPool = pool
-	cfg.DBQueries = repository.New(pool)
-	mockSQS := queuemock.NewMockSQSClient(t)
-	cfg.QueueClient = mockSQS
-
-	runner := New(validator.New(), &Services{
-		Document: documentinit.New(cfg),
-	})
-	assert.NotNil(t, runner)
-
-	t.Run("valid", func(t *testing.T) {
-		j := client.Client{
-			ID: uuid.New(),
-		}
-		bucketName := "bucketName"
-		location := fmt.Sprintf("%s/aaa", j.ID.String())
-		docinfo := document.DocumentSummary{
-			ID:       uuid.New(),
-			ClientID: j.ID,
-			Hash:     "example_hash",
-		}
-		record := S3EventRecord{
-			EventName: EventS3ObjectCreatedPut,
-			S3: S3EventRecordDetails{
-				Bucket: S3Bucket{
-					Name: bucketName,
-				},
-				Object: S3Object{
-					Key:  location,
-					ETag: docinfo.Hash,
-				},
-			},
-		}
-
-		pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, j.ID).WillReturnRows(
-			pgxmock.NewRows([]string{"id"}),
-		)
-		pool.ExpectBegin()
-		pool.ExpectQuery("name: CreateDocument :one").WithArgs(j.ID, docinfo.Hash).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id"}).
-					AddRow(docinfo.ID),
-			)
-		pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(docinfo.ID, bucketName, location).
-			WillReturnResult(pgxmock.NewResult("", 1))
-		pool.ExpectCommit()
-		pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(docinfo.ID).
-			WillReturnRows(
-				pgxmock.NewRows([]string{"id", "clientId", "hash"}).
-					AddRow(docinfo.ID, docinfo.ClientID, "example"),
-			)
-
-		mockSQS.EXPECT().
-			SendMessage(
-				mock.Anything,
-				mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
-					return *in.QueueUrl == cfg.DocumentSyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", docinfo.ID.String())
-				}),
-				mock.Anything,
-			).
-			Return(&sqs.SendMessageOutput{}, nil)
-
-		err = runner.processRecord(ctx, record)
-		require.NoError(t, err)
-	})
-	t.Run("invalid_event", func(t *testing.T) {
-		record := S3EventRecord{
-			EventName: "invalid_event",
-		}
-
-		err = runner.processRecord(ctx, record)
-		require.NoError(t, err)
-	})
-	t.Run("invalid id", func(t *testing.T) {
-		location := "cc/aaa"
-		record := S3EventRecord{
-			EventName: EventS3ObjectCreatedPut,
-			S3: S3EventRecordDetails{
-				Object: S3Object{
-					Key: location,
-				},
-			},
-		}
-
-		err = runner.processRecord(ctx, record)
-		require.NoError(t, err)
-	})
-}
-
-func TestIsSupportedEvent(t *testing.T) {
-	runner := Runner{}
-
-	t.Run("put", func(t *testing.T) {
-		assert.True(t, runner.isSupportedEvent(EventS3ObjectCreatedPut))
-	})
-	t.Run("invalid", func(t *testing.T) {
-		assert.False(t, runner.isSupportedEvent("invalid"))
+		assert.True(t, runner.Process(ctx, doc))
 	})
 }