jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/standardisefilepath (pull request #111)

Browse Source 
Feature/standardisefilepath

* baseprocessing

* generalstructure
This commit is contained in:
Michael McGuinness
2025-04-03 12:13:16 +00:00
parent 81e7223560
commit 1d49313a9f
17 changed files with 466 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docInitRunner/runner_test.go
+8 -3
View File
@@ -10,6 +10,7 @@ import (
"queryorchestration/internal/document"
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentsync"
queuemock "queryorchestration/mocks/queue"
@@ -47,7 +48,11 @@ func TestDocInitRunner(t *testing.T) {
t.Run("valid", func(t *testing.T) {
clientId := "clientid"
bucketName := "bucketName"
location := fmt.Sprintf("%s/import/aaa", clientId)
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientId,
Filename: "aaa",
}
docinfo := document.DocumentSummary{
ID: uuid.New(),
ClientID: "hello",
@@ -55,7 +60,7 @@ func TestDocInitRunner(t *testing.T) {
}
doc := docinitrunner.Body{
Bucket: bucketName,
Key: location,
Key: location.String(),
Hash: docinfo.Hash,
ClientID: clientId,
}
@@ -69,7 +74,7 @@ func TestDocInitRunner(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(docinfo.ID),
)
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(docinfo.ID, bucketName, location).
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(docinfo.ID, bucketName, location.String()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(docinfo.ID).
api/docTextProcessRunner/runner.go
+9 -2
View File
@@ -5,6 +5,7 @@ import (
"log/slog"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/serviceconfig/objectstore"
"github.com/go-playground/validator/v10"
)
@@ -35,9 +36,15 @@ type Body struct {
}
func (s Runner) Process(ctx context.Context, body Body) bool {
err := s.svc.Text.Process(ctx, &documenttext.ProcessParams{
key, err := objectstore.ParseBucketKey(body.Key)
if err != nil {
slog.Error("unable to parse key", "key", body.Key)
return true
}
err = s.svc.Text.Process(ctx, &documenttext.ProcessParams{
Bucket: body.Bucket,
Key: body.Key,
Key: key,
Hash: body.Hash,
ClientID: body.ClientID,
})
api/docTextProcessRunner/runner_test.go
+44 -3
View File
@@ -2,10 +2,12 @@ package doctextprocessrunner
import (
"context"
"errors"
"fmt"
"io"
"strings"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
@@ -55,7 +57,7 @@ func TestDocTextProcessRunner(t *testing.T) {
})
assert.NotNil(t, runner)
t.Run("valid", func(t *testing.T) {
t.Run("pass through", func(t *testing.T) {
triggerId := uuid.New()
docId := uuid.New()
textractId := "heeloo"
@@ -67,10 +69,15 @@ func TestDocTextProcessRunner(t *testing.T) {
ClientID: "hello",
Hash: `"5d41402abc4b2a76b9719d911017c592"`,
}
location := fmt.Sprintf("%s/text/textract/%s", docinfo.ClientID, triggerId)
key := objectstore.BucketKey{
ClientID: docinfo.ClientID,
Location: objectstore.TextTextract,
Filename: triggerId.String(),
CreatedAt: time.Now().UTC(),
}
doc := Body{
Bucket: bucketName,
Key: location,
Key: key.String(),
Hash: docinfo.Hash,
ClientID: docinfo.ClientID,
}
@@ -113,4 +120,38 @@ func TestDocTextProcessRunner(t *testing.T) {
assert.True(t, runner.Process(ctx, doc))
})
t.Run("invalid key", func(t *testing.T) {
key := objectstore.BucketKey{}
doc := Body{
Key: key.String(),
}
assert.True(t, runner.Process(ctx, doc))
})
t.Run("unable to process", func(t *testing.T) {
triggerId := uuid.New()
bucketName := "bucketName"
docinfo := document.DocumentSummary{
ID: uuid.New(),
ClientID: "hello",
Hash: `"5d41402abc4b2a76b9719d911017c592"`,
}
key := objectstore.BucketKey{
ClientID: docinfo.ClientID,
Location: objectstore.TextTextract,
Filename: triggerId.String(),
CreatedAt: time.Now().UTC(),
}
doc := Body{
Bucket: bucketName,
Key: key.String(),
Hash: docinfo.Hash,
ClientID: docinfo.ClientID,
}
pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).
WillReturnError(errors.New("db fail"))
assert.False(t, runner.Process(ctx, doc))
})
}
api/storeEventRunner/runner_test.go
+10 -4
View File
@@ -2,13 +2,14 @@ package storeeventrunner
import (
"context"
"fmt"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documentstore "queryorchestration/internal/document/store"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentinit"
"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
queuemock "queryorchestration/mocks/queue"
@@ -50,7 +51,12 @@ func TestDocInitRunner(t *testing.T) {
t.Run("valid", func(t *testing.T) {
clientId := "hi"
bucketName := "bucketName"
location := fmt.Sprintf("%s/import/aaa", clientId)
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientId,
Filename: "aaa",
CreatedAt: time.Now().UTC(),
}
docinfo := document.DocumentSummary{
ID: uuid.New(),
ClientID: clientId,
@@ -65,7 +71,7 @@ func TestDocInitRunner(t *testing.T) {
Name: bucketName,
},
Object: S3Object{
Key: location,
Key: location.String(),
ETag: docinfo.Hash,
},
},
@@ -82,7 +88,7 @@ func TestDocInitRunner(t *testing.T) {
pgxmock.NewRows([]string{"id"}).
AddRow(docinfo.ID),
)
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(docinfo.ID, bucketName, location).
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(docinfo.ID, bucketName, location.String()).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(docinfo.ID).