jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/richname (pull request #112)

Browse Source 
Standardised rich name for Files

* tests
This commit is contained in:
Michael McGuinness
2025-04-03 19:17:24 +00:00
parent 1d49313a9f
commit 60812aba91
31 changed files with 744 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/docCleanRunner/runner_test.go
+12 -7
View File
@@ -6,6 +6,7 @@ import (
"io"
"strings"
"testing"
"time"
doccleanrunner "queryorchestration/api/docCleanRunner"
"queryorchestration/internal/database/repository"
@@ -63,10 +64,14 @@ func TestDocCleanRunner(t *testing.T) {
ClientID: "hello",
Hash: "example_hash",
}
inloc := document.Location{
Bucket: "bucket_name",
Key: "/i/am/here",
bucket := "hello"
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
ClientID: "hi",
EntityID: uuid.New(),
Location: objectstore.Import,
}
keyStr := key.String()
dbid := doc.ID
pool.ExpectQuery("name: HasDocumentCleanEntry :one").WithArgs(dbid).WillReturnRows(
@@ -80,7 +85,7 @@ func TestDocCleanRunner(t *testing.T) {
)
pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
AddRow(dbid, inloc.Bucket, inloc.Key),
AddRow(dbid, bucket, keyStr),
)
mimeType := "application/pdf"
dbmimetype := repository.NullCleanmimetype{
@@ -93,7 +98,7 @@ func TestDocCleanRunner(t *testing.T) {
WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}),
)
pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}).
pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(cleanid),
@@ -116,7 +121,7 @@ func TestDocCleanRunner(t *testing.T) {
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key
return *in.Bucket == bucket && *in.Key == keyStr
}),
mock.Anything,
).
@@ -129,7 +134,7 @@ func TestDocCleanRunner(t *testing.T) {
GetObject(
mock.Anything,
mock.MatchedBy(func(in *s3.GetObjectInput) bool {
return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key
return *in.Bucket == bucket && *in.Key == keyStr
}),
mock.Anything,
).
api/docInitRunner/runner.go
+13 -11
View File
@@ -4,8 +4,8 @@ import (
"context"
"log/slog"
"queryorchestration/internal/document"
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/serviceconfig/objectstore"
"github.com/go-playground/validator/v10"
)
@@ -29,20 +29,22 @@ func New(validator *validator.Validate, svc *Services) Runner {
}
type Body struct {
Bucket string `json:"bucket" validate:"required"`
Key string `json:"key" validate:"required"`
Hash string `json:"hash" validate:"required"`
ClientID string `json:"clientId" validate:"required"`
Bucket string `json:"bucket" validate:"required"`
Key string `json:"key" validate:"required"`
Hash string `json:"hash" validate:"required"`
}
func (s Runner) Process(ctx context.Context, body Body) bool {
key, err := objectstore.ParseBucketKey(body.Key)
if err != nil {
slog.Error("unable to parse key", "key", body.Key)
return false
}
id, err := s.svc.Document.Create(ctx, &documentinit.Create{
ClientID: body.ClientID,
Location: document.Location{
Bucket: body.Bucket,
Key: body.Key,
},
Hash: body.Hash,
Key: key,
Bucket: body.Bucket,
Hash: body.Hash,
})
if err != nil {
return false
api/docInitRunner/runner_test.go
+7 -7
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"testing"
"time"
docinitrunner "queryorchestration/api/docInitRunner"
"queryorchestration/internal/database/repository"
@@ -49,9 +50,9 @@ func TestDocInitRunner(t *testing.T) {
clientId := "clientid"
bucketName := "bucketName"
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientId,
Filename: "aaa",
Location: objectstore.Import,
ClientID: clientId,
CreatedAt: time.Now().UTC(),
}
docinfo := document.DocumentSummary{
ID: uuid.New(),
@@ -59,10 +60,9 @@ func TestDocInitRunner(t *testing.T) {
Hash: "example_hash",
}
doc := docinitrunner.Body{
Bucket: bucketName,
Key: location.String(),
Hash: docinfo.Hash,
ClientID: clientId,
Bucket: bucketName,
Key: location.String(),
Hash: docinfo.Hash,
}
pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, clientId).WillReturnRows(
api/docTextProcessRunner/runner.go
+6 -8
View File
@@ -29,10 +29,9 @@ func New(validator *validator.Validate, svc *Services) Runner {
}
type Body struct {
Bucket string `json:"bucket" validate:"required"`
Key string `json:"key" validate:"required"`
Hash string `json:"hash" validate:"required"`
ClientID string `json:"clientId" validate:"required"`
Bucket string `json:"bucket" validate:"required"`
Key string `json:"key" validate:"required"`
Hash string `json:"hash" validate:"required"`
}
func (s Runner) Process(ctx context.Context, body Body) bool {
@@ -43,10 +42,9 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
}
err = s.svc.Text.Process(ctx, &documenttext.ProcessParams{
Bucket: body.Bucket,
Key: key,
Hash: body.Hash,
ClientID: body.ClientID,
Bucket: body.Bucket,
Key: key,
Hash: body.Hash,
})
if err != nil {
slog.Error("unable to process", "bucket", body.Bucket, "key", body.Key, "hash", body.Hash, "error", err)
api/docTextProcessRunner/runner_test.go
+8 -10
View File
@@ -72,14 +72,13 @@ func TestDocTextProcessRunner(t *testing.T) {
key := objectstore.BucketKey{
ClientID: docinfo.ClientID,
Location: objectstore.TextTextract,
Filename: triggerId.String(),
EntityID: triggerId,
CreatedAt: time.Now().UTC(),
}
doc := Body{
Bucket: bucketName,
Key: key.String(),
Hash: docinfo.Hash,
ClientID: docinfo.ClientID,
Bucket: bucketName,
Key: key.String(),
Hash: docinfo.Hash,
}
pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).WillReturnRows(
@@ -139,14 +138,13 @@ func TestDocTextProcessRunner(t *testing.T) {
key := objectstore.BucketKey{
ClientID: docinfo.ClientID,
Location: objectstore.TextTextract,
Filename: triggerId.String(),
EntityID: triggerId,
CreatedAt: time.Now().UTC(),
}
doc := Body{
Bucket: bucketName,
Key: key.String(),
Hash: docinfo.Hash,
ClientID: docinfo.ClientID,
Bucket: bucketName,
Key: key.String(),
Hash: docinfo.Hash,
}
pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).
api/docTextRunner/runner_test.go
+9 -5
View File
@@ -3,10 +3,10 @@ package doctextrunner_test
import (
"context"
"testing"
"time"
doctextrunner "queryorchestration/api/docTextRunner"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/aws"
@@ -62,10 +62,14 @@ func TestDocCleanRunner(t *testing.T) {
}
cleanId := uuid.New()
inloc := document.Location{
Bucket: "bucket_name",
Key: "/i/am/here",
bucket := "hello"
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
ClientID: "hi",
EntityID: uuid.New(),
Location: objectstore.Import,
}
keyStr := key.String()
hash := "example"
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows(
@@ -79,7 +83,7 @@ func TestDocCleanRunner(t *testing.T) {
}
pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).WillReturnRows(
pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}).
AddRow(cleanId, doc.DocumentID, &inloc.Bucket, &inloc.Key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, "clientId"),
AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, "clientId"),
)
jobId := "hello"
triggerId := uuid.New()
api/storeEventRunner/runner_test.go
-1
View File
@@ -54,7 +54,6 @@ func TestDocInitRunner(t *testing.T) {
location := objectstore.BucketKey{
Location: objectstore.Import,
ClientID: clientId,
Filename: "aaa",
CreatedAt: time.Now().UTC(),
}
docinfo := document.DocumentSummary{