jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/standardisefilepath (pull request #111)

Browse Source 
Feature/standardisefilepath

* baseprocessing

* generalstructure
This commit is contained in:
Michael McGuinness
2025-04-03 12:13:16 +00:00
parent 81e7223560
commit 1d49313a9f
17 changed files with 466 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/store/process.go
+11 -35
View File
@@ -2,13 +2,11 @@ package documentstore
import (
"context"
"fmt"
"log/slog"
"regexp"
docinitrunner "queryorchestration/api/docInitRunner"
doctextprocessrunner "queryorchestration/api/docTextProcessRunner"
"queryorchestration/internal/client"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue"
)
@@ -31,25 +29,29 @@ func (s *Service) Process(ctx context.Context, params Params) error {
return nil
}
task, clientId := getKeyMetadata(params.Key)
key, err := objectstore.ParseBucketKey(params.Key)
if err != nil {
slog.Error("unable to parse key", "key", params.Key)
return nil
}
queueParams := &queue.SendParams{}
switch task {
case DocInit:
switch key.Location {
case objectstore.Import:
queueParams.QueueURL = s.cfg.GetDocInitURL()
queueParams.Body = docinitrunner.Body{
Bucket: params.Bucket,
Key: params.Key,
Hash: params.Hash,
ClientID: clientId,
ClientID: key.ClientID,
}
case DocTextProcess:
case objectstore.TextTextract:
queueParams.QueueURL = s.cfg.GetDocumentTextProcessURL()
queueParams.Body = doctextprocessrunner.Body{
Bucket: params.Bucket,
Key: params.Key,
Hash: params.Hash,
ClientID: clientId,
ClientID: key.ClientID,
}
default:
slog.Info("unsupported key", "key", params.Key)
@@ -59,32 +61,6 @@ func (s *Service) Process(ctx context.Context, params Params) error {
return s.cfg.SendToQueue(ctx, queueParams)
}
type Task string
const (
DocInit Task = "import"
DocTextProcess Task = "text/textract"
Invalid Task = ""
)
func getKeyMetadata(key string) (Task, string) {
regexStr := fmt.Sprintf(`^(%s)/(import|text/textract)/.+$`, client.CLIENT_ID_REGEX)
re := regexp.MustCompile(regexStr)
match := re.FindStringSubmatch(key)
if len(match) < 3 {
return Invalid, ""
}
task := Task(match[2])
if task == Invalid {
return Invalid, ""
}
clientId := match[1]
return task, clientId
}
func (s *Service) isSupportedEvent(name EventS3) bool {
return name == EventS3ObjectCreatedPut
}
internal/document/store/process_test.go
+16 -27
View File
@@ -2,10 +2,11 @@ package documentstore
import (
"context"
"fmt"
"testing"
"time"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentinit"
"queryorchestration/internal/serviceconfig/queue/documenttextprocess"
queuemock "queryorchestration/mocks/queue"
@@ -56,9 +57,15 @@ func TestProcess(t *testing.T) {
).
Return(&sqs.SendMessageOutput{}, nil)
location := objectstore.BucketKey{
ClientID: "7db16095-9155-47d4-8004-b3b3ead93c83",
Location: objectstore.Import,
Filename: "aaa",
CreatedAt: time.Now().UTC(),
}
err := svc.Process(ctx, Params{
Event: EventS3ObjectCreatedPut,
Key: "7db16095-9155-47d4-8004-b3b3ead93c83/import/a",
Key: location.String(),
})
assert.NoError(t, err)
})
@@ -73,9 +80,15 @@ func TestProcess(t *testing.T) {
).
Return(&sqs.SendMessageOutput{}, nil)
location := objectstore.BucketKey{
ClientID: "7db16095-9155-47d4-8004-b3b3ead93c83",
Location: objectstore.TextTextract,
Filename: "aaa",
CreatedAt: time.Now().UTC(),
}
err := svc.Process(ctx, Params{
Event: EventS3ObjectCreatedPut,
Key: "7db16095-9155-47d4-8004-b3b3ead93c83/text/textract/a",
Key: location.String(),
})
assert.NoError(t, err)
})
@@ -91,27 +104,3 @@ func TestIsSupportedEvent(t *testing.T) {
assert.False(t, svc.isSupportedEvent("invalid"))
})
}
func TestGetKeyMetadata(t *testing.T) {
baseClientId := "hello"
task, clientId := getKeyMetadata(fmt.Sprintf("%s/import/a", baseClientId))
assert.Equal(t, DocInit, task)
assert.Equal(t, baseClientId, clientId)
task, clientId = getKeyMetadata(fmt.Sprintf("%s/text/textract/a", baseClientId))
assert.Equal(t, DocTextProcess, task)
assert.Equal(t, baseClientId, clientId)
task, clientId = getKeyMetadata("a")
assert.Equal(t, Invalid, task)
assert.Equal(t, "", clientId)
task, clientId = getKeyMetadata("fdee2e22-ad0f-4e30-91c8-d63c3501162f/something/a")
assert.Equal(t, Invalid, task)
assert.Equal(t, "", clientId)
task, clientId = getKeyMetadata("a/import/a")
assert.Equal(t, DocInit, task)
assert.Equal(t, "a", clientId)
}
internal/document/text/processExtract.go
+14 -10
View File
@@ -5,12 +5,13 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"strings"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/google/uuid"
@@ -18,7 +19,7 @@ import (
type ProcessParams struct {
Bucket string
Key string
Key objectstore.BucketKey
Hash string
ClientID string
}
@@ -44,9 +45,10 @@ func (s *Service) mergePages(pages []*Page) (io.Reader, error) {
}
func (s *Service) processTrigger(ctx context.Context, trigger *repository.GetDocumentTextTriggerRow, params *ProcessParams) error {
keyStr := params.Key.String()
response, err := s.cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{
Bucket: &params.Bucket,
Key: &params.Key,
Key: &keyStr,
IfMatch: &params.Hash,
})
if err != nil {
@@ -71,10 +73,6 @@ func (s *Service) processTrigger(ctx context.Context, trigger *repository.GetDoc
return nil
}
func (s *Service) createTextKey(clientId string, textId uuid.UUID) string {
return fmt.Sprintf("%s/text/out/%s", clientId, textId)
}
func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *repository.GetDocumentTextTriggerRow, params *ProcessParams) error {
version := build.GetVersionUnixTimestamp()
@@ -99,11 +97,17 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
var textId uuid.UUID
if existingId == uuid.Nil || errors.Is(err, sql.ErrNoRows) {
key := s.createTextKey(params.ClientID, trigger.ID)
key := objectstore.BucketKey{
ClientID: params.ClientID,
Location: objectstore.TextOut,
CreatedAt: time.Now().UTC(),
Filename: trigger.ID.String(),
}
keyStr := key.String()
textId, err = q.AddDocumentText(ctx, &repository.AddDocumentTextParams{
Bucket: params.Bucket,
Key: key,
Key: keyStr,
Hash: hash,
})
if err != nil {
@@ -112,7 +116,7 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep
_, err = s.cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: &params.Bucket,
Key: &key,
Key: &keyStr,
Body: storeReader,
})
if err != nil {
internal/document/text/processExtract_test.go
+27 -8
View File
@@ -7,8 +7,10 @@ import (
"io"
"strings"
"testing"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/objectstore"
objectstoremock "queryorchestration/mocks/objectstore"
queuemock "queryorchestration/mocks/queue"
@@ -82,9 +84,14 @@ func TestProcess(t *testing.T) {
).
Return(&sqs.SendMessageOutput{}, nil)
key := objectstore.BucketKey{
ClientID: "my_client",
Location: objectstore.TextTextract,
Filename: triggerId.String(),
}
err = svc.Process(ctx, &ProcessParams{
Bucket: "bucket",
Key: "6ece4a77-702c-4cc9-8654-43c557e21658/text/textract/6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d",
Key: key,
Hash: "hash",
})
assert.NoError(t, err)
@@ -131,16 +138,22 @@ func TestStoreTrigger(t *testing.T) {
Cleanid: cleanId,
}, &ProcessParams{
Bucket: bucket,
Key: objectstore.BucketKey{},
})
require.NoError(t, err)
})
t.Run("not exists", func(t *testing.T) {
key := fmt.Sprintf("%s/text/out/%s", clientId, triggerId)
key := objectstore.BucketKey{
ClientID: clientId,
Location: objectstore.TextOut,
CreatedAt: time.Now().UTC(),
Filename: triggerId.String(),
}
hash := `"09644372e99020106946045c6fd2d70b"`
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
WillReturnError(sql.ErrNoRows)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key, hash).
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
@@ -152,7 +165,7 @@ func TestStoreTrigger(t *testing.T) {
mock.MatchedBy(func(in *s3.PutObjectInput) bool {
bod, err := io.ReadAll(in.Body)
require.NoError(t, err)
return string(bod) == "byebye" && *in.Key == key && *in.Bucket == bucket
return string(bod) == "byebye" && *in.Key == key.String() && *in.Bucket == bucket
}),
mock.Anything,
).
@@ -167,6 +180,7 @@ func TestStoreTrigger(t *testing.T) {
ID: triggerId,
Cleanid: cleanId,
}, &ProcessParams{
Key: key,
Bucket: bucket,
ClientID: clientId,
})
@@ -209,13 +223,18 @@ func TestProcessTrigger(t *testing.T) {
triggerId := uuid.New()
clientId := "hello"
bucket := "bucket"
key := fmt.Sprintf("%s/text/out/%s", clientId, triggerId)
key := objectstore.BucketKey{
ClientID: clientId,
Location: objectstore.TextOut,
Filename: objectstore.BucketFilename(triggerId.String()),
CreatedAt: time.Now().UTC(),
}
hash := `"5d41402abc4b2a76b9719d911017c592"`
pool.ExpectBegin()
pool.ExpectQuery("name: GetDocumentTextExtractionByHash :one").WithArgs(cleanId, hash).
WillReturnError(sql.ErrNoRows)
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key, hash).
pool.ExpectQuery("name: AddDocumentText :one").WithArgs(bucket, key.String(), hash).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(textId),
@@ -226,7 +245,7 @@ func TestProcessTrigger(t *testing.T) {
GetObject(
mock.Anything,
mock.MatchedBy(func(in *s3.GetObjectInput) bool {
return *in.Key == key && *in.Bucket == bucket && *in.IfMatch == hash
return *in.Key == key.String() && *in.Bucket == bucket && *in.IfMatch == hash
}),
mock.Anything,
).
@@ -240,7 +259,7 @@ func TestProcessTrigger(t *testing.T) {
mock.MatchedBy(func(in *s3.PutObjectInput) bool {
bod, err := io.ReadAll(in.Body)
require.NoError(t, err)
return string(bod) == "hello" && *in.Key == key && *in.Bucket == bucket
return string(bod) == "hello" && *in.Key == key.String() && *in.Bucket == bucket
}),
mock.Anything,
).
internal/document/text/triggerExtract.go
+17 -19
View File
@@ -3,12 +3,11 @@ package documenttext
import (
"context"
"errors"
"fmt"
"regexp"
"time"
"queryorchestration/internal/client"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/serviceconfig/build"
"queryorchestration/internal/serviceconfig/objectstore"
"github.com/aws/aws-sdk-go-v2/service/textract"
"github.com/aws/aws-sdk-go-v2/service/textract/types"
@@ -29,11 +28,19 @@ func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentc
jobTag := triggerId.String()
key, err := s.getOutputLocation(clean.Clientid, triggerId)
filename, err := s.getOutputFilename(triggerId)
if err != nil {
return err
}
key := objectstore.BucketKey{
CreatedAt: time.Now().UTC(),
Location: objectstore.TextTextract,
ClientID: clean.Clientid,
Filename: filename,
}
keyStr := key.String()
res, err := s.cfg.GetTextractClient().StartDocumentTextDetection(ctx, &textract.StartDocumentTextDetectionInput{
DocumentLocation: &types.DocumentLocation{
S3Object: &types.S3Object{
@@ -44,7 +51,7 @@ func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentc
JobTag: &jobTag,
OutputConfig: &types.OutputConfig{
S3Bucket: clean.Bucket,
S3Prefix: &key,
S3Prefix: &keyStr,
},
})
if err != nil {
@@ -65,25 +72,16 @@ func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentc
})
}
func (s *Service) getOutputLocation(clientId string, triggerId uuid.UUID) (string, error) {
if clientId == "" {
return "", errors.New("client id required")
} else if triggerId == uuid.Nil {
func (s *Service) getOutputFilename(triggerId uuid.UUID) (string, error) {
if triggerId == uuid.Nil {
return "", errors.New("trigger id required")
}
return fmt.Sprintf("%s/text/textract/%s", clientId, triggerId), nil
return triggerId.String(), nil
}
func (s *Service) getTriggerIdFromKey(key string) (uuid.UUID, error) {
regexPattern := fmt.Sprintf(`^%s/text/textract/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$`, client.CLIENT_ID_REGEX)
re := regexp.MustCompile(regexPattern)
match := re.FindStringSubmatch(key)
if len(match) < 2 {
return uuid.Nil, errors.New("Trigger Id not found")
}
triggerId, err := uuid.Parse(match[1])
func (s *Service) getTriggerIdFromKey(key objectstore.BucketKey) (uuid.UUID, error) {
triggerId, err := uuid.Parse(key.Filename)
if err != nil {
return uuid.Nil, err
}
internal/document/text/triggerExtract_test.go
+14 -22
View File
@@ -2,11 +2,11 @@ package documenttext
import (
"context"
"fmt"
"testing"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig/objectstore"
objectstoremock "queryorchestration/mocks/objectstore"
textractmock "queryorchestration/mocks/textract"
@@ -82,40 +82,32 @@ func TestTriggerExtract(t *testing.T) {
})
}
func TestGetOutputLocation(t *testing.T) {
func TestGetOutputFilename(t *testing.T) {
svc := Service{}
_, err := svc.getOutputLocation("", uuid.Nil)
assert.EqualError(t, err, "client id required")
_, err = svc.getOutputLocation("a", uuid.Nil)
_, err := svc.getOutputFilename(uuid.Nil)
assert.EqualError(t, err, "trigger id required")
triggerId := uuid.New()
key, err := svc.getOutputLocation("a", triggerId)
key, err := svc.getOutputFilename(triggerId)
assert.NoError(t, err)
assert.Equal(t, fmt.Sprintf("a/text/textract/%s", triggerId), key)
assert.Equal(t, triggerId.String(), key)
}
func TestGetTriggerIdFromKey(t *testing.T) {
svc := Service{}
_, err := svc.getTriggerIdFromKey("")
assert.EqualError(t, err, "Trigger Id not found")
_, err := svc.getTriggerIdFromKey(objectstore.BucketKey{})
assert.EqualError(t, err, "invalid UUID length: 0")
_, err = svc.getTriggerIdFromKey("aa")
assert.EqualError(t, err, "Trigger Id not found")
_, err = svc.getTriggerIdFromKey(objectstore.BucketKey{
Filename: "aa",
})
assert.EqualError(t, err, "invalid UUID length: 2")
_, err = svc.getTriggerIdFromKey("6ece4a77-702c-4cc9-8654-43c557e21658")
assert.EqualError(t, err, "Trigger Id not found")
_, err = svc.getTriggerIdFromKey("6ece4a77-702c-4cc9-8654-43c557e21658/text/textract/")
assert.EqualError(t, err, "Trigger Id not found")
_, err = svc.getTriggerIdFromKey("6ece4a77-702c-4cc9-8654-43c557e21658/text/textract/a")
assert.EqualError(t, err, "Trigger Id not found")
triggerId, err := svc.getTriggerIdFromKey("6ece4a77-702c-4cc9-8654-43c557e21658/text/textract/6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d")
triggerId, err := svc.getTriggerIdFromKey(objectstore.BucketKey{
Filename: "6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d",
})
assert.NoError(t, err)
assert.Equal(t, "6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d", triggerId.String())
}