jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/standardisefilepath (pull request #111)

Browse Source 
Feature/standardisefilepath

* baseprocessing

* generalstructure
This commit is contained in:
Michael McGuinness
2025-04-03 12:13:16 +00:00
parent 81e7223560
commit 1d49313a9f
17 changed files with 466 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/ecosystem.go
+12 -15
View File
@@ -10,7 +10,6 @@ import (
"io"
"log/slog"
"net/http"
"regexp"
"testing"
"time"
@@ -279,10 +278,9 @@ func CreateMockExpectation(t testing.TB, server MockServer, expectation MockExpe
}
type StartDocTextDetectionExpectationParams struct {
Bucket string
ClientID string
Filename string
JobID string
Bucket string
JobID string
Key objectstore.BucketKey
}
func CreateStartDocTextDetectionExpectation(t testing.TB, mockServer MockServer, params StartDocTextDetectionExpectationParams) MockExpectation {
@@ -297,7 +295,7 @@ func CreateStartDocTextDetectionExpectation(t testing.TB, mockServer MockServer,
"DocumentLocation": {
"S3Object": map[string]any{
"Bucket": params.Bucket,
"Name": fmt.Sprintf("%s/import/%s", params.ClientID, params.Filename),
"Name": params.Key.String(),
},
},
"OutputConfig": {
@@ -434,17 +432,16 @@ func WaitForMockTextractOutput(t testing.TB, ctx context.Context, cfg objectstor
bucket, ok := outputConfig["S3Bucket"].(string)
require.True(t, ok)
key, ok := outputConfig["S3Prefix"].(string)
keyStr, ok := outputConfig["S3Prefix"].(string)
require.True(t, ok)
matches := regexp.MustCompile(`^(.+)/text/textract/(.+)$`).
FindStringSubmatch(key)
require.Len(t, matches, 3)
key, err := objectstore.ParseBucketKey(keyStr)
require.NoError(t, err)
require.Equal(t, objectstore.TextTextract, key.Location)
PutTextractObject(t, ctx, cfg, PutObjectParams{
ClientId: matches[1],
Bucket: bucket,
Filepath: matches[2],
File: params.File,
PutObject(t, ctx, cfg, PutObjectParams{
Key: key,
Bucket: bucket,
File: params.File,
})
}
internal/test/ecosystem_test.go
+19 -5
View File
@@ -8,6 +8,7 @@ import (
"net/http/httptest"
"strings"
"testing"
"time"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
@@ -174,14 +175,27 @@ func TestWaitForTextractMockEndpoint(t *testing.T) {
deps, cleanup := CreateFullDependencies(t, ctx, cfg)
defer cleanup()
clientId := "CLIENTID"
importKey := objectstore.BucketKey{
ClientID: clientId,
CreatedAt: time.Now().UTC(),
Location: objectstore.Import,
Filename: "input",
}
outputKey := objectstore.BucketKey{
ClientID: clientId,
CreatedAt: time.Now().UTC(),
Location: objectstore.TextTextract,
Filename: "output",
}
e := CreateStartDocTextDetectionExpectation(t, deps.MockServer, StartDocTextDetectionExpectationParams{
Bucket: deps.BucketName,
ClientID: "hi",
Filename: "hi",
JobID: "hi",
Bucket: deps.BucketName,
Key: importKey,
JobID: "hi",
})
body := strings.NewReader(fmt.Sprintf(`{"DocumentLocation":{"S3Object":{"Bucket":"%s","Name":"hi/import/hi"}},"OutputConfig":{"S3Bucket":"%s","S3Prefix":"hi/text/textract/hi"}}`, deps.BucketName, deps.BucketName))
body := strings.NewReader(fmt.Sprintf(`{"DocumentLocation":{"S3Object":{"Bucket":"%s","Name":"%s"}},"OutputConfig":{"S3Bucket":"%s","S3Prefix":"%s"}}`, deps.BucketName, importKey.String(), deps.BucketName, outputKey.String()))
req, err := http.NewRequest("POST", string(deps.MockServer.External), body)
require.NoError(t, err)
req.Header.Add("X-Amz-Target", "Textract.StartDocumentTextDetection")
internal/test/objectstore.go
+6 -18
View File
@@ -2,7 +2,6 @@ package test
import (
"context"
"fmt"
"io"
"log/slog"
"testing"
@@ -55,29 +54,18 @@ func SetStoreClient(t testing.TB, ctx context.Context, cfg objectstore.ConfigPro
}
type PutObjectParams struct {
ClientId string
Bucket string
Filepath string
File io.Reader
Key objectstore.BucketKey
Bucket string
File io.Reader
}
func PutObject(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, params PutObjectParams) {
location := fmt.Sprintf("%s/%s", params.ClientId, params.Filepath)
key := params.Key.String()
_, err := cfg.GetStoreClient().PutObject(ctx, &s3.PutObjectInput{
Bucket: &params.Bucket,
Key: &location,
Key: &key,
Body: params.File,
})
require.NoError(t, err)
slog.Info("put object", "bucket", params.Bucket, "key", location)
}
func PutImportObject(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, params PutObjectParams) {
params.Filepath = fmt.Sprintf("import/%s", params.Filepath)
PutObject(t, ctx, cfg, params)
}
func PutTextractObject(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, params PutObjectParams) {
params.Filepath = fmt.Sprintf("text/textract/%s", params.Filepath)
PutObject(t, ctx, cfg, params)
slog.Info("put object", "bucket", params.Bucket, "key", key)
}
internal/test/objectstore_test.go
+8 -73
View File
@@ -2,7 +2,6 @@ package test
import (
"context"
"fmt"
"strings"
"testing"
@@ -63,90 +62,26 @@ func TestPutObject(t *testing.T) {
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
clientId := "byebye"
bucketName := "bucket"
file := strings.NewReader("hello")
filename := "hello"
expectedLocation := fmt.Sprintf("%s/%s", clientId, filename)
key := objectstore.BucketKey{
ClientID: "byebye",
Filename: "hello",
}
mockS3.EXPECT().
PutObject(
mock.Anything,
mock.MatchedBy(func(in *s3.PutObjectInput) bool {
return *in.Bucket == bucketName && *in.Key == expectedLocation
return *in.Bucket == bucketName && *in.Key == key.String()
}),
mock.Anything,
).
Return(&s3.PutObjectOutput{}, nil)
PutObject(t, ctx, cfg, PutObjectParams{
ClientId: clientId,
Bucket: bucketName,
Filepath: filename,
File: file,
})
}
func TestPutImportObject(t *testing.T) {
ctx := context.Background()
cfg := &StoreConfig{}
SetCfgProvider(t, cfg)
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
clientId := "byebye"
bucketName := "bucket"
file := strings.NewReader("hello")
filename := "hello"
expectedLocation := fmt.Sprintf("%s/import/%s", clientId, filename)
mockS3.EXPECT().
PutObject(
mock.Anything,
mock.MatchedBy(func(in *s3.PutObjectInput) bool {
return *in.Bucket == bucketName && *in.Key == expectedLocation
}),
mock.Anything,
).
Return(&s3.PutObjectOutput{}, nil)
PutImportObject(t, ctx, cfg, PutObjectParams{
ClientId: clientId,
Bucket: bucketName,
Filepath: filename,
File: file,
})
}
func TestPutTextractObject(t *testing.T) {
ctx := context.Background()
cfg := &StoreConfig{}
SetCfgProvider(t, cfg)
mockS3 := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockS3
clientId := "byebye"
bucketName := "bucket"
file := strings.NewReader("hello")
filename := "hello"
expectedLocation := fmt.Sprintf("%s/text/textract/%s", clientId, filename)
mockS3.EXPECT().
PutObject(
mock.Anything,
mock.MatchedBy(func(in *s3.PutObjectInput) bool {
return *in.Bucket == bucketName && *in.Key == expectedLocation
}),
mock.Anything,
).
Return(&s3.PutObjectOutput{}, nil)
PutTextractObject(t, ctx, cfg, PutObjectParams{
ClientId: clientId,
Bucket: bucketName,
Filepath: filename,
File: file,
Bucket: bucketName,
File: file,
Key: key,
})
}