jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/standardisefilepath (pull request #111)

Browse Source 
Feature/standardisefilepath

* baseprocessing

* generalstructure
This commit is contained in:
Michael McGuinness
2025-04-03 12:13:16 +00:00
parent 81e7223560
commit 1d49313a9f
17 changed files with 466 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/serviceconfig/objectstore/config.go
+76
View File
@@ -4,11 +4,15 @@ import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"regexp"
"strconv"
"time"
"queryorchestration/internal/client"
awsc "queryorchestration/internal/serviceconfig/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -140,3 +144,75 @@ func (c *ObjectStoreConfig) CalculateETag(ctx context.Context, doc io.Reader) (s
return fmt.Sprintf(`"%s-%d"`, hex.EncodeToString(finalHash.Sum(nil)), len(partHashes)), nil
}
type BucketLocation string
type BucketFilename = string
type BucketKey struct {
ClientID string
Location BucketLocation
CreatedAt time.Time
Part *uint8
Filename BucketFilename
}
const (
Import BucketLocation = "import"
TextTextract BucketLocation = "text/textract"
TextOut BucketLocation = "text/out"
Export BucketLocation = "export"
)
func (c *BucketKey) String() string {
dateStr := c.CreatedAt.Format("20060102")
partStr := ""
if c.Part != nil {
partStr = fmt.Sprintf("/%d", *c.Part)
}
return fmt.Sprintf("%s/%s/%s%s/%s", c.ClientID, c.Location, dateStr, partStr, c.Filename)
}
func ParseBucketKey(key string) (BucketKey, error) {
locations := fmt.Sprintf("%s|%s|%s|%s", Import, TextTextract, TextOut, Export)
pattern := fmt.Sprintf(`^(%s)/(%s)/([0-9]{8})/(.+)$`, client.CLIENT_ID_REGEX, locations)
reg := regexp.MustCompile(pattern)
matches := reg.FindStringSubmatch(key)
if len(matches) != 5 {
return BucketKey{}, errors.New("does not match expectation")
}
formattedKey := BucketKey{}
formattedKey.ClientID = matches[1]
formattedKey.Location = BucketLocation(matches[2])
parsedTime, err := time.Parse("20060102", matches[3])
if err != nil {
return BucketKey{}, fmt.Errorf("failed to parse date: %w", err)
}
formattedKey.CreatedAt = parsedTime
filename := `[a-zA-Z0-9\_\.\-]+`
reg = regexp.MustCompile(fmt.Sprintf(`^([0-9]+)/(%s)$`, filename))
subMatches := reg.FindStringSubmatch(matches[4])
if len(subMatches) == 3 {
part, err := strconv.ParseUint(subMatches[1], 10, 8)
if err != nil {
return BucketKey{}, err
}
safePart := uint8(part)
formattedKey.Part = &safePart
formattedKey.Filename = subMatches[2]
} else {
reg = regexp.MustCompile(fmt.Sprintf(`^(%s)$`, filename))
subMatches := reg.FindStringSubmatch(matches[4])
if len(subMatches) != 2 {
return BucketKey{}, errors.New("does not match expectation")
}
formattedKey.Filename = subMatches[1]
}
return formattedKey, nil
}
internal/serviceconfig/objectstore/config_test.go
+157
View File
@@ -115,3 +115,160 @@ func TestCalculateETag(t *testing.T) {
assert.Equal(t, *res.ETag, hash)
}
func TestGetBucketKey(t *testing.T) {
key := objectstore.BucketKey{}
assert.Equal(t, "//00010101/", key.String())
key.CreatedAt = time.Date(2025, time.March, 31, 0, 0, 0, 0, time.UTC)
assert.Equal(t, "//20250331/", key.String())
key.ClientID = "AAA"
assert.Equal(t, "AAA//20250331/", key.String())
key.Filename = "coolfile"
assert.Equal(t, "AAA//20250331/coolfile", key.String())
key.Location = objectstore.Import
assert.Equal(t, "AAA/import/20250331/coolfile", key.String())
part := uint8(3)
key.Part = &part
assert.Equal(t, "AAA/import/20250331/3/coolfile", key.String())
key.Location = objectstore.TextTextract
assert.Equal(t, "AAA/text/textract/20250331/3/coolfile", key.String())
key.Location = objectstore.TextOut
assert.Equal(t, "AAA/text/out/20250331/3/coolfile", key.String())
key.Location = objectstore.Export
assert.Equal(t, "AAA/export/20250331/3/coolfile", key.String())
}
func TestParseBucketKey(t *testing.T) {
_, err := objectstore.ParseBucketKey("")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("//00010101/")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("//20250331/")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA//20250331/")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA//20250331/coolfile")
assert.Error(t, err)
key, err := objectstore.ParseBucketKey("AAA/import/20250331/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Import,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = objectstore.ParseBucketKey("AAA/text/textract/20250331/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.TextTextract,
Filename: "coolfile",
CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = objectstore.ParseBucketKey("AAA/text/out/20250331/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.TextOut,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = objectstore.ParseBucketKey("AAA/export/20250331/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Export,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
part := uint8(3)
key, err = objectstore.ParseBucketKey("AAA/import/20250331/3/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Import,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = objectstore.ParseBucketKey("AAA/text/textract/20250331/3/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.TextTextract,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = objectstore.ParseBucketKey("AAA/text/out/20250331/3/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.TextOut,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = objectstore.ParseBucketKey("AAA/export/20250331/3/coolfile")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Export,
Filename: "coolfile",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = objectstore.ParseBucketKey("AAA/import/20250331/cool-file")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Import,
Filename: "cool-file",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = objectstore.ParseBucketKey("AAA/import/20250331/cool_file")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Import,
Filename: "cool_file",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = objectstore.ParseBucketKey("AAA/import/20250331/coolfile365")
assert.NoError(t, err)
assert.EqualExportedValues(t, objectstore.BucketKey{
ClientID: "AAA",
Location: objectstore.Import,
Filename: "coolfile365",
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
_, err = objectstore.ParseBucketKey("AAA/import/99999999/coolfile")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA/import/20250331/99999999/coolfile")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA/import/20250331/999/coolfile")
assert.Error(t, err)
_, err = objectstore.ParseBucketKey("AAA/import/20250331/1/cool**file")
assert.Error(t, err)
}