jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/pathstandard (pull request #146)

Browse Source 
Change Path Separator

* clientid
This commit is contained in:
Michael McGuinness
2025-05-20 17:00:50 +00:00
parent 61d12bf8df
commit 731b68e4af
3 changed files with 99 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/serviceconfig/objectstore/bucketkey.go
+2 -2
View File
@@ -64,7 +64,7 @@ func (c *BucketKey) Prefix() string {
func (c *BucketKey) Filename() string {
location := strings.ReplaceAll(string(c.Location), "/", "-")
name := fmt.Sprintf("%s_%s_%s_%s", c.CreatedAt.Format(TimeFormat), c.ClientID, location, c.EntityID)
name := fmt.Sprintf("%s~%s~%s~%s", c.CreatedAt.Format(TimeFormat), c.ClientID, location, c.EntityID)
if c.FileType != nil {
name = fmt.Sprintf("%s.%s", name, *c.FileType)
}
@@ -80,7 +80,7 @@ func (c *BucketKey) parseFilename(name string) error {
c.parseFileType(parts[1])
}
metadata := strings.Split(parts[0], "_")
metadata := strings.Split(parts[0], "~")
if len(metadata) != 4 {
return errors.New("incorrect amount of metadata elements")
}
internal/serviceconfig/objectstore/bucketkey_test.go
+95 -26
View File
@@ -10,30 +10,36 @@ import (
func TestGetBucketKey(t *testing.T) {
key := BucketKey{}
assert.Equal(t, "//00010101/0/0001-01-01T000000Z___00000000-0000-0000-0000-000000000000", key.String())
assert.Equal(t, "//00010101/0/0001-01-01T000000Z~~~00000000-0000-0000-0000-000000000000", key.String())
key.CreatedAt = time.Date(2025, time.March, 31, 4, 8, 16, 222, time.UTC)
assert.Equal(t, "//20250331/0/2025-03-31T040816Z___00000000-0000-0000-0000-000000000000", key.String())
assert.Equal(t, "//20250331/0/2025-03-31T040816Z~~~00000000-0000-0000-0000-000000000000", key.String())
key.ClientID = "A_A"
assert.Equal(t, "A_A//20250331/0/2025-03-31T040816Z~A_A~~00000000-0000-0000-0000-000000000000", key.String())
key.ClientID = "A-A"
assert.Equal(t, "A-A//20250331/0/2025-03-31T040816Z~A-A~~00000000-0000-0000-0000-000000000000", key.String())
key.ClientID = "A#A"
assert.Equal(t, "A#A//20250331/0/2025-03-31T040816Z~A#A~~00000000-0000-0000-0000-000000000000", key.String())
key.ClientID = "AAA"
assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__00000000-0000-0000-0000-000000000000", key.String())
assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z~AAA~~00000000-0000-0000-0000-000000000000", key.String())
key.EntityID = uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40")
assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z~AAA~~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Import
assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Text
assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Export
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
part := uint16(3)
key.Part = &part
key.Location = Import
assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Text
assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
key.Location = Export
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String())
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40", key.String())
filetype := "csv"
key.FileType = &filetype
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40.csv", key.String())
assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40.csv", key.String())
}
func TestParseBucketKey(t *testing.T) {
_, err := ParseBucketKey("")
@@ -54,7 +60,7 @@ func TestParseBucketKey(t *testing.T) {
_, err = ParseBucketKey("AAA//20250331/coolfile")
assert.Error(t, err)
key, err := ParseBucketKey("AAA/import/20250331/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40")
key, err := ParseBucketKey("AAA/import/20250331/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -63,7 +69,7 @@ func TestParseBucketKey(t *testing.T) {
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -72,7 +78,7 @@ func TestParseBucketKey(t *testing.T) {
CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC),
}, key)
key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -82,7 +88,7 @@ func TestParseBucketKey(t *testing.T) {
}, key)
part := uint16(3)
key, err = ParseBucketKey("AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/import/20250331/3/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -92,7 +98,7 @@ func TestParseBucketKey(t *testing.T) {
Part: &part,
}, key)
key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -102,7 +108,7 @@ func TestParseBucketKey(t *testing.T) {
Part: &part,
}, key)
key, err = ParseBucketKey("AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("AAA/export/20250331/3/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "AAA",
@@ -112,16 +118,46 @@ func TestParseBucketKey(t *testing.T) {
Part: &part,
}, key)
_, err = ParseBucketKey("AAA/import/99999999/2025-03-31T990816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40")
key, err = ParseBucketKey("A_A/export/20250331/3/2025-03-31T040816Z~A_A~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "A_A",
Location: Export,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = ParseBucketKey("A-A/export/20250331/3/2025-03-31T040816Z~A-A~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "A-A",
Location: Export,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
key, err = ParseBucketKey("A#A/export/20250331/3/2025-03-31T040816Z~A#A~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
ClientID: "A#A",
Location: Export,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC),
Part: &part,
}, key)
_, err = ParseBucketKey("AAA/import/99999999/2025-03-31T990816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40")
assert.Error(t, err)
_, err = ParseBucketKey("AAA/import/20250331/99999999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40")
_, err = ParseBucketKey("AAA/import/20250331/99999999/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40")
assert.Error(t, err)
_, err = ParseBucketKey("AAA/import/20250331/999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40")
_, err = ParseBucketKey("AAA/import/20250331/999/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40")
assert.Error(t, err)
_, err = ParseBucketKey("****AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40")
_, err = ParseBucketKey("*AAA/export/20250331/3/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40")
assert.Error(t, err)
}
@@ -137,19 +173,19 @@ func TestParseFilename(t *testing.T) {
err = key.parseFilename("a.a.a")
assert.EqualError(t, err, "incorrect amount of dots")
err = key.parseFilename("a_a_a_a.a")
err = key.parseFilename("a~a~a~a.a")
assert.EqualError(t, err, "failed to parse date: parsing time \"a\" as \"2006-01-02T150405Z\": cannot parse \"a\" as \"2006\"")
err = key.parseFilename("2025-03-31T040816Z_*_a_a.a")
err = key.parseFilename("2025-03-31T040816Z~*~a~a.a")
assert.EqualError(t, err, "invalid client id")
err = key.parseFilename("2025-03-31T040816Z_a_a_a.a")
err = key.parseFilename("2025-03-31T040816Z~a~a~a.a")
assert.EqualError(t, err, "invalid location")
err = key.parseFilename("2025-03-31T040816Z_a_import_a.a")
err = key.parseFilename("2025-03-31T040816Z~a~import~a.a")
assert.EqualError(t, err, "invalid UUID length: 1")
err = key.parseFilename("2025-03-31T040816Z_a_import_b745910f-f529-43c5-87e1-25629d46ef40.a")
err = key.parseFilename("2025-03-31T040816Z~a~import~b745910f-f529-43c5-87e1-25629d46ef40.a")
assert.NoError(t, err)
filetype := "a"
assert.EqualExportedValues(t, BucketKey{
@@ -159,4 +195,37 @@ func TestParseFilename(t *testing.T) {
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
FileType: &filetype,
}, key)
err = key.parseFilename("2025-03-31T040816Z~a_a~import~b745910f-f529-43c5-87e1-25629d46ef40.a")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC),
ClientID: "a_a",
Location: Import,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
FileType: &filetype,
}, key)
err = key.parseFilename("2025-03-31T040816Z~a-a~import~b745910f-f529-43c5-87e1-25629d46ef40.a")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC),
ClientID: "a-a",
Location: Import,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
FileType: &filetype,
}, key)
err = key.parseFilename("2025-03-31T040816Z~a#a~import~b745910f-f529-43c5-87e1-25629d46ef40.a")
assert.NoError(t, err)
assert.EqualExportedValues(t, BucketKey{
CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC),
ClientID: "a#a",
Location: Import,
EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"),
FileType: &filetype,
}, key)
err = key.parseFilename("2025-03-31T040816Z~a~a~import~b745910f-f529-43c5-87e1-25629d46ef40.a")
assert.EqualError(t, err, "incorrect amount of metadata elements")
}
internal/test/mockserver.go
+2
View File
@@ -129,6 +129,7 @@ func CreateMockExpectation(t testing.TB, server *MockServer, expectation MockExp
func CreateDetectDocumentTextExpectation(t testing.TB, mockServer *MockServer, body string) MockExpectation {
childId := uuid.NewString()
pageId := uuid.NewString()
expectation := MockExpectation{
Request: MockRequest{
Method: "POST",
@@ -150,6 +151,7 @@ func CreateDetectDocumentTextExpectation(t testing.TB, mockServer *MockServer, b
Body: map[string]interface{}{
"Blocks": []map[string]interface{}{
{
"Id": pageId,
"BlockType": "PAGE",
"Relationships": []map[string]interface{}{
{