diff --git a/internal/serviceconfig/objectstore/bucketkey.go b/internal/serviceconfig/objectstore/bucketkey.go index f11b1559..7cedeaac 100644 --- a/internal/serviceconfig/objectstore/bucketkey.go +++ b/internal/serviceconfig/objectstore/bucketkey.go @@ -64,7 +64,7 @@ func (c *BucketKey) Prefix() string { func (c *BucketKey) Filename() string { location := strings.ReplaceAll(string(c.Location), "/", "-") - name := fmt.Sprintf("%s_%s_%s_%s", c.CreatedAt.Format(TimeFormat), c.ClientID, location, c.EntityID) + name := fmt.Sprintf("%s~%s~%s~%s", c.CreatedAt.Format(TimeFormat), c.ClientID, location, c.EntityID) if c.FileType != nil { name = fmt.Sprintf("%s.%s", name, *c.FileType) } @@ -80,7 +80,7 @@ func (c *BucketKey) parseFilename(name string) error { c.parseFileType(parts[1]) } - metadata := strings.Split(parts[0], "_") + metadata := strings.Split(parts[0], "~") if len(metadata) != 4 { return errors.New("incorrect amount of metadata elements") } diff --git a/internal/serviceconfig/objectstore/bucketkey_test.go b/internal/serviceconfig/objectstore/bucketkey_test.go index 9b325348..ab56b525 100644 --- a/internal/serviceconfig/objectstore/bucketkey_test.go +++ b/internal/serviceconfig/objectstore/bucketkey_test.go @@ -10,30 +10,36 @@ import ( func TestGetBucketKey(t *testing.T) { key := BucketKey{} - assert.Equal(t, "//00010101/0/0001-01-01T000000Z___00000000-0000-0000-0000-000000000000", key.String()) + assert.Equal(t, "//00010101/0/0001-01-01T000000Z~~~00000000-0000-0000-0000-000000000000", key.String()) key.CreatedAt = time.Date(2025, time.March, 31, 4, 8, 16, 222, time.UTC) - assert.Equal(t, "//20250331/0/2025-03-31T040816Z___00000000-0000-0000-0000-000000000000", key.String()) + assert.Equal(t, "//20250331/0/2025-03-31T040816Z~~~00000000-0000-0000-0000-000000000000", key.String()) + key.ClientID = "A_A" + assert.Equal(t, "A_A//20250331/0/2025-03-31T040816Z~A_A~~00000000-0000-0000-0000-000000000000", key.String()) + key.ClientID = "A-A" + assert.Equal(t, "A-A//20250331/0/2025-03-31T040816Z~A-A~~00000000-0000-0000-0000-000000000000", key.String()) + key.ClientID = "A#A" + assert.Equal(t, "A#A//20250331/0/2025-03-31T040816Z~A#A~~00000000-0000-0000-0000-000000000000", key.String()) key.ClientID = "AAA" - assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__00000000-0000-0000-0000-000000000000", key.String()) + assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z~AAA~~00000000-0000-0000-0000-000000000000", key.String()) key.EntityID = uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40") - assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z_AAA__b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA//20250331/0/2025-03-31T040816Z~AAA~~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) key.Location = Import - assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/import/20250331/0/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) key.Location = Text - assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/text/20250331/0/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) key.Location = Export - assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) part := uint16(3) key.Part = &part key.Location = Import - assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) key.Location = Text - assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/text/20250331/3/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) key.Location = Export - assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40", key.String()) filetype := "csv" key.FileType = &filetype - assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40.csv", key.String()) + assert.Equal(t, "AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40.csv", key.String()) } func TestParseBucketKey(t *testing.T) { _, err := ParseBucketKey("") @@ -54,7 +60,7 @@ func TestParseBucketKey(t *testing.T) { _, err = ParseBucketKey("AAA//20250331/coolfile") assert.Error(t, err) - key, err := ParseBucketKey("AAA/import/20250331/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + key, err := ParseBucketKey("AAA/import/20250331/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -63,7 +69,7 @@ func TestParseBucketKey(t *testing.T) { CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), }, key) - key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("AAA/text/20250331/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -72,7 +78,7 @@ func TestParseBucketKey(t *testing.T) { CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC), }, key) - key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -82,7 +88,7 @@ func TestParseBucketKey(t *testing.T) { }, key) part := uint16(3) - key, err = ParseBucketKey("AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("AAA/import/20250331/3/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -92,7 +98,7 @@ func TestParseBucketKey(t *testing.T) { Part: &part, }, key) - key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z_AAA_text_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("AAA/text/20250331/3/2025-03-31T040816Z~AAA~text~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -102,7 +108,7 @@ func TestParseBucketKey(t *testing.T) { Part: &part, }, key) - key, err = ParseBucketKey("AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("AAA/export/20250331/3/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40") assert.NoError(t, err) assert.EqualExportedValues(t, BucketKey{ ClientID: "AAA", @@ -112,16 +118,46 @@ func TestParseBucketKey(t *testing.T) { Part: &part, }, key) - _, err = ParseBucketKey("AAA/import/99999999/2025-03-31T990816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + key, err = ParseBucketKey("A_A/export/20250331/3/2025-03-31T040816Z~A_A~export~b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "A_A", + Location: Export, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + key, err = ParseBucketKey("A-A/export/20250331/3/2025-03-31T040816Z~A-A~export~b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "A-A", + Location: Export, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + key, err = ParseBucketKey("A#A/export/20250331/3/2025-03-31T040816Z~A#A~export~b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "A#A", + Location: Export, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + _, err = ParseBucketKey("AAA/import/99999999/2025-03-31T990816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40") assert.Error(t, err) - _, err = ParseBucketKey("AAA/import/20250331/99999999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + _, err = ParseBucketKey("AAA/import/20250331/99999999/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40") assert.Error(t, err) - _, err = ParseBucketKey("AAA/import/20250331/999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + _, err = ParseBucketKey("AAA/import/20250331/999/2025-03-31T040816Z~AAA~import~b745910f-f529-43c5-87e1-25629d46ef40") assert.Error(t, err) - _, err = ParseBucketKey("****AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + _, err = ParseBucketKey("*AAA/export/20250331/3/2025-03-31T040816Z~AAA~export~b745910f-f529-43c5-87e1-25629d46ef40") assert.Error(t, err) } @@ -137,19 +173,19 @@ func TestParseFilename(t *testing.T) { err = key.parseFilename("a.a.a") assert.EqualError(t, err, "incorrect amount of dots") - err = key.parseFilename("a_a_a_a.a") + err = key.parseFilename("a~a~a~a.a") assert.EqualError(t, err, "failed to parse date: parsing time \"a\" as \"2006-01-02T150405Z\": cannot parse \"a\" as \"2006\"") - err = key.parseFilename("2025-03-31T040816Z_*_a_a.a") + err = key.parseFilename("2025-03-31T040816Z~*~a~a.a") assert.EqualError(t, err, "invalid client id") - err = key.parseFilename("2025-03-31T040816Z_a_a_a.a") + err = key.parseFilename("2025-03-31T040816Z~a~a~a.a") assert.EqualError(t, err, "invalid location") - err = key.parseFilename("2025-03-31T040816Z_a_import_a.a") + err = key.parseFilename("2025-03-31T040816Z~a~import~a.a") assert.EqualError(t, err, "invalid UUID length: 1") - err = key.parseFilename("2025-03-31T040816Z_a_import_b745910f-f529-43c5-87e1-25629d46ef40.a") + err = key.parseFilename("2025-03-31T040816Z~a~import~b745910f-f529-43c5-87e1-25629d46ef40.a") assert.NoError(t, err) filetype := "a" assert.EqualExportedValues(t, BucketKey{ @@ -159,4 +195,37 @@ func TestParseFilename(t *testing.T) { EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), FileType: &filetype, }, key) + + err = key.parseFilename("2025-03-31T040816Z~a_a~import~b745910f-f529-43c5-87e1-25629d46ef40.a") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC), + ClientID: "a_a", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + FileType: &filetype, + }, key) + + err = key.parseFilename("2025-03-31T040816Z~a-a~import~b745910f-f529-43c5-87e1-25629d46ef40.a") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC), + ClientID: "a-a", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + FileType: &filetype, + }, key) + + err = key.parseFilename("2025-03-31T040816Z~a#a~import~b745910f-f529-43c5-87e1-25629d46ef40.a") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC), + ClientID: "a#a", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + FileType: &filetype, + }, key) + + err = key.parseFilename("2025-03-31T040816Z~a~a~import~b745910f-f529-43c5-87e1-25629d46ef40.a") + assert.EqualError(t, err, "incorrect amount of metadata elements") } diff --git a/internal/test/mockserver.go b/internal/test/mockserver.go index 3d6fe03c..a015e09e 100644 --- a/internal/test/mockserver.go +++ b/internal/test/mockserver.go @@ -129,6 +129,7 @@ func CreateMockExpectation(t testing.TB, server *MockServer, expectation MockExp func CreateDetectDocumentTextExpectation(t testing.TB, mockServer *MockServer, body string) MockExpectation { childId := uuid.NewString() + pageId := uuid.NewString() expectation := MockExpectation{ Request: MockRequest{ Method: "POST", @@ -150,6 +151,7 @@ func CreateDetectDocumentTextExpectation(t testing.TB, mockServer *MockServer, b Body: map[string]interface{}{ "Blocks": []map[string]interface{}{ { + "Id": pageId, "BlockType": "PAGE", "Relationships": []map[string]interface{}{ {