From 60812aba91e38ec3167bf16988ce296ab0196165 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Thu, 3 Apr 2025 19:17:24 +0000 Subject: [PATCH] Merged in feature/richname (pull request #112) Standardised rich name for Files * tests --- api/docCleanRunner/runner_test.go | 19 +- api/docInitRunner/runner.go | 24 +-- api/docInitRunner/runner_test.go | 14 +- api/docTextProcessRunner/runner.go | 14 +- api/docTextProcessRunner/runner_test.go | 18 +- api/docTextRunner/runner_test.go | 14 +- api/storeEventRunner/runner_test.go | 1 - internal/document/clean/clean.go | 46 +++-- internal/document/clean/clean_test.go | 106 +++++++---- internal/document/clean/content.go | 5 +- internal/document/clean/contentType.go | 5 +- internal/document/clean/contentType_test.go | 138 ++++++++------ internal/document/clean/content_test.go | 21 ++- internal/document/clean/create_test.go | 19 +- internal/document/init/create.go | 37 ++-- internal/document/init/create_test.go | 112 ++++++----- internal/document/service.go | 5 - internal/document/store/process.go | 14 +- internal/document/store/process_test.go | 2 - internal/document/text/extract_test.go | 9 +- internal/document/text/processExtract.go | 21 +-- internal/document/text/processExtract_test.go | 18 +- internal/document/text/triggerExtract.go | 29 +-- internal/document/text/triggerExtract_test.go | 43 +---- .../serviceconfig/objectstore/bucketkey.go | 175 +++++++++++++++++ .../objectstore/bucketkey_test.go | 178 ++++++++++++++++++ internal/serviceconfig/objectstore/config.go | 76 -------- .../serviceconfig/objectstore/config_test.go | 157 --------------- internal/test/ecosystem_test.go | 2 - internal/test/objectstore_test.go | 1 - test/process_test.go | 22 +-- 31 files changed, 744 insertions(+), 601 deletions(-) create mode 100644 internal/serviceconfig/objectstore/bucketkey.go create mode 100644 internal/serviceconfig/objectstore/bucketkey_test.go diff --git a/api/docCleanRunner/runner_test.go b/api/docCleanRunner/runner_test.go index 760177db..e39e74d5 100644 --- a/api/docCleanRunner/runner_test.go +++ b/api/docCleanRunner/runner_test.go @@ -6,6 +6,7 @@ import ( "io" "strings" "testing" + "time" doccleanrunner "queryorchestration/api/docCleanRunner" "queryorchestration/internal/database/repository" @@ -63,10 +64,14 @@ func TestDocCleanRunner(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "hello" + key := objectstore.BucketKey{ + CreatedAt: time.Now().UTC(), + ClientID: "hi", + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() dbid := doc.ID pool.ExpectQuery("name: HasDocumentCleanEntry :one").WithArgs(dbid).WillReturnRows( @@ -80,7 +85,7 @@ func TestDocCleanRunner(t *testing.T) { ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(dbid, inloc.Bucket, inloc.Key), + AddRow(dbid, bucket, keyStr), ) mimeType := "application/pdf" dbmimetype := repository.NullCleanmimetype{ @@ -93,7 +98,7 @@ func TestDocCleanRunner(t *testing.T) { WillReturnRows( pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}), ) - pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). + pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(cleanid), @@ -116,7 +121,7 @@ func TestDocCleanRunner(t *testing.T) { HeadObject( mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). @@ -129,7 +134,7 @@ func TestDocCleanRunner(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). diff --git a/api/docInitRunner/runner.go b/api/docInitRunner/runner.go index 51ce4fea..ccba1382 100644 --- a/api/docInitRunner/runner.go +++ b/api/docInitRunner/runner.go @@ -4,8 +4,8 @@ import ( "context" "log/slog" - "queryorchestration/internal/document" documentinit "queryorchestration/internal/document/init" + "queryorchestration/internal/serviceconfig/objectstore" "github.com/go-playground/validator/v10" ) @@ -29,20 +29,22 @@ func New(validator *validator.Validate, svc *Services) Runner { } type Body struct { - Bucket string `json:"bucket" validate:"required"` - Key string `json:"key" validate:"required"` - Hash string `json:"hash" validate:"required"` - ClientID string `json:"clientId" validate:"required"` + Bucket string `json:"bucket" validate:"required"` + Key string `json:"key" validate:"required"` + Hash string `json:"hash" validate:"required"` } func (s Runner) Process(ctx context.Context, body Body) bool { + key, err := objectstore.ParseBucketKey(body.Key) + if err != nil { + slog.Error("unable to parse key", "key", body.Key) + return false + } + id, err := s.svc.Document.Create(ctx, &documentinit.Create{ - ClientID: body.ClientID, - Location: document.Location{ - Bucket: body.Bucket, - Key: body.Key, - }, - Hash: body.Hash, + Key: key, + Bucket: body.Bucket, + Hash: body.Hash, }) if err != nil { return false diff --git a/api/docInitRunner/runner_test.go b/api/docInitRunner/runner_test.go index df9771cf..a4880fb1 100644 --- a/api/docInitRunner/runner_test.go +++ b/api/docInitRunner/runner_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "time" docinitrunner "queryorchestration/api/docInitRunner" "queryorchestration/internal/database/repository" @@ -49,9 +50,9 @@ func TestDocInitRunner(t *testing.T) { clientId := "clientid" bucketName := "bucketName" location := objectstore.BucketKey{ - Location: objectstore.Import, - ClientID: clientId, - Filename: "aaa", + Location: objectstore.Import, + ClientID: clientId, + CreatedAt: time.Now().UTC(), } docinfo := document.DocumentSummary{ ID: uuid.New(), @@ -59,10 +60,9 @@ func TestDocInitRunner(t *testing.T) { Hash: "example_hash", } doc := docinitrunner.Body{ - Bucket: bucketName, - Key: location.String(), - Hash: docinfo.Hash, - ClientID: clientId, + Bucket: bucketName, + Key: location.String(), + Hash: docinfo.Hash, } pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(docinfo.Hash, clientId).WillReturnRows( diff --git a/api/docTextProcessRunner/runner.go b/api/docTextProcessRunner/runner.go index ad99c96f..554562c8 100644 --- a/api/docTextProcessRunner/runner.go +++ b/api/docTextProcessRunner/runner.go @@ -29,10 +29,9 @@ func New(validator *validator.Validate, svc *Services) Runner { } type Body struct { - Bucket string `json:"bucket" validate:"required"` - Key string `json:"key" validate:"required"` - Hash string `json:"hash" validate:"required"` - ClientID string `json:"clientId" validate:"required"` + Bucket string `json:"bucket" validate:"required"` + Key string `json:"key" validate:"required"` + Hash string `json:"hash" validate:"required"` } func (s Runner) Process(ctx context.Context, body Body) bool { @@ -43,10 +42,9 @@ func (s Runner) Process(ctx context.Context, body Body) bool { } err = s.svc.Text.Process(ctx, &documenttext.ProcessParams{ - Bucket: body.Bucket, - Key: key, - Hash: body.Hash, - ClientID: body.ClientID, + Bucket: body.Bucket, + Key: key, + Hash: body.Hash, }) if err != nil { slog.Error("unable to process", "bucket", body.Bucket, "key", body.Key, "hash", body.Hash, "error", err) diff --git a/api/docTextProcessRunner/runner_test.go b/api/docTextProcessRunner/runner_test.go index 80769649..520d80db 100644 --- a/api/docTextProcessRunner/runner_test.go +++ b/api/docTextProcessRunner/runner_test.go @@ -72,14 +72,13 @@ func TestDocTextProcessRunner(t *testing.T) { key := objectstore.BucketKey{ ClientID: docinfo.ClientID, Location: objectstore.TextTextract, - Filename: triggerId.String(), + EntityID: triggerId, CreatedAt: time.Now().UTC(), } doc := Body{ - Bucket: bucketName, - Key: key.String(), - Hash: docinfo.Hash, - ClientID: docinfo.ClientID, + Bucket: bucketName, + Key: key.String(), + Hash: docinfo.Hash, } pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId).WillReturnRows( @@ -139,14 +138,13 @@ func TestDocTextProcessRunner(t *testing.T) { key := objectstore.BucketKey{ ClientID: docinfo.ClientID, Location: objectstore.TextTextract, - Filename: triggerId.String(), + EntityID: triggerId, CreatedAt: time.Now().UTC(), } doc := Body{ - Bucket: bucketName, - Key: key.String(), - Hash: docinfo.Hash, - ClientID: docinfo.ClientID, + Bucket: bucketName, + Key: key.String(), + Hash: docinfo.Hash, } pool.ExpectQuery("name: GetDocumentTextTrigger :one").WithArgs(triggerId). diff --git a/api/docTextRunner/runner_test.go b/api/docTextRunner/runner_test.go index 7a9542f4..6d79f092 100644 --- a/api/docTextRunner/runner_test.go +++ b/api/docTextRunner/runner_test.go @@ -3,10 +3,10 @@ package doctextrunner_test import ( "context" "testing" + "time" doctextrunner "queryorchestration/api/docTextRunner" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" documenttext "queryorchestration/internal/document/text" "queryorchestration/internal/server/runner" "queryorchestration/internal/serviceconfig/aws" @@ -62,10 +62,14 @@ func TestDocCleanRunner(t *testing.T) { } cleanId := uuid.New() - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "hello" + key := objectstore.BucketKey{ + CreatedAt: time.Now().UTC(), + ClientID: "hi", + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() hash := "example" pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(doc.DocumentID).WillReturnRows( @@ -79,7 +83,7 @@ func TestDocCleanRunner(t *testing.T) { } pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(doc.DocumentID).WillReturnRows( pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}). - AddRow(cleanId, doc.DocumentID, &inloc.Bucket, &inloc.Key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, "clientId"), + AddRow(cleanId, doc.DocumentID, &bucket, &keyStr, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, "clientId"), ) jobId := "hello" triggerId := uuid.New() diff --git a/api/storeEventRunner/runner_test.go b/api/storeEventRunner/runner_test.go index 0a4e8b19..8dccc6b3 100644 --- a/api/storeEventRunner/runner_test.go +++ b/api/storeEventRunner/runner_test.go @@ -54,7 +54,6 @@ func TestDocInitRunner(t *testing.T) { location := objectstore.BucketKey{ Location: objectstore.Import, ClientID: clientId, - Filename: "aaa", CreatedAt: time.Now().UTC(), } docinfo := document.DocumentSummary{ diff --git a/internal/document/clean/clean.go b/internal/document/clean/clean.go index 58c545ae..6b12bad8 100644 --- a/internal/document/clean/clean.go +++ b/internal/document/clean/clean.go @@ -8,31 +8,34 @@ import ( "log/slog" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" documenttypes "queryorchestration/internal/document/types" "queryorchestration/internal/serviceconfig/build" + "queryorchestration/internal/serviceconfig/objectstore" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/google/uuid" ) type CleanParams struct { - ID uuid.UUID - Hash string - Location document.Location + ID uuid.UUID + Hash string + Bucket string + Key objectstore.BucketKey } type ExecuteCleanResponse struct { - location *document.Location + key *objectstore.BucketKey + bucket *string mimetype *documenttypes.MimeType hash *string failReason *documenttypes.InvalidDocumentReason } func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (*ExecuteCleanResponse, error) { + keyStr := params.Key.String() out, err := s.cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{ - Bucket: ¶ms.Location.Bucket, - Key: ¶ms.Location.Key, + Bucket: ¶ms.Bucket, + Key: &keyStr, IfMatch: ¶ms.Hash, }) if err != nil { @@ -67,8 +70,9 @@ func (s *Service) executeCleanTasks(ctx context.Context, params *CleanParams) (* } return &ExecuteCleanResponse{ - location: ¶ms.Location, mimetype: &mimeType, + bucket: ¶ms.Bucket, + key: ¶ms.Key, hash: ¶ms.Hash, }, nil } @@ -87,13 +91,16 @@ func (s *Service) clean(ctx context.Context, id uuid.UUID) error { return err } + key, err := objectstore.ParseBucketKey(entry.Key) + if err != nil { + return err + } + out, err := s.executeCleanTasks(ctx, &CleanParams{ - ID: id, - Hash: doc.Hash, - Location: document.Location{ - Bucket: entry.Bucket, - Key: entry.Key, - }, + ID: id, + Hash: doc.Hash, + Bucket: entry.Bucket, + Key: key, }) if err != nil { return err @@ -118,8 +125,9 @@ func (s *Service) storeClean(ctx context.Context, id uuid.UUID, out *ExecuteClea slog.Info("Failed document", "id", id, "reason", *out.failReason) params.Fail = documenttypes.ToDBNullFailType(*out.failReason) } else { - params.Bucket = &out.location.Bucket - params.Key = &out.location.Key + params.Bucket = out.bucket + key := out.key.String() + params.Key = &key params.Hash = out.hash params.Mimetype = documenttypes.ToDBNullMimeType(*out.mimetype) } @@ -174,11 +182,11 @@ func (s *Service) isNewClean(lastEntry *repository.GetMostRecentDocumentCleanEnt return true } - if out.location == nil || out.mimetype == nil || lastEntry.Bucket == nil || lastEntry.Key == nil { + if out.bucket == nil || out.key == nil || out.mimetype == nil || lastEntry.Bucket == nil || lastEntry.Key == nil { return true } - return !(out.location.Bucket == *lastEntry.Bucket && - out.location.Key == *lastEntry.Key && + return !(*out.bucket == *lastEntry.Bucket && + out.key.String() == *lastEntry.Key && *out.mimetype == documenttypes.ParseDBNullMimeType(lastEntry.Mimetype)) } diff --git a/internal/document/clean/clean_test.go b/internal/document/clean/clean_test.go index 9c5f0bdc..e2136013 100644 --- a/internal/document/clean/clean_test.go +++ b/internal/document/clean/clean_test.go @@ -5,10 +5,12 @@ import ( "io" "strings" "testing" + "time" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" documenttypes "queryorchestration/internal/document/types" + "queryorchestration/internal/serviceconfig/objectstore" objectstoremock "queryorchestration/mocks/objectstore" "github.com/stretchr/testify/require" @@ -40,10 +42,14 @@ func TestClean(t *testing.T) { ClientID: "yuyu", Hash: "example_hash", } - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() dbid := doc.ID pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(dbid). @@ -53,7 +59,7 @@ func TestClean(t *testing.T) { ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(dbid, inloc.Bucket, inloc.Key), + AddRow(dbid, bucket, keyStr), ) mimeType := "application/pdf" dbmimetype := repository.NullCleanmimetype{ @@ -66,7 +72,7 @@ func TestClean(t *testing.T) { pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}), ) cleanid := uuid.New() - pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). + pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(cleanid), @@ -80,7 +86,7 @@ func TestClean(t *testing.T) { HeadObject( mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). @@ -94,7 +100,7 @@ func TestClean(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). @@ -118,17 +124,21 @@ func TestExecuteCleanTasks(t *testing.T) { id := uuid.New() hash := "example_hash" - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: "client", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() mimeType := "application/pdf" mockS3.EXPECT(). HeadObject( mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). @@ -141,7 +151,7 @@ func TestExecuteCleanTasks(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == keyStr }), mock.Anything, ). @@ -150,14 +160,14 @@ func TestExecuteCleanTasks(t *testing.T) { }, nil) outloc, err := svc.executeCleanTasks(ctx, &CleanParams{ - ID: id, - Hash: hash, - Location: inloc, + ID: id, + Hash: hash, + Bucket: bucket, + Key: key, }) require.NoError(t, err) assert.EqualExportedValues(t, ExecuteCleanResponse{ - hash: &hash, - location: &inloc, + hash: &hash, }, *outloc) } @@ -177,17 +187,22 @@ func TestStoreClean(t *testing.T) { doc := document.DocumentSummary{ ID: uuid.New(), } - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() t.Run("no prev entry", func(t *testing.T) { mimeType := "application/pdf" hash := "heyman" params := &ExecuteCleanResponse{ - location: &inloc, hash: &hash, mimetype: (*documenttypes.MimeType)(&mimeType), + key: &key, + bucket: &bucket, } dbid := doc.ID @@ -201,7 +216,7 @@ func TestStoreClean(t *testing.T) { pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}), ) cleanid := uuid.New() - pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, params.hash, dbmimetype, repository.NullCleanfailtype{}). + pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, params.hash, dbmimetype, repository.NullCleanfailtype{}). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(cleanid), @@ -217,8 +232,9 @@ func TestStoreClean(t *testing.T) { mimeType := "application/pdf" hash := "heyman" params := &ExecuteCleanResponse{ - location: &inloc, hash: &hash, + bucket: &bucket, + key: &key, mimetype: (*documenttypes.MimeType)(&mimeType), } dbid := doc.ID @@ -234,7 +250,7 @@ func TestStoreClean(t *testing.T) { AddRow(uuid.New(), dbid, nil, nil, int64(1), nil, nil, repository.CleanfailtypeInvalidRead), ) cleanid := uuid.New() - pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, params.hash, dbmimetype, repository.NullCleanfailtype{}). + pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, params.hash, dbmimetype, repository.NullCleanfailtype{}). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(cleanid), @@ -308,47 +324,59 @@ func TestIsNewClean(t *testing.T) { assert.True(t, svc.isNewClean(lastEntry, params)) }) t.Run("same location", func(t *testing.T) { - loc := document.Location{ - Bucket: "bucket", - Key: "hi", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: "diw", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() mimeType := documenttypes.MimeTypePDF var lastEntry *repository.GetMostRecentDocumentCleanEntryRow = &repository.GetMostRecentDocumentCleanEntryRow{ ID: uuid.New(), - Bucket: &loc.Bucket, - Key: &loc.Key, + Bucket: &bucket, + Key: &keyStr, Mimetype: repository.NullCleanmimetype{ Valid: true, Cleanmimetype: repository.Cleanmimetype(mimeType), }, } params := &ExecuteCleanResponse{ - location: &loc, + bucket: &bucket, + key: &key, mimetype: &mimeType, } assert.False(t, svc.isNewClean(lastEntry, params)) }) t.Run("different location", func(t *testing.T) { - ogloc := document.Location{ - Bucket: "bucket", - Key: "hi", + bucket := "example_bucket" + oldKey := objectstore.BucketKey{ + ClientID: "hi", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } - newloc := document.Location{ - Bucket: "bucket", - Key: "different_location", + oldKeyStr := oldKey.String() + newKey := objectstore.BucketKey{ + ClientID: "hi", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } mimeType := documenttypes.MimeTypePDF var lastEntry *repository.GetMostRecentDocumentCleanEntryRow = &repository.GetMostRecentDocumentCleanEntryRow{ ID: uuid.New(), - Bucket: &ogloc.Bucket, - Key: &ogloc.Key, + Bucket: &bucket, + Key: &oldKeyStr, Mimetype: repository.NullCleanmimetype{ Valid: true, Cleanmimetype: repository.Cleanmimetype(mimeType), }, } params := &ExecuteCleanResponse{ - location: &newloc, + bucket: &bucket, + key: &newKey, mimetype: &mimeType, } assert.True(t, svc.isNewClean(lastEntry, params)) diff --git a/internal/document/clean/content.go b/internal/document/clean/content.go index 1e287c48..cf705b40 100644 --- a/internal/document/clean/content.go +++ b/internal/document/clean/content.go @@ -14,9 +14,10 @@ type Content interface { } func (s *Service) getContent(ctx context.Context, params *CleanParams, mimeType documenttypes.MimeType) (Content, error) { + key := params.Key.String() resp, err := s.cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{ - Bucket: ¶ms.Location.Bucket, - Key: ¶ms.Location.Key, + Bucket: ¶ms.Bucket, + Key: &key, IfMatch: ¶ms.Hash, }) if err != nil { diff --git a/internal/document/clean/contentType.go b/internal/document/clean/contentType.go index f9f45411..333dc8f7 100644 --- a/internal/document/clean/contentType.go +++ b/internal/document/clean/contentType.go @@ -83,9 +83,10 @@ func (s *Service) getBytesBuffer(ctx context.Context, params *CleanParams, start } byteRange := fmt.Sprintf("bytes=%d-%d", start, length-1) + key := params.Key.String() out, err := s.cfg.GetStoreClient().GetObject(ctx, &s3.GetObjectInput{ - Bucket: ¶ms.Location.Bucket, - Key: ¶ms.Location.Key, + Bucket: ¶ms.Bucket, + Key: &key, IfMatch: ¶ms.Hash, Range: &byteRange, }) diff --git a/internal/document/clean/contentType_test.go b/internal/document/clean/contentType_test.go index cf2e7bb6..30f26f09 100644 --- a/internal/document/clean/contentType_test.go +++ b/internal/document/clean/contentType_test.go @@ -6,12 +6,14 @@ import ( "io" "strings" "testing" + "time" - "queryorchestration/internal/document" documenttypes "queryorchestration/internal/document/types" + "queryorchestration/internal/serviceconfig/objectstore" objectstoremock "queryorchestration/mocks/objectstore" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -28,10 +30,13 @@ func TestGetBodyMimeType(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -41,7 +46,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -63,10 +68,13 @@ func TestGetBodyMimeType(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -76,7 +84,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -87,7 +95,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -109,10 +117,13 @@ func TestGetBodyMimeType(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -122,7 +133,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -133,7 +144,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -155,10 +166,13 @@ func TestGetBodyMimeType(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -168,7 +182,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -179,7 +193,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -201,10 +215,13 @@ func TestGetBodyMimeType(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -214,7 +231,7 @@ func TestGetBodyMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -261,9 +278,12 @@ func TestGetAcceptedMimeType(t *testing.T) { } params := &CleanParams{ - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -273,7 +293,7 @@ func TestGetAcceptedMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() }), mock.Anything, ). @@ -309,9 +329,12 @@ func TestGetAcceptedMimeType(t *testing.T) { } params := &CleanParams{ - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } @@ -321,7 +344,7 @@ func TestGetAcceptedMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() }), mock.Anything, ). @@ -332,7 +355,7 @@ func TestGetAcceptedMimeType(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash }), mock.Anything, ). @@ -358,10 +381,13 @@ func TestGetBytesBuffer(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } start := int64(0) @@ -373,7 +399,7 @@ func TestGetBytesBuffer(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash && *in.Range == "bytes=0-6" + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash && *in.Range == "bytes=0-6" }), mock.Anything, ). @@ -395,10 +421,13 @@ func TestGetBytesBuffer(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } start := int64(-1) @@ -410,7 +439,7 @@ func TestGetBytesBuffer(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash && *in.Range == "bytes=0-6" + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash && *in.Range == "bytes=0-6" }), mock.Anything, ). @@ -432,10 +461,13 @@ func TestGetBytesBuffer(t *testing.T) { } params := &CleanParams{ - Hash: "example_hash", - Location: document.Location{ - Bucket: "buck", - Key: "Key", + Hash: "example_hash", + Bucket: "example_bucket", + Key: objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, }, } start := int64(0) @@ -447,7 +479,7 @@ func TestGetBytesBuffer(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == params.Location.Bucket && *in.Key == params.Location.Key && *in.IfMatch == params.Hash && *in.Range == "bytes=0-0" + return *in.Bucket == params.Bucket && *in.Key == params.Key.String() && *in.IfMatch == params.Hash && *in.Range == "bytes=0-0" }), mock.Anything, ). diff --git a/internal/document/clean/content_test.go b/internal/document/clean/content_test.go index 49ce8de5..303133e3 100644 --- a/internal/document/clean/content_test.go +++ b/internal/document/clean/content_test.go @@ -5,12 +5,14 @@ import ( "io" "strings" "testing" + "time" - "queryorchestration/internal/document" documenttypes "queryorchestration/internal/document/types" + "queryorchestration/internal/serviceconfig/objectstore" objectstoremock "queryorchestration/mocks/objectstore" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -28,14 +30,19 @@ func TestGetContent(t *testing.T) { cfg: cfg, } - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: "ddddddd", + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } params := &CleanParams{ - Hash: "hash", - Location: inloc, + Hash: "hash", + Bucket: bucket, + Key: key, } + mimeType := documenttypes.MimeTypePDF body := io.NopCloser(strings.NewReader(pdfHelloWorld)) @@ -43,7 +50,7 @@ func TestGetContent(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == key.String() }), mock.Anything, ). diff --git a/internal/document/clean/create_test.go b/internal/document/clean/create_test.go index 0414caeb..e940103d 100644 --- a/internal/document/clean/create_test.go +++ b/internal/document/clean/create_test.go @@ -6,6 +6,7 @@ import ( "io" "strings" "testing" + "time" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" @@ -55,10 +56,14 @@ func TestCreate(t *testing.T) { ClientID: "yuyu", Hash: "example_hash", } - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } + keyStr := key.String() dbid := doc.ID pool.ExpectQuery("name: HasDocumentCleanEntry :one").WithArgs(dbid).WillReturnRows( @@ -72,7 +77,7 @@ func TestCreate(t *testing.T) { ) pool.ExpectQuery("name: GetDocumentEntry :one").WithArgs(dbid).WillReturnRows( pgxmock.NewRows([]string{"documentId", "bucket", "key"}). - AddRow(dbid, inloc.Bucket, inloc.Key), + AddRow(dbid, bucket, keyStr), ) mimeType := "application/pdf" dbmimetype := repository.NullCleanmimetype{ @@ -85,7 +90,7 @@ func TestCreate(t *testing.T) { pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail"}), ) cleanid := uuid.New() - pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &inloc.Bucket, &inloc.Key, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). + pool.ExpectQuery("name: AddDocumentClean :one").WithArgs(dbid, &bucket, &keyStr, &doc.Hash, dbmimetype, repository.NullCleanfailtype{}). WillReturnRows( pgxmock.NewRows([]string{"id"}). AddRow(cleanid), @@ -108,7 +113,7 @@ func TestCreate(t *testing.T) { HeadObject( mock.Anything, mock.MatchedBy(func(in *s3.HeadObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == key.String() }), mock.Anything, ). @@ -121,7 +126,7 @@ func TestCreate(t *testing.T) { GetObject( mock.Anything, mock.MatchedBy(func(in *s3.GetObjectInput) bool { - return *in.Bucket == inloc.Bucket && *in.Key == inloc.Key + return *in.Bucket == bucket && *in.Key == key.String() }), mock.Anything, ). diff --git a/internal/document/init/create.go b/internal/document/init/create.go index 810a79b2..2cb6b294 100644 --- a/internal/document/init/create.go +++ b/internal/document/init/create.go @@ -8,17 +8,16 @@ import ( docsyncrunner "queryorchestration/api/docSyncRunner" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" + "queryorchestration/internal/serviceconfig/objectstore" "queryorchestration/internal/serviceconfig/queue" "github.com/google/uuid" ) type Create struct { - ClientID string - Location document.Location - Bucket string - Hash string + Bucket string + Key objectstore.BucketKey + Hash string } func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) { @@ -48,15 +47,15 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) { } type createDocumentParams struct { - ID *uuid.UUID - ClientID string - Hash string - Location document.Location + ID *uuid.UUID + Hash string + Bucket string + Key objectstore.BucketKey } func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) { idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{ - Clientid: doc.ClientID, + Clientid: doc.Key.ClientID, Hash: doc.Hash, }) var docID *uuid.UUID @@ -67,10 +66,10 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu } return &createDocumentParams{ - ID: docID, - ClientID: doc.ClientID, - Hash: doc.Hash, - Location: doc.Location, + ID: docID, + Hash: doc.Hash, + Key: doc.Key, + Bucket: doc.Bucket, }, nil } @@ -81,25 +80,25 @@ func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams var dbid uuid.UUID if params.ID == nil { createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{ - Clientid: params.ClientID, + Clientid: params.Key.ClientID, Hash: params.Hash, }) if err != nil { return err } - slog.Debug("document created", "id", createid.String(), "client", params.ClientID) + slog.Debug("document created", "id", createid.String(), "client", params.Key.ClientID) dbid = createid } else { - slog.Debug("document exists", "id", params.ID.String(), "client", params.ClientID) + slog.Debug("document exists", "id", params.ID.String(), "client", params.Key.ClientID) dbid = *params.ID } err := s.cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{ Documentid: dbid, - Bucket: params.Location.Bucket, - Key: params.Location.Key, + Bucket: params.Bucket, + Key: params.Key.String(), }) if err != nil { return err diff --git a/internal/document/init/create_test.go b/internal/document/init/create_test.go index d04dc953..535a2a61 100644 --- a/internal/document/init/create_test.go +++ b/internal/document/init/create_test.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "testing" + "time" "queryorchestration/internal/database/repository" "queryorchestration/internal/document" + "queryorchestration/internal/serviceconfig/objectstore" queuemock "queryorchestration/mocks/queue" "github.com/stretchr/testify/require" @@ -39,9 +41,12 @@ func TestCreate(t *testing.T) { ClientID: clientId, Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows( @@ -53,7 +58,7 @@ func TestCreate(t *testing.T) { pgxmock.NewRows([]string{"id"}). AddRow(doc.ID), ) - pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, location.Bucket, location.Key). + pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() pool.ExpectQuery("name: GetDocumentSummary :one").WithArgs(doc.ID). @@ -81,9 +86,9 @@ func TestCreate(t *testing.T) { Return(&sqs.SendMessageOutput{}, nil) id, err := svc.Create(ctx, &Create{ - ClientID: doc.ClientID, - Location: location, - Hash: doc.Hash, + Hash: doc.Hash, + Key: key, + Bucket: bucket, }) require.NoError(t, err) assert.Equal(t, doc.ID, id) @@ -104,9 +109,12 @@ func TestGetCreateParams(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows( @@ -114,15 +122,15 @@ func TestGetCreateParams(t *testing.T) { ) params, err := svc.getCreateParams(ctx, &Create{ - ClientID: doc.ClientID, - Location: location, - Hash: doc.Hash, + Hash: doc.Hash, + Bucket: bucket, + Key: key, }) require.NoError(t, err) assert.Equal(t, &createDocumentParams{ - ClientID: doc.ClientID, - Hash: doc.Hash, - Location: location, + Hash: doc.Hash, + Bucket: bucket, + Key: key, }, params) } @@ -142,9 +150,12 @@ func TestGetCreateParamsExisting(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID).WillReturnRows( @@ -153,17 +164,17 @@ func TestGetCreateParamsExisting(t *testing.T) { ) params, err := svc.getCreateParams(ctx, &Create{ - ClientID: doc.ClientID, - Location: location, - Hash: doc.Hash, + Bucket: bucket, + Key: key, + Hash: doc.Hash, }) require.NoError(t, err) dbid := doc.ID assert.Equal(t, &createDocumentParams{ - ID: &dbid, - ClientID: doc.ClientID, - Hash: doc.Hash, - Location: location, + ID: &dbid, + Hash: doc.Hash, + Bucket: bucket, + Key: key, }, params) } @@ -182,17 +193,20 @@ func TestGetCreateParamsCurrentDocErr(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectQuery("name: GetDocumentIDByHash :one").WithArgs(doc.Hash, doc.ClientID). WillReturnError(errors.New("db err")) _, err = svc.getCreateParams(ctx, &Create{ - ClientID: doc.ClientID, - Location: location, + Bucket: bucket, + Key: key, }) assert.Error(t, err) } @@ -213,9 +227,12 @@ func TestSubmitCreate(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectBegin() @@ -224,14 +241,14 @@ func TestSubmitCreate(t *testing.T) { pgxmock.NewRows([]string{"id"}). AddRow(doc.ID), ) - pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, location.Bucket, location.Key). + pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() id, err := svc.submitCreate(ctx, &createDocumentParams{ - ClientID: doc.ClientID, - Location: location, - Hash: doc.Hash, + Hash: doc.Hash, + Bucket: bucket, + Key: key, }) require.NoError(t, err) assert.Equal(t, doc.ID, id) @@ -253,22 +270,25 @@ func TestSubmitCreateExists(t *testing.T) { ClientID: "hello", Hash: "example_hash", } - location := document.Location{ - Bucket: "example_bucket", - Key: "/i/am/here", + bucket := "example_bucket" + key := objectstore.BucketKey{ + ClientID: doc.ClientID, + CreatedAt: time.Now().UTC(), + EntityID: uuid.New(), + Location: objectstore.Import, } pool.ExpectBegin() - pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, location.Bucket, location.Key). + pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(doc.ID, bucket, key.String()). WillReturnResult(pgxmock.NewResult("", 1)) pool.ExpectCommit() docid := doc.ID id, err := svc.submitCreate(ctx, &createDocumentParams{ - ID: &docid, - ClientID: doc.ClientID, - Location: location, - Hash: doc.Hash, + ID: &docid, + Hash: doc.Hash, + Bucket: bucket, + Key: key, }) require.NoError(t, err) assert.Equal(t, doc.ID, id) diff --git a/internal/document/service.go b/internal/document/service.go index 25524955..093a9979 100644 --- a/internal/document/service.go +++ b/internal/document/service.go @@ -6,11 +6,6 @@ import ( "github.com/google/uuid" ) -type Location struct { - Bucket string - Key string -} - type DocumentSummary struct { ID uuid.UUID ClientID string diff --git a/internal/document/store/process.go b/internal/document/store/process.go index 825adf76..2230f3ad 100644 --- a/internal/document/store/process.go +++ b/internal/document/store/process.go @@ -40,18 +40,16 @@ func (s *Service) Process(ctx context.Context, params Params) error { case objectstore.Import: queueParams.QueueURL = s.cfg.GetDocInitURL() queueParams.Body = docinitrunner.Body{ - Bucket: params.Bucket, - Key: params.Key, - Hash: params.Hash, - ClientID: key.ClientID, + Bucket: params.Bucket, + Key: params.Key, + Hash: params.Hash, } case objectstore.TextTextract: queueParams.QueueURL = s.cfg.GetDocumentTextProcessURL() queueParams.Body = doctextprocessrunner.Body{ - Bucket: params.Bucket, - Key: params.Key, - Hash: params.Hash, - ClientID: key.ClientID, + Bucket: params.Bucket, + Key: params.Key, + Hash: params.Hash, } default: slog.Info("unsupported key", "key", params.Key) diff --git a/internal/document/store/process_test.go b/internal/document/store/process_test.go index cb17386e..08729b63 100644 --- a/internal/document/store/process_test.go +++ b/internal/document/store/process_test.go @@ -60,7 +60,6 @@ func TestProcess(t *testing.T) { location := objectstore.BucketKey{ ClientID: "7db16095-9155-47d4-8004-b3b3ead93c83", Location: objectstore.Import, - Filename: "aaa", CreatedAt: time.Now().UTC(), } err := svc.Process(ctx, Params{ @@ -83,7 +82,6 @@ func TestProcess(t *testing.T) { location := objectstore.BucketKey{ ClientID: "7db16095-9155-47d4-8004-b3b3ead93c83", Location: objectstore.TextTextract, - Filename: "aaa", CreatedAt: time.Now().UTC(), } err := svc.Process(ctx, Params{ diff --git a/internal/document/text/extract_test.go b/internal/document/text/extract_test.go index ef5c1057..e8f3d794 100644 --- a/internal/document/text/extract_test.go +++ b/internal/document/text/extract_test.go @@ -6,7 +6,6 @@ import ( "testing" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/aws" "queryorchestration/internal/serviceconfig/objectstore" @@ -57,10 +56,8 @@ func TestCreate(t *testing.T) { t.Run("valid", func(t *testing.T) { docId := uuid.New() cleanId := uuid.New() - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", - } + bucket := "bucket_name" + key := "/i/am/here" clientId := "AAA" hash := "example" @@ -75,7 +72,7 @@ func TestCreate(t *testing.T) { } pool.ExpectQuery("name: GetCleanEntryByDocId :one").WithArgs(docId).WillReturnRows( pgxmock.NewRows([]string{"id", "documentId", "bucket", "key", "version", "hash", "mimetype", "fail", "externalClientId"}). - AddRow(cleanId, docId, &inloc.Bucket, &inloc.Key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, clientId), + AddRow(cleanId, docId, &bucket, &key, int64(1), &hash, dbmimetype, repository.NullCleanfailtype{}, clientId), ) jobId := "hello" diff --git a/internal/document/text/processExtract.go b/internal/document/text/processExtract.go index 2fe67fc2..c92ffa54 100644 --- a/internal/document/text/processExtract.go +++ b/internal/document/text/processExtract.go @@ -18,10 +18,9 @@ import ( ) type ProcessParams struct { - Bucket string - Key objectstore.BucketKey - Hash string - ClientID string + Bucket string + Key objectstore.BucketKey + Hash string } type Page string @@ -97,11 +96,12 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep var textId uuid.UUID if existingId == uuid.Nil || errors.Is(err, sql.ErrNoRows) { + createdAt := time.Now().UTC() key := objectstore.BucketKey{ - ClientID: params.ClientID, + ClientID: params.Key.ClientID, Location: objectstore.TextOut, - CreatedAt: time.Now().UTC(), - Filename: trigger.ID.String(), + CreatedAt: createdAt, + EntityID: trigger.ID, } keyStr := key.String() @@ -141,12 +141,7 @@ func (s *Service) storeProcess(ctx context.Context, text io.Reader, trigger *rep } func (s *Service) Process(ctx context.Context, params *ProcessParams) error { - triggerId, err := s.getTriggerIdFromKey(params.Key) - if err != nil { - return err - } - - trigger, err := s.cfg.GetDBQueries().GetDocumentTextTrigger(ctx, triggerId) + trigger, err := s.cfg.GetDBQueries().GetDocumentTextTrigger(ctx, params.Key.EntityID) if err != nil { return err } diff --git a/internal/document/text/processExtract_test.go b/internal/document/text/processExtract_test.go index 3e778a3e..3accfee9 100644 --- a/internal/document/text/processExtract_test.go +++ b/internal/document/text/processExtract_test.go @@ -87,7 +87,7 @@ func TestProcess(t *testing.T) { key := objectstore.BucketKey{ ClientID: "my_client", Location: objectstore.TextTextract, - Filename: triggerId.String(), + EntityID: triggerId, } err = svc.Process(ctx, &ProcessParams{ Bucket: "bucket", @@ -147,7 +147,7 @@ func TestStoreTrigger(t *testing.T) { ClientID: clientId, Location: objectstore.TextOut, CreatedAt: time.Now().UTC(), - Filename: triggerId.String(), + EntityID: triggerId, } hash := `"09644372e99020106946045c6fd2d70b"` pool.ExpectBegin() @@ -180,9 +180,8 @@ func TestStoreTrigger(t *testing.T) { ID: triggerId, Cleanid: cleanId, }, &ProcessParams{ - Key: key, - Bucket: bucket, - ClientID: clientId, + Key: key, + Bucket: bucket, }) require.NoError(t, err) }) @@ -226,7 +225,7 @@ func TestProcessTrigger(t *testing.T) { key := objectstore.BucketKey{ ClientID: clientId, Location: objectstore.TextOut, - Filename: objectstore.BucketFilename(triggerId.String()), + EntityID: triggerId, CreatedAt: time.Now().UTC(), } hash := `"5d41402abc4b2a76b9719d911017c592"` @@ -273,10 +272,9 @@ func TestProcessTrigger(t *testing.T) { ID: triggerId, Cleanid: cleanId, }, &ProcessParams{ - ClientID: clientId, - Bucket: bucket, - Key: key, - Hash: hash, + Bucket: bucket, + Key: key, + Hash: hash, }) require.NoError(t, err) } diff --git a/internal/document/text/triggerExtract.go b/internal/document/text/triggerExtract.go index a557f6af..130fd746 100644 --- a/internal/document/text/triggerExtract.go +++ b/internal/document/text/triggerExtract.go @@ -11,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/textract" "github.com/aws/aws-sdk-go-v2/service/textract/types" - "github.com/google/uuid" ) func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentcleanentry) error { @@ -26,21 +25,16 @@ func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentc return err } - jobTag := triggerId.String() - - filename, err := s.getOutputFilename(triggerId) - if err != nil { - return err - } - key := objectstore.BucketKey{ CreatedAt: time.Now().UTC(), Location: objectstore.TextTextract, ClientID: clean.Clientid, - Filename: filename, + EntityID: triggerId, } keyStr := key.String() + jobTag := triggerId.String() + res, err := s.cfg.GetTextractClient().StartDocumentTextDetection(ctx, &textract.StartDocumentTextDetectionInput{ DocumentLocation: &types.DocumentLocation{ S3Object: &types.S3Object{ @@ -71,20 +65,3 @@ func (s *Service) triggerExtract(ctx context.Context, clean *repository.Currentc return nil }) } - -func (s *Service) getOutputFilename(triggerId uuid.UUID) (string, error) { - if triggerId == uuid.Nil { - return "", errors.New("trigger id required") - } - - return triggerId.String(), nil -} - -func (s *Service) getTriggerIdFromKey(key objectstore.BucketKey) (uuid.UUID, error) { - triggerId, err := uuid.Parse(key.Filename) - if err != nil { - return uuid.Nil, err - } - - return triggerId, nil -} diff --git a/internal/document/text/triggerExtract_test.go b/internal/document/text/triggerExtract_test.go index 8026297c..7a0ad4fc 100644 --- a/internal/document/text/triggerExtract_test.go +++ b/internal/document/text/triggerExtract_test.go @@ -5,13 +5,10 @@ import ( "testing" "queryorchestration/internal/database/repository" - "queryorchestration/internal/document" - "queryorchestration/internal/serviceconfig/objectstore" objectstoremock "queryorchestration/mocks/objectstore" textractmock "queryorchestration/mocks/textract" "github.com/aws/aws-sdk-go-v2/service/textract" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -38,10 +35,8 @@ func TestTriggerExtract(t *testing.T) { t.Run("successful clean", func(t *testing.T) { cleanId := uuid.New() - inloc := document.Location{ - Bucket: "bucket_name", - Key: "/i/am/here", - } + bucket := "bucket_name" + key := "/i/am/here" hash := "hhasshhh" jobId := "hello" @@ -70,8 +65,8 @@ func TestTriggerExtract(t *testing.T) { err = svc.triggerExtract(ctx, &repository.Currentcleanentry{ ID: cleanId, Clientid: "AA", - Key: &inloc.Key, - Bucket: &inloc.Bucket, + Key: &key, + Bucket: &bucket, Hash: &hash, Mimetype: repository.NullCleanmimetype{ Valid: true, @@ -81,33 +76,3 @@ func TestTriggerExtract(t *testing.T) { require.NoError(t, err) }) } - -func TestGetOutputFilename(t *testing.T) { - svc := Service{} - - _, err := svc.getOutputFilename(uuid.Nil) - assert.EqualError(t, err, "trigger id required") - - triggerId := uuid.New() - key, err := svc.getOutputFilename(triggerId) - assert.NoError(t, err) - assert.Equal(t, triggerId.String(), key) -} - -func TestGetTriggerIdFromKey(t *testing.T) { - svc := Service{} - - _, err := svc.getTriggerIdFromKey(objectstore.BucketKey{}) - assert.EqualError(t, err, "invalid UUID length: 0") - - _, err = svc.getTriggerIdFromKey(objectstore.BucketKey{ - Filename: "aa", - }) - assert.EqualError(t, err, "invalid UUID length: 2") - - triggerId, err := svc.getTriggerIdFromKey(objectstore.BucketKey{ - Filename: "6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d", - }) - assert.NoError(t, err) - assert.Equal(t, "6ac21b2b-f36c-4ae7-a815-307d4a9bfe4d", triggerId.String()) -} diff --git a/internal/serviceconfig/objectstore/bucketkey.go b/internal/serviceconfig/objectstore/bucketkey.go new file mode 100644 index 00000000..ec92b62d --- /dev/null +++ b/internal/serviceconfig/objectstore/bucketkey.go @@ -0,0 +1,175 @@ +package objectstore + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + "time" + + "queryorchestration/internal/client" + + "github.com/google/uuid" +) + +type BucketLocation string +type BucketKey struct { + ClientID string + Location BucketLocation + CreatedAt time.Time + Part *uint8 + EntityID uuid.UUID + FileType *string +} + +const ( + Import BucketLocation = "import" + TextTextract BucketLocation = "text/textract" + TextOut BucketLocation = "text/out" + Export BucketLocation = "export" +) + +var timeFormat = "2006-01-02T150405Z" +var dateFormat = "20060102" + +func (c *BucketKey) String() string { + dateStr := c.CreatedAt.Format(dateFormat) + partStr := "" + if c.Part != nil { + partStr = fmt.Sprintf("/%d", *c.Part) + } + + return fmt.Sprintf("%s/%s/%s%s/%s", c.ClientID, c.Location, dateStr, partStr, c.Filename()) +} + +func (c *BucketKey) Filename() string { + location := strings.ReplaceAll(string(c.Location), "/", "-") + name := fmt.Sprintf("%s_%s_%s_%s", c.CreatedAt.Format(timeFormat), c.ClientID, location, c.EntityID) + if c.FileType != nil { + name = fmt.Sprintf("%s.%s", name, *c.FileType) + } + + return name +} + +func (c *BucketKey) parseFilename(name string) error { + parts := strings.Split(name, ".") + if len(parts) != 1 && len(parts) != 2 { + return errors.New("incorrect amount of dots") + } else if len(parts) == 2 { + c.parseFileType(parts[1]) + } + + metadata := strings.Split(parts[0], "_") + if len(metadata) != 4 { + return errors.New("incorrect amount of metadata elements") + } + + err := c.parseTime(metadata[0]) + if err != nil { + return err + } + + err = c.parseClientID(metadata[1]) + if err != nil { + return err + } + + err = c.parseLocation(metadata[2]) + if err != nil { + return err + } + + err = c.parseEntityID(metadata[3]) + if err != nil { + return err + } + + return nil +} +func (c *BucketKey) parseFileType(name string) { + c.FileType = &name +} +func (c *BucketKey) parseTime(name string) error { + parsedTime, err := time.Parse(timeFormat, name) + if err != nil { + return fmt.Errorf("failed to parse date: %w", err) + } + c.CreatedAt = parsedTime + + return nil +} +func (c *BucketKey) parseClientID(name string) error { + c.ClientID = name + pattern := fmt.Sprintf(`^(%s)$`, client.CLIENT_ID_REGEX) + reg := regexp.MustCompile(pattern) + matches := reg.FindStringSubmatch(c.ClientID) + if len(matches) != 2 { + return errors.New("invalid client id") + } + + return nil +} +func (c *BucketKey) parseLocation(name string) error { + c.Location = BucketLocation(strings.ReplaceAll(name, "-", "/")) + pattern := fmt.Sprintf(`^(%s|%s|%s|%s)$`, Import, TextTextract, TextOut, Export) + reg := regexp.MustCompile(pattern) + matches := reg.FindStringSubmatch(string(c.Location)) + if len(matches) != 2 { + return errors.New("invalid location") + } + + return nil +} +func (c *BucketKey) parseEntityID(name string) error { + entityId, err := uuid.Parse(name) + if err != nil { + return err + } + c.EntityID = entityId + + return nil +} + +func ParseBucketKey(key string) (BucketKey, error) { + locations := "[a-z]+/?[a-z]+?" + pattern := fmt.Sprintf(`^(%s)/(%s)/([0-9]{8})/(.+)$`, client.CLIENT_ID_REGEX, locations) + reg := regexp.MustCompile(pattern) + matches := reg.FindStringSubmatch(key) + if len(matches) != 5 { + return BucketKey{}, errors.New("does not match expectation") + } + + formattedKey := BucketKey{} + + filename := `.+` + reg = regexp.MustCompile(fmt.Sprintf(`^([0-9]+)/(%s)$`, filename)) + subMatches := reg.FindStringSubmatch(matches[4]) + if len(subMatches) == 3 { + part, err := strconv.ParseUint(subMatches[1], 10, 8) + if err != nil { + return BucketKey{}, err + } + safePart := uint8(part) + formattedKey.Part = &safePart + + err = formattedKey.parseFilename(subMatches[2]) + if err != nil { + return BucketKey{}, err + } + } else { + reg = regexp.MustCompile(fmt.Sprintf(`^(%s)$`, filename)) + subMatches := reg.FindStringSubmatch(matches[4]) + if len(subMatches) != 2 { + return BucketKey{}, errors.New("does not match expectation") + } + + err := formattedKey.parseFilename(subMatches[1]) + if err != nil { + return BucketKey{}, err + } + } + + return formattedKey, nil +} diff --git a/internal/serviceconfig/objectstore/bucketkey_test.go b/internal/serviceconfig/objectstore/bucketkey_test.go new file mode 100644 index 00000000..b79e1a59 --- /dev/null +++ b/internal/serviceconfig/objectstore/bucketkey_test.go @@ -0,0 +1,178 @@ +package objectstore + +import ( + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +func TestGetBucketKey(t *testing.T) { + key := BucketKey{} + assert.Equal(t, "//00010101/0001-01-01T000000Z___00000000-0000-0000-0000-000000000000", key.String()) + key.CreatedAt = time.Date(2025, time.March, 31, 4, 8, 16, 222, time.UTC) + assert.Equal(t, "//20250331/2025-03-31T040816Z___00000000-0000-0000-0000-000000000000", key.String()) + key.ClientID = "AAA" + assert.Equal(t, "AAA//20250331/2025-03-31T040816Z_AAA__00000000-0000-0000-0000-000000000000", key.String()) + key.EntityID = uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40") + assert.Equal(t, "AAA//20250331/2025-03-31T040816Z_AAA__b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + key.Location = Import + assert.Equal(t, "AAA/import/20250331/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + part := uint8(3) + key.Part = &part + assert.Equal(t, "AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + key.Location = TextTextract + assert.Equal(t, "AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + key.Location = TextOut + assert.Equal(t, "AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + key.Location = Export + assert.Equal(t, "AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40", key.String()) + filetype := "csv" + key.FileType = &filetype + assert.Equal(t, "AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40.csv", key.String()) +} +func TestParseBucketKey(t *testing.T) { + _, err := ParseBucketKey("") + assert.Error(t, err) + + _, err = ParseBucketKey("AAA") + assert.Error(t, err) + + _, err = ParseBucketKey("//00010101/") + assert.Error(t, err) + + _, err = ParseBucketKey("//20250331/") + assert.Error(t, err) + + _, err = ParseBucketKey("AAA//20250331/") + assert.Error(t, err) + + _, err = ParseBucketKey("AAA//20250331/coolfile") + assert.Error(t, err) + + key, err := ParseBucketKey("AAA/import/20250331/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + }, key) + + key, err = ParseBucketKey("AAA/text/textract/20250331/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: TextTextract, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC), + }, key) + + key, err = ParseBucketKey("AAA/text/out/20250331/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: TextOut, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + }, key) + + key, err = ParseBucketKey("AAA/export/20250331/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: Export, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + }, key) + + part := uint8(3) + key, err = ParseBucketKey("AAA/import/20250331/3/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + key, err = ParseBucketKey("AAA/text/textract/20250331/3/2025-03-31T040816Z_AAA_text-textract_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: TextTextract, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + key, err = ParseBucketKey("AAA/text/out/20250331/3/2025-03-31T040816Z_AAA_text-out_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: TextOut, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + key, err = ParseBucketKey("AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + assert.NoError(t, err) + assert.EqualExportedValues(t, BucketKey{ + ClientID: "AAA", + Location: Export, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), + Part: &part, + }, key) + + _, err = ParseBucketKey("AAA/import/99999999/2025-03-31T990816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + assert.Error(t, err) + + _, err = ParseBucketKey("AAA/import/20250331/99999999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + assert.Error(t, err) + + _, err = ParseBucketKey("AAA/import/20250331/999/2025-03-31T040816Z_AAA_import_b745910f-f529-43c5-87e1-25629d46ef40") + assert.Error(t, err) + + _, err = ParseBucketKey("****AAA/export/20250331/3/2025-03-31T040816Z_AAA_export_b745910f-f529-43c5-87e1-25629d46ef40") + assert.Error(t, err) +} + +func TestParseFilename(t *testing.T) { + key := BucketKey{} + + err := key.parseFilename("") + assert.EqualError(t, err, "incorrect amount of metadata elements") + + err = key.parseFilename("a.a") + assert.EqualError(t, err, "incorrect amount of metadata elements") + + err = key.parseFilename("a.a.a") + assert.EqualError(t, err, "incorrect amount of dots") + + err = key.parseFilename("a_a_a_a.a") + assert.EqualError(t, err, "failed to parse date: parsing time \"a\" as \"2006-01-02T150405Z\": cannot parse \"a\" as \"2006\"") + + err = key.parseFilename("2025-03-31T040816Z_*_a_a.a") + assert.EqualError(t, err, "invalid client id") + + err = key.parseFilename("2025-03-31T040816Z_a_a_a.a") + assert.EqualError(t, err, "invalid location") + + err = key.parseFilename("2025-03-31T040816Z_a_import_a.a") + assert.EqualError(t, err, "invalid UUID length: 1") + + err = key.parseFilename("2025-03-31T040816Z_a_import_b745910f-f529-43c5-87e1-25629d46ef40.a") + assert.NoError(t, err) + filetype := "a" + assert.EqualExportedValues(t, BucketKey{ + CreatedAt: time.Date(2025, time.March, 32, 4, 8, 16, 0, time.UTC), + ClientID: "a", + Location: Import, + EntityID: uuid.MustParse("b745910f-f529-43c5-87e1-25629d46ef40"), + FileType: &filetype, + }, key) +} diff --git a/internal/serviceconfig/objectstore/config.go b/internal/serviceconfig/objectstore/config.go index 1b7ac01c..6bb2c633 100644 --- a/internal/serviceconfig/objectstore/config.go +++ b/internal/serviceconfig/objectstore/config.go @@ -4,15 +4,11 @@ import ( "context" "crypto/md5" "encoding/hex" - "errors" "fmt" "io" "log/slog" - "regexp" - "strconv" "time" - "queryorchestration/internal/client" awsc "queryorchestration/internal/serviceconfig/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -144,75 +140,3 @@ func (c *ObjectStoreConfig) CalculateETag(ctx context.Context, doc io.Reader) (s return fmt.Sprintf(`"%s-%d"`, hex.EncodeToString(finalHash.Sum(nil)), len(partHashes)), nil } - -type BucketLocation string -type BucketFilename = string -type BucketKey struct { - ClientID string - Location BucketLocation - CreatedAt time.Time - Part *uint8 - Filename BucketFilename -} - -const ( - Import BucketLocation = "import" - TextTextract BucketLocation = "text/textract" - TextOut BucketLocation = "text/out" - Export BucketLocation = "export" -) - -func (c *BucketKey) String() string { - dateStr := c.CreatedAt.Format("20060102") - partStr := "" - if c.Part != nil { - partStr = fmt.Sprintf("/%d", *c.Part) - } - - return fmt.Sprintf("%s/%s/%s%s/%s", c.ClientID, c.Location, dateStr, partStr, c.Filename) -} - -func ParseBucketKey(key string) (BucketKey, error) { - locations := fmt.Sprintf("%s|%s|%s|%s", Import, TextTextract, TextOut, Export) - pattern := fmt.Sprintf(`^(%s)/(%s)/([0-9]{8})/(.+)$`, client.CLIENT_ID_REGEX, locations) - reg := regexp.MustCompile(pattern) - matches := reg.FindStringSubmatch(key) - if len(matches) != 5 { - return BucketKey{}, errors.New("does not match expectation") - } - - formattedKey := BucketKey{} - - formattedKey.ClientID = matches[1] - formattedKey.Location = BucketLocation(matches[2]) - - parsedTime, err := time.Parse("20060102", matches[3]) - if err != nil { - return BucketKey{}, fmt.Errorf("failed to parse date: %w", err) - } - formattedKey.CreatedAt = parsedTime - - filename := `[a-zA-Z0-9\_\.\-]+` - reg = regexp.MustCompile(fmt.Sprintf(`^([0-9]+)/(%s)$`, filename)) - subMatches := reg.FindStringSubmatch(matches[4]) - if len(subMatches) == 3 { - part, err := strconv.ParseUint(subMatches[1], 10, 8) - if err != nil { - return BucketKey{}, err - } - safePart := uint8(part) - formattedKey.Part = &safePart - - formattedKey.Filename = subMatches[2] - } else { - reg = regexp.MustCompile(fmt.Sprintf(`^(%s)$`, filename)) - subMatches := reg.FindStringSubmatch(matches[4]) - if len(subMatches) != 2 { - return BucketKey{}, errors.New("does not match expectation") - } - - formattedKey.Filename = subMatches[1] - } - - return formattedKey, nil -} diff --git a/internal/serviceconfig/objectstore/config_test.go b/internal/serviceconfig/objectstore/config_test.go index 42f44cf1..5385da39 100644 --- a/internal/serviceconfig/objectstore/config_test.go +++ b/internal/serviceconfig/objectstore/config_test.go @@ -115,160 +115,3 @@ func TestCalculateETag(t *testing.T) { assert.Equal(t, *res.ETag, hash) } - -func TestGetBucketKey(t *testing.T) { - key := objectstore.BucketKey{} - assert.Equal(t, "//00010101/", key.String()) - key.CreatedAt = time.Date(2025, time.March, 31, 0, 0, 0, 0, time.UTC) - assert.Equal(t, "//20250331/", key.String()) - key.ClientID = "AAA" - assert.Equal(t, "AAA//20250331/", key.String()) - key.Filename = "coolfile" - assert.Equal(t, "AAA//20250331/coolfile", key.String()) - key.Location = objectstore.Import - assert.Equal(t, "AAA/import/20250331/coolfile", key.String()) - part := uint8(3) - key.Part = &part - assert.Equal(t, "AAA/import/20250331/3/coolfile", key.String()) - key.Location = objectstore.TextTextract - assert.Equal(t, "AAA/text/textract/20250331/3/coolfile", key.String()) - key.Location = objectstore.TextOut - assert.Equal(t, "AAA/text/out/20250331/3/coolfile", key.String()) - key.Location = objectstore.Export - assert.Equal(t, "AAA/export/20250331/3/coolfile", key.String()) -} -func TestParseBucketKey(t *testing.T) { - _, err := objectstore.ParseBucketKey("") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("//00010101/") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("//20250331/") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA//20250331/") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA//20250331/coolfile") - assert.Error(t, err) - - key, err := objectstore.ParseBucketKey("AAA/import/20250331/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Import, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - key, err = objectstore.ParseBucketKey("AAA/text/textract/20250331/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.TextTextract, - Filename: "coolfile", - CreatedAt: time.Date(2025, 2, 31, 0, 0, 0, 0, time.UTC), - }, key) - - key, err = objectstore.ParseBucketKey("AAA/text/out/20250331/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.TextOut, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - key, err = objectstore.ParseBucketKey("AAA/export/20250331/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Export, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - part := uint8(3) - key, err = objectstore.ParseBucketKey("AAA/import/20250331/3/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Import, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - Part: &part, - }, key) - - key, err = objectstore.ParseBucketKey("AAA/text/textract/20250331/3/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.TextTextract, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - Part: &part, - }, key) - - key, err = objectstore.ParseBucketKey("AAA/text/out/20250331/3/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.TextOut, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - Part: &part, - }, key) - - key, err = objectstore.ParseBucketKey("AAA/export/20250331/3/coolfile") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Export, - Filename: "coolfile", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - Part: &part, - }, key) - - key, err = objectstore.ParseBucketKey("AAA/import/20250331/cool-file") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Import, - Filename: "cool-file", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - key, err = objectstore.ParseBucketKey("AAA/import/20250331/cool_file") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Import, - Filename: "cool_file", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - key, err = objectstore.ParseBucketKey("AAA/import/20250331/coolfile365") - assert.NoError(t, err) - assert.EqualExportedValues(t, objectstore.BucketKey{ - ClientID: "AAA", - Location: objectstore.Import, - Filename: "coolfile365", - CreatedAt: time.Date(2025, 3, 31, 0, 0, 0, 0, time.UTC), - }, key) - - _, err = objectstore.ParseBucketKey("AAA/import/99999999/coolfile") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA/import/20250331/99999999/coolfile") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA/import/20250331/999/coolfile") - assert.Error(t, err) - - _, err = objectstore.ParseBucketKey("AAA/import/20250331/1/cool**file") - assert.Error(t, err) -} diff --git a/internal/test/ecosystem_test.go b/internal/test/ecosystem_test.go index 03cf5844..df9f3bbe 100644 --- a/internal/test/ecosystem_test.go +++ b/internal/test/ecosystem_test.go @@ -180,13 +180,11 @@ func TestWaitForTextractMockEndpoint(t *testing.T) { ClientID: clientId, CreatedAt: time.Now().UTC(), Location: objectstore.Import, - Filename: "input", } outputKey := objectstore.BucketKey{ ClientID: clientId, CreatedAt: time.Now().UTC(), Location: objectstore.TextTextract, - Filename: "output", } e := CreateStartDocTextDetectionExpectation(t, deps.MockServer, StartDocTextDetectionExpectationParams{ diff --git a/internal/test/objectstore_test.go b/internal/test/objectstore_test.go index b10cb182..150543d6 100644 --- a/internal/test/objectstore_test.go +++ b/internal/test/objectstore_test.go @@ -66,7 +66,6 @@ func TestPutObject(t *testing.T) { file := strings.NewReader("hello") key := objectstore.BucketKey{ ClientID: "byebye", - Filename: "hello", } mockS3.EXPECT(). diff --git a/test/process_test.go b/test/process_test.go index d255915a..a280a44b 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -14,6 +14,7 @@ import ( queryapitest "queryorchestration/internal/test/queryAPI" queryapi "queryorchestration/pkg/queryAPI" + "github.com/google/uuid" "github.com/oapi-codegen/runtime/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,28 +65,23 @@ func TestProcess(t *testing.T) { queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC) - filename := "objectname" + importKey := objectstore.BucketKey{ + ClientID: client.Id, + EntityID: uuid.New(), + Location: objectstore.Import, + CreatedAt: time.Now().UTC(), + } expectation := test.CreateStartDocTextDetectionExpectation(t, net.Dependencies.MockServer, test.StartDocTextDetectionExpectationParams{ Bucket: net.Dependencies.BucketName, JobID: "textractId", - Key: objectstore.BucketKey{ - ClientID: client.Id, - Filename: filename, - Location: objectstore.Import, - CreatedAt: time.Now().UTC(), - }, + Key: importKey, }) body := strings.NewReader(pdfHelloWorld) test.PutObject(t, ctx, cfg, test.PutObjectParams{ Bucket: net.Dependencies.BucketName, File: body, - Key: objectstore.BucketKey{ - ClientID: client.Id, - Filename: filename, - Location: objectstore.Import, - CreatedAt: time.Now().UTC(), - }, + Key: importKey, }) queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED)