jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/doctextstructure (pull request #56)

Browse Source 
DocTextRunner Structure

* firstround

* shorttests
This commit is contained in:
Michael McGuinness
2025-02-07 14:15:06 +00:00
parent 90353d8161
commit 24a038ec3d
41 changed files with 929 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/clean/create.go
+2 -2
View File
@@ -2,8 +2,8 @@ package documentclean
import (
"context"
doctextrunner "queryorchestration/api/docTextRunner"
"queryorchestration/internal/database"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/serviceconfig/queue"
"github.com/google/uuid"
@@ -33,7 +33,7 @@ func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
func (s *Service) informClean(ctx context.Context, id uuid.UUID) error {
err := s.cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: s.cfg.GetDocumentTextURL(),
Body: documenttext.Create{
Body: doctextrunner.Create{
ID: id,
},
})
internal/document/text/create.go
+45
View File
@@ -0,0 +1,45 @@
package documenttext
import (
"context"
querysyncrunner "queryorchestration/api/querySyncRunner"
"queryorchestration/internal/database"
"queryorchestration/internal/serviceconfig/queue"
"github.com/google/uuid"
)
func (s *Service) Create(ctx context.Context, id uuid.UUID) error {
isextracted, err := s.cfg.GetDBQueries().IsDocumentTextExtracted(ctx, database.MustToDBUUID(id))
if err != nil {
return err
}
if !isextracted {
err = s.extract(ctx, id)
if err != nil {
return err
}
}
err = s.informExtraction(ctx, id)
if err != nil {
return err
}
return nil
}
func (s *Service) informExtraction(ctx context.Context, id uuid.UUID) error {
err := s.cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: s.cfg.GetQuerySyncURL(),
Body: querysyncrunner.Create{
ID: id,
},
})
if err != nil {
return err
}
return nil
}
internal/document/text/create_test.go
+105
View File
@@ -0,0 +1,105 @@
package documenttext
import (
"context"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/querysync"
queuemock "queryorchestration/mocks/queue"
"testing"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
type DocTextConfig struct {
serviceconfig.BaseConfig
objectstore.ObjectStoreConfig
querysync.QuerySyncConfig
}
func TestCreate(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
mockSQS := queuemock.NewMockSQSClient(t)
cfg := &DocTextConfig{}
cfg.QueueClient = mockSQS
cfg.QuerySyncURL = "/i/am/here"
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
cfg: cfg,
svc: &Services{
Document: document.New(cfg),
},
}
id := uuid.New()
inloc := document.Location{
Bucket: "bucket_name",
Key: "/i/am/here",
}
pool.ExpectQuery("name: IsDocumentTextExtracted :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"isextracted"}).
AddRow(false),
)
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
WillReturnResult(pgxmock.NewResult("", 1))
mockSQS.EXPECT().
SendMessage(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
}),
mock.Anything,
).
Return(&sqs.SendMessageOutput{}, nil)
err = svc.Create(ctx, id)
assert.NoError(t, err)
}
func TestInformClean(t *testing.T) {
ctx := context.Background()
mockSQS := queuemock.NewMockSQSClient(t)
cfg := &DocTextConfig{}
cfg.QueueClient = mockSQS
cfg.QuerySyncURL = "/i/am/here"
svc := Service{
cfg: cfg,
}
id := uuid.New()
mockSQS.EXPECT().
SendMessage(
mock.Anything,
mock.MatchedBy(func(in *sqs.SendMessageInput) bool {
return *in.QueueUrl == cfg.QuerySyncURL && *in.MessageBody == fmt.Sprintf("{\"id\":\"%s\"}", id)
}),
mock.Anything,
).
Return(&sqs.SendMessageOutput{}, nil)
err := svc.informExtraction(ctx, id)
assert.NoError(t, err)
}
internal/document/text/extract.go
+54
View File
@@ -0,0 +1,54 @@
package documenttext
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"github.com/google/uuid"
)
type ExtractionParams struct {
ID uuid.UUID
Location document.Location
}
func (s *Service) executeExtraction(params *ExtractionParams) (*document.Location, error) {
// TODO - various extraction tasks
return &params.Location, nil
}
func (s *Service) extract(ctx context.Context, id uuid.UUID) error {
docId := database.MustToDBUUID(id)
entry, err := s.cfg.GetDBQueries().GetDocumentCleanEntry(ctx, docId)
if err != nil {
return err
}
outLocation, err := s.executeExtraction(&ExtractionParams{
ID: id,
Location: document.Location{
Bucket: entry.Bucket,
Key: entry.Key,
},
})
if err != nil {
return err
}
version := s.svc.Document.GetTextVersion()
err = s.cfg.GetDBQueries().AddDocumentTextEntry(ctx, &repository.AddDocumentTextEntryParams{
Documentid: docId,
Version: version,
Bucket: outLocation.Bucket,
Key: outLocation.Key,
})
if err != nil {
return err
}
return nil
}
internal/document/text/extract_test.go
+62
View File
@@ -0,0 +1,62 @@
package documenttext
import (
"context"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
"testing"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
)
func TestExtract(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocTextConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := Service{
cfg: cfg,
svc: &Services{
Document: document.New(cfg),
},
}
id := uuid.New()
inloc := document.Location{
Bucket: "bucket_name",
Key: "/i/am/here",
}
pool.ExpectQuery("name: GetDocumentCleanEntry :one").WithArgs(database.MustToDBUUID(id)).WillReturnRows(
pgxmock.NewRows([]string{"documentId", "bucket", "key"}).
AddRow(database.MustToDBUUID(id), inloc.Bucket, inloc.Key),
)
pool.ExpectExec("name: AddDocumentTextEntry :exec").WithArgs(database.MustToDBUUID(id), int32(1), inloc.Bucket, inloc.Key).
WillReturnResult(pgxmock.NewResult("", 1))
err = svc.extract(ctx, id)
assert.NoError(t, err)
}
func TestExecuteExtraction(t *testing.T) {
svc := Service{}
id := uuid.New()
location := document.Location{}
outloc, err := svc.executeExtraction(&ExtractionParams{
ID: id,
Location: location,
})
assert.NoError(t, err)
assert.EqualExportedValues(t, location, *outloc)
}
internal/document/text/service.go
+21 -18
View File
@@ -1,27 +1,30 @@
package documenttext
import (
"github.com/google/uuid"
"queryorchestration/internal/document"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/querysync"
)
type ConfigProvider interface {
serviceconfig.ConfigProvider
objectstore.ConfigProvider
querysync.ConfigProvider
}
type Services struct {
Document *document.Service
}
type Service struct {
cfg ConfigProvider
svc *Services
}
func New() *Service {
return &Service{}
}
// TODO - move to api/
type Create struct {
ID uuid.UUID `json:"id" validate:"required,uuid"`
}
type IsExtractedParams struct {
DocumentID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
}
func (s *Service) IsExtracted(params *IsExtractedParams) error {
return nil
func New(cfg ConfigProvider, svc *Services) *Service {
return &Service{
cfg: cfg,
svc: svc,
}
}
internal/document/text/service_test.go
+14 -12
View File
@@ -1,24 +1,26 @@
package documenttext_test
import (
"queryorchestration/internal/document"
documenttext "queryorchestration/internal/document/text"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/querysync"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
type DocTextConfig struct {
serviceconfig.BaseConfig
objectstore.ObjectStoreConfig
querysync.QuerySyncConfig
}
func TestService(t *testing.T) {
svc := documenttext.New()
cfg := &DocTextConfig{}
svc := documenttext.New(cfg, &documenttext.Services{
Document: document.New(cfg),
})
assert.NotNil(t, svc)
}
func TestIsExtracted(t *testing.T) {
svc := documenttext.New()
assert.Nil(t, svc.IsExtracted(&documenttext.IsExtractedParams{
DocumentID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(1),
}))
}
internal/document/version.go
+13
View File
@@ -2,8 +2,21 @@ package document
import (
"fmt"
"github.com/google/uuid"
)
type IsTextExtractedParams struct {
DocumentID uuid.UUID
MinCleanVersion int32
MinTextVersion int32
}
func (s *Service) IsTextExtracted(params *IsTextExtractedParams) error {
// TODO
return nil
}
func (s *Service) GetCleanVersion() int32 {
// TODO - actual version
return 1
internal/document/version_test.go
+12
View File
@@ -5,6 +5,7 @@ import (
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
@@ -41,3 +42,14 @@ func TestGetTextVersion(t *testing.T) {
assert.Equal(t, int32(1), svc.GetTextVersion())
}
func TestIsTextExtracted(t *testing.T) {
cfg := &serviceconfig.BaseConfig{}
svc := document.New(cfg)
assert.Nil(t, svc.IsTextExtracted(&document.IsTextExtractedParams{
DocumentID: uuid.New(),
MinCleanVersion: int32(1),
MinTextVersion: int32(1),
}))
}