jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/docinit (pull request #48)

Browse Source 
Feature/docinit

* createunittests

* cleanup

* skip

* cleanup
This commit is contained in:
Michael McGuinness
2025-02-05 17:44:01 +00:00
parent 92334ad1dd
commit 5c253b3592
28 changed files with 536 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/document/get.go
+3 -4
View File
@@ -14,9 +14,8 @@ func (s *Service) Get(ctx context.Context, id uuid.UUID) (*Document, error) {
}
return &Document{
ID: database.MustToUUID(doc.ID),
JobID: database.MustToUUID(doc.Jobid),
Hash: doc.Hash,
Location: doc.Location,
ID: database.MustToUUID(doc.ID),
JobID: database.MustToUUID(doc.Jobid),
Hash: doc.Hash,
}, nil
}
internal/document/get_test.go
+5 -6
View File
@@ -27,16 +27,15 @@ func TestGet(t *testing.T) {
svc := document.New(cfg)
doc := document.Document{
ID: uuid.New(),
JobID: uuid.New(),
Hash: "example_hash",
Location: "example_location",
ID: uuid.New(),
JobID: uuid.New(),
Hash: "example_hash",
}
pool.ExpectQuery("name: GetDocument :one").WithArgs(database.MustToDBUUID(doc.ID)).
WillReturnRows(
pgxmock.NewRows([]string{"id", "jobId", "hash", "location"}).
AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash, doc.Location),
pgxmock.NewRows([]string{"id", "jobId", "hash"}).
AddRow(database.MustToDBUUID(doc.ID), database.MustToDBUUID(doc.JobID), doc.Hash),
)
adoc, err := svc.Get(ctx, doc.ID)
internal/document/init/create.go
+65 -16
View File
@@ -2,6 +2,8 @@ package documentinit
import (
"context"
"database/sql"
"errors"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
@@ -9,13 +11,16 @@ import (
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig/queue"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Create struct {
JobID uuid.UUID `json:"jobId" validate:"required,uuid"`
Location document.Location `json:"location" validate:"required"`
Bucket string `json:"bucket" validate:"required"`
}
func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -42,38 +47,82 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
return id, nil
}
func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*repository.CreateDocumentParams, error) {
// TODO - get document
type createDocumentParams struct {
ID *pgtype.UUID
JobID pgtype.UUID
Hash string
Bucket string
Location string
}
hash, err := s.getHash()
func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
res, err := s.cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &doc.Bucket,
Key: &doc.Location,
})
if err != nil {
return nil, err
}
return &repository.CreateDocumentParams{
Jobid: database.MustToDBUUID(doc.JobID),
Hash: hash,
idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
Jobid: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
})
var docID *pgtype.UUID
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return nil, err
} else if err == nil {
docID = &idbyhash
}
return &createDocumentParams{
ID: docID,
JobID: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
Bucket: doc.Bucket,
Location: doc.Location,
}, nil
}
func (s *Service) submitCreate(ctx context.Context, params *repository.CreateDocumentParams) (uuid.UUID, error) {
// TODO create - or if hash exists log attempt
dbid, err := s.cfg.GetDBQueries().CreateDocument(ctx, params)
func (s *Service) submitCreate(ctx context.Context, params *createDocumentParams) (uuid.UUID, error) {
var id uuid.UUID
err := s.cfg.ExecuteDBTransaction(ctx, func(ctx context.Context, q *repository.Queries) error {
var dbid pgtype.UUID
if params.ID == nil {
createid, err := s.cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Jobid: params.JobID,
Hash: params.Hash,
})
if err != nil {
return err
}
dbid = createid
} else {
dbid = *params.ID
}
err := s.cfg.GetDBQueries().AddDocumentEntry(ctx, &repository.AddDocumentEntryParams{
Documentid: dbid,
Bucket: params.Bucket,
Location: params.Location,
})
if err != nil {
return err
}
id = database.MustToUUID(dbid)
return nil
})
if err != nil {
return uuid.Nil, err
}
id := database.MustToUUID(dbid)
return id, nil
}
func (s *Service) getHash() (string, error) {
// TODO
return "example_hash", nil
}
func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) error {
if !j.CanSync {
return nil
internal/document/init/create_test.go
+286 -5
View File
@@ -2,6 +2,7 @@ package documentinit
import (
"context"
"errors"
"fmt"
"queryorchestration/internal/client"
"queryorchestration/internal/database"
@@ -9,10 +10,13 @@ import (
"queryorchestration/internal/document"
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentclean"
objectstoremock "queryorchestration/mocks/objectstore"
queuemock "queryorchestration/mocks/queue"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
@@ -23,6 +27,7 @@ import (
type DocInitConfig struct {
serviceconfig.BaseConfig
documentclean.DocCleanConfig
objectstore.ObjectStoreConfig
}
func TestCreate(t *testing.T) {
@@ -35,6 +40,8 @@ func TestCreate(t *testing.T) {
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
@@ -47,10 +54,12 @@ func TestCreate(t *testing.T) {
ClientID: uuid.New(),
}
doc := document.Document{
ID: uuid.New(),
JobID: j.ID,
Location: "example_location",
ID: uuid.New(),
JobID: j.ID,
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectQuery("name: GetJob :one").WithArgs(database.MustToDBUUID(j.ID)).WillReturnRows(
pgxmock.NewRows([]string{"id", "clientId", "canSync"}).
@@ -60,15 +69,35 @@ func TestCreate(t *testing.T) {
pgxmock.NewRows([]string{"id", "name", "canSync"}).
AddRow(database.MustToDBUUID(j.ClientID), "client_name", true),
)
pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), pgxmock.AnyArg(), doc.Location).
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}),
)
pool.ExpectBegin()
pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(doc.ID)),
)
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
id, err := svc.Create(ctx, &Create{
JobID: doc.JobID,
Location: doc.Location,
Location: location,
Bucket: bucket,
})
assert.NoError(t, err)
assert.Equal(t, doc.ID, id)
@@ -104,3 +133,255 @@ func TestInformCreate(t *testing.T) {
err = svc.informCreate(ctx, id, j)
assert.NoError(t, err)
}
func TestGetCreateParams(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
Client: client.New(cfg),
}),
})
doc := document.Document{
JobID: uuid.New(),
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}),
)
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
params, err := svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
})
assert.NoError(t, err)
assert.Equal(t, &createDocumentParams{
JobID: database.MustToDBUUID(doc.JobID),
Hash: doc.Hash,
Bucket: bucket,
Location: location,
}, params)
}
func TestGetCreateParamsExisting(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
Client: client.New(cfg),
}),
})
doc := document.Document{
ID: uuid.New(),
JobID: uuid.New(),
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(doc.ID)),
)
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
params, err := svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
})
assert.NoError(t, err)
dbid := database.MustToDBUUID(doc.ID)
assert.Equal(t, &createDocumentParams{
ID: &dbid,
JobID: database.MustToDBUUID(doc.JobID),
Hash: doc.Hash,
Bucket: bucket,
Location: location,
}, params)
}
func TestGetCreateParamsCurrentDocErr(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
mockStore := objectstoremock.NewMockS3Client(t)
cfg.StoreClient = mockStore
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
Client: client.New(cfg),
}),
})
doc := document.Document{
JobID: uuid.New(),
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).
WillReturnError(errors.New("db err"))
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
_, err = svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
})
assert.Error(t, err)
}
func TestSubmitCreate(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
Client: client.New(cfg),
}),
})
doc := document.Document{
ID: uuid.New(),
JobID: uuid.New(),
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectBegin()
pool.ExpectQuery("name: CreateDocument :one").WithArgs(database.MustToDBUUID(doc.JobID), doc.Hash).
WillReturnRows(
pgxmock.NewRows([]string{"id"}).
AddRow(database.MustToDBUUID(doc.ID)),
)
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
id, err := svc.submitCreate(ctx, &createDocumentParams{
JobID: database.MustToDBUUID(doc.JobID),
Location: location,
Bucket: bucket,
Hash: doc.Hash,
})
assert.NoError(t, err)
assert.Equal(t, doc.ID, id)
}
func TestSubmitCreateExists(t *testing.T) {
ctx := context.Background()
pool, err := pgxmock.NewPool()
if err != nil {
t.Fatalf("failed to open pgxmock database: %v", err)
}
cfg := &DocInitConfig{}
cfg.DBPool = pool
cfg.DBQueries = repository.New(pool)
svc := New(cfg, &Services{
Job: job.New(cfg, &job.Services{
Client: client.New(cfg),
}),
})
doc := document.Document{
ID: uuid.New(),
JobID: uuid.New(),
Hash: "example_hash",
}
bucket := "eample_bucket"
location := "/i/am/here"
pool.ExpectBegin()
pool.ExpectExec("name: AddDocumentEntry :exec").WithArgs(database.MustToDBUUID(doc.ID), bucket, location).
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
docid := database.MustToDBUUID(doc.ID)
id, err := svc.submitCreate(ctx, &createDocumentParams{
ID: &docid,
JobID: database.MustToDBUUID(doc.JobID),
Location: location,
Bucket: bucket,
Hash: doc.Hash,
})
assert.NoError(t, err)
assert.Equal(t, doc.ID, id)
}
internal/document/init/service.go
+2
View File
@@ -3,6 +3,7 @@ package documentinit
import (
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentclean"
)
@@ -13,6 +14,7 @@ type Services struct {
type ConfigProvider interface {
serviceconfig.ConfigProvider
documentclean.ConfigProvider
objectstore.ConfigProvider
}
type Service struct {
internal/document/service.go
+3 -4
View File
@@ -9,10 +9,9 @@ import (
type Location = string
type Document struct {
ID uuid.UUID
JobID uuid.UUID
Hash string
Location Location
ID uuid.UUID
JobID uuid.UUID
Hash string
}
type Service struct {