jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/localstackands3object (pull request #54)

Browse Source 
LocalStack and S3 Objects

* firstround

* cleanupintegration

* tidyports
This commit is contained in:
Michael McGuinness
2025-02-06 15:00:26 +00:00
parent 9e6328d8f9
commit e146725825
17 changed files with 381 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -1,3 +1,6 @@
**/*.http
**/*.cast
# Ignore the binary files generated by `go build`
/main
api/docInitRunner/runner.go
+59 -4
View File
@@ -3,6 +3,9 @@ package docinitrunner
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
documentinit "queryorchestration/internal/document/init"
"github.com/go-playground/validator/v10"
@@ -28,8 +31,30 @@ func New(validator *validator.Validate, svc *Services) Runner {
}
}
type S3EventNotification struct {
Records []struct {
EventVersion string `json:"eventVersion"`
EventSource string `json:"eventSource"`
AwsRegion string `json:"awsRegion"`
EventTime string `json:"eventTime"`
EventName string `json:"eventName"`
S3 struct {
Bucket struct {
Name string `json:"name"`
Arn string `json:"arn"`
} `json:"bucket"`
Object struct {
Key string `json:"key"`
Size int64 `json:"size"`
ETag string `json:"eTag"`
VersionId string `json:"versionId"`
} `json:"object"`
} `json:"s3"`
} `json:"Records"`
}
func (s Runner) Process(ctx context.Context, req *types.Message) error {
var body documentinit.Create
var body S3EventNotification
err := json.Unmarshal([]byte(*req.Body), &body)
if err != nil {
return err
@@ -40,9 +65,39 @@ func (s Runner) Process(ctx context.Context, req *types.Message) error {
return err
}
_, err = s.svc.Document.Create(ctx, &body)
if err != nil {
return err
hasErr := false
for _, record := range body.Records {
err := func() error {
if record.EventName != "ObjectCreated:Put" {
return fmt.Errorf("invalid event name: %s", record.EventName)
}
jobID, err := s.svc.Document.GetJobIDFromKey(record.S3.Object.Key)
if err != nil {
return err
}
_, err = s.svc.Document.Create(ctx, &documentinit.Create{
JobID: jobID,
Location: record.S3.Object.Key,
Bucket: record.S3.Bucket.Name,
Hash: record.S3.Object.ETag,
})
if err != nil {
return err
}
return nil
}()
if err != nil {
hasErr = true
slog.Error("Error processing record", "err", err)
}
}
if hasErr {
return errors.New("error processing records")
}
return nil
api/docInitRunner/runner_test.go
-14
View File
@@ -17,13 +17,11 @@ import (
objectstoremock "queryorchestration/mocks/objectstore"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/go-playground/validator/v10"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
type DocInitConfig struct {
@@ -98,18 +96,6 @@ func TestDocInitRunner(t *testing.T) {
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == doc.Bucket && *in.Key == doc.Location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &docinfo.Hash,
}, nil)
err = runner.Process(ctx, msg)
assert.NoError(t, err)
}
cmd/docInitRunner/main.go
-8
View File
@@ -13,7 +13,6 @@ import (
"queryorchestration/internal/job/collector"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig"
objectstore "queryorchestration/internal/serviceconfig/objectstore"
documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean"
_ "github.com/lib/pq"
@@ -22,7 +21,6 @@ import (
type DocInitConfig struct {
runner.BaseConfig
documentcleanc.DocCleanConfig
objectstore.ObjectStoreConfig
}
func main() {
@@ -62,11 +60,5 @@ func main() {
os.Exit(1)
}
err = cfg.SetStoreClient(ctx)
if err != nil {
slog.Error(err.Error())
os.Exit(1)
}
server.Listen(ctx)
}
deployments/compose.local.yaml
+70
View File
@@ -1,5 +1,75 @@
---
services:
doc_init_runner:
image: queryorchestration:latest
command: ["./docInitRunner"]
depends_on:
- db
- localstack
environment:
QUEUE_URL: ${DOCUMENT_INIT_URL}
DOCUMENT_CLEAN_URL: ${DOCUMENT_CLEAN_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
AWS_REGION: ${AWS_REGION}
DB_USER: ${DB_USER}
DB_PASS: ${DB_PASS}
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME}
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
networks:
- server-network
query_runner:
image: queryorchestration:latest
command: ["./queryRunner"]
depends_on:
- db
- localstack
environment:
QUEUE_URL: ${QUERY_RUNNER_URL}
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
AWS_REGION: ${AWS_REGION}
DB_USER: ${DB_USER}
DB_PASS: ${DB_PASS}
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME}
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
networks:
- server-network
query_service:
image: queryorchestration:latest
command: ["./queryService"]
depends_on:
- db
- localstack
ports:
- 8080:8080
expose:
- 8080
environment:
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
AWS_REGION: ${AWS_REGION}
DB_USER: ${DB_USER}
DB_PASS: ${DB_PASS}
DB_HOST: db
DB_PORT: 5432
DB_NAME: ${DB_NAME}
DB_NOSSL: ${DB_NOSSL}
AWS_ENDPOINT_URL: "http://localstack:4566"
AWS_S3_USE_PATH_STYLE: true
networks:
- server-network
db:
image: postgres:17.2-alpine3.21
ports:
deployments/compose.test.yaml
+2 -2
View File
@@ -3,13 +3,13 @@ services:
test-db:
image: postgres:17.2-alpine3.21
ports:
- 5432:5432
- 5430:5432
environment:
POSTGRES_DB: ${DB_NAME}
POSTGRES_PASSWORD: ${DB_PASS}
POSTGRES_USER: ${DB_USER}
expose:
- 5432
- 5430
volumes:
- test-db-data:/var/lib/postgresql/data
healthcheck:
devbox.json
+2
View File
@@ -22,6 +22,7 @@
"shell": {
"init_hook": [
"export DB_URI=postgres://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT}/${DB_NAME}?sslmode=disable",
"export DB_URI_TEST=postgres://${DB_USER}:${DB_PASS}@${DB_HOST}:${DB_PORT_TEST}/${DB_NAME}?sslmode=disable",
"echo 'Welcome to the DoczyAI devbox!'"
]
},
@@ -31,6 +32,7 @@
"DB_PASS": "pass",
"DB_HOST": "localhost",
"DB_PORT": "5432",
"DB_PORT_TEST": "5430",
"DB_NAME": "query_orchestration",
"DB_NOSSL": "true",
"AWS_ACCESS_KEY_ID": "test",
internal/document/init/create.go
+28 -17
View File
@@ -4,23 +4,25 @@ import (
"context"
"database/sql"
"errors"
"fmt"
"queryorchestration/internal/database"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/document"
documentclean "queryorchestration/internal/document/clean"
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig/queue"
"regexp"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Create struct {
JobID uuid.UUID `json:"jobId" validate:"required,uuid"`
Location document.Location `json:"location" validate:"required"`
Bucket string `json:"bucket" validate:"required"`
JobID uuid.UUID
Location document.Location
Bucket string
Hash string
}
func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
@@ -39,9 +41,11 @@ func (s *Service) Create(ctx context.Context, doc *Create) (uuid.UUID, error) {
return uuid.Nil, err
}
err = s.informCreate(ctx, id, j)
if err != nil {
return uuid.Nil, err
if params.ID == nil {
err = s.informCreate(ctx, id, j)
if err != nil {
return uuid.Nil, err
}
}
return id, nil
@@ -56,17 +60,9 @@ type createDocumentParams struct {
}
func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocumentParams, error) {
res, err := s.cfg.GetStoreClient().HeadObject(ctx, &s3.HeadObjectInput{
Bucket: &doc.Bucket,
Key: &doc.Location,
})
if err != nil {
return nil, err
}
idbyhash, err := s.cfg.GetDBQueries().GetDocumentIDByHash(ctx, &repository.GetDocumentIDByHashParams{
Jobid: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
Hash: doc.Hash,
})
var docID *pgtype.UUID
if err != nil && !errors.Is(err, sql.ErrNoRows) {
@@ -78,7 +74,7 @@ func (s *Service) getCreateParams(ctx context.Context, doc *Create) (*createDocu
return &createDocumentParams{
ID: docID,
JobID: database.MustToDBUUID(doc.JobID),
Hash: *res.ETag,
Hash: doc.Hash,
Bucket: doc.Bucket,
Location: doc.Location,
}, nil
@@ -141,3 +137,18 @@ func (s *Service) informCreate(ctx context.Context, id uuid.UUID, j *job.Job) er
return nil
}
func (s *Service) GetJobIDFromKey(key string) (uuid.UUID, error) {
re := regexp.MustCompile(`^.+/(.+)/.+$`)
match := re.FindStringSubmatch(key)
if match == nil || len(match) != 2 {
return uuid.Nil, fmt.Errorf("no job id match found in key")
}
id, err := uuid.Parse(match[1])
if err != nil {
return uuid.Nil, err
}
return id, nil
}
internal/document/init/create_test.go
+21 -49
View File
@@ -16,7 +16,6 @@ import (
queuemock "queryorchestration/mocks/queue"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/sqs"
"github.com/google/uuid"
"github.com/pashagolub/pgxmock/v3"
@@ -82,22 +81,11 @@ func TestCreate(t *testing.T) {
WillReturnResult(pgxmock.NewResult("", 1))
pool.ExpectCommit()
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
id, err := svc.Create(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
Hash: doc.Hash,
})
assert.NoError(t, err)
assert.Equal(t, doc.ID, id)
@@ -164,22 +152,11 @@ func TestGetCreateParams(t *testing.T) {
pgxmock.NewRows([]string{"id"}),
)
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
params, err := svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
Hash: doc.Hash,
})
assert.NoError(t, err)
assert.Equal(t, &createDocumentParams{
@@ -222,22 +199,11 @@ func TestGetCreateParamsExisting(t *testing.T) {
AddRow(database.MustToDBUUID(doc.ID)),
)
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
params, err := svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
Bucket: bucket,
Hash: doc.Hash,
})
assert.NoError(t, err)
dbid := database.MustToDBUUID(doc.ID)
@@ -279,18 +245,6 @@ func TestGetCreateParamsCurrentDocErr(t *testing.T) {
pool.ExpectQuery("-- name: GetDocumentIDByHash :one").WithArgs(doc.Hash, database.MustToDBUUID(doc.JobID)).
WillReturnError(errors.New("db err"))
mockStore.EXPECT().
HeadObject(
mock.Anything,
mock.MatchedBy(func(in *s3.HeadObjectInput) bool {
return *in.Bucket == bucket && *in.Key == location
}),
mock.Anything,
).
Return(&s3.HeadObjectOutput{
ETag: &doc.Hash,
}, nil)
_, err = svc.getCreateParams(ctx, &Create{
JobID: doc.JobID,
Location: location,
@@ -385,3 +339,21 @@ func TestSubmitCreateExists(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, doc.ID, id)
}
func TestGetJobIDFromKey(t *testing.T) {
svc := Service{}
_, err := svc.GetJobIDFromKey("")
assert.Error(t, err)
_, err = svc.GetJobIDFromKey("jkhaskhweuifhwhieh")
assert.Error(t, err)
_, err = svc.GetJobIDFromKey("aaa/bbb/ccc")
assert.Error(t, err)
id := uuid.New()
aid, err := svc.GetJobIDFromKey(fmt.Sprintf("aaa/%s/bbb", id.String()))
assert.NoError(t, err)
assert.Equal(t, id, aid)
}
internal/document/init/service.go
-2
View File
@@ -3,7 +3,6 @@ package documentinit
import (
"queryorchestration/internal/job"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue/documentclean"
)
@@ -14,7 +13,6 @@ type Services struct {
type ConfigProvider interface {
serviceconfig.ConfigProvider
documentclean.ConfigProvider
objectstore.ConfigProvider
}
type Service struct {
internal/server/runner/poll.go
+3 -3
View File
@@ -43,11 +43,11 @@ func (c *Server) pollMessage(ctx context.Context) error {
}
for _, message := range result.Messages {
slog.Info("processing message", "id", message.MessageId)
slog.Info("processing message", "id", *message.MessageId)
err := c.cfg.GetController().Process(ctx, &message)
if err != nil {
slog.Error("message process fail", "err", err)
slog.Error("message process fail", "id", *message.MessageId, "err", err)
}
err = c.cfg.DeleteFromQueue(ctx, &queue.DeleteParams{
@@ -55,7 +55,7 @@ func (c *Server) pollMessage(ctx context.Context) error {
ReceiptHandle: message.ReceiptHandle,
})
if err != nil {
slog.Error("message delete fail", "err", err)
slog.Error("message delete fail", "id", *message.MessageId, "err", err)
}
}
internal/serviceconfig/objectstore/client.go
+4
View File
@@ -47,4 +47,8 @@ type S3Client interface {
GetBucketLifecycleConfiguration(ctx context.Context, params *s3.GetBucketLifecycleConfigurationInput, opts ...func(*s3.Options)) (*s3.GetBucketLifecycleConfigurationOutput, error)
PutBucketLifecycleConfiguration(ctx context.Context, params *s3.PutBucketLifecycleConfigurationInput, opts ...func(*s3.Options)) (*s3.PutBucketLifecycleConfigurationOutput, error)
DeleteBucketLifecycle(ctx context.Context, params *s3.DeleteBucketLifecycleInput, opts ...func(*s3.Options)) (*s3.DeleteBucketLifecycleOutput, error)
// Bucket Notifications
PutBucketNotificationConfiguration(ctx context.Context, params *s3.PutBucketNotificationConfigurationInput, opts ...func(*s3.Options)) (*s3.PutBucketNotificationConfigurationOutput, error)
GetBucketNotificationConfiguration(ctx context.Context, params *s3.GetBucketNotificationConfigurationInput, opts ...func(*s3.Options)) (*s3.GetBucketNotificationConfigurationOutput, error)
}
mocks/objectstore/mock_S3Client.go
+148
View File
@@ -911,6 +911,80 @@ func (_c *MockS3Client_GetBucketLifecycleConfiguration_Call) RunAndReturn(run fu
return _c
}
// GetBucketNotificationConfiguration provides a mock function with given fields: ctx, params, opts
func (_m *MockS3Client) GetBucketNotificationConfiguration(ctx context.Context, params *s3.GetBucketNotificationConfigurationInput, opts ...func(*s3.Options)) (*s3.GetBucketNotificationConfigurationOutput, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, params)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for GetBucketNotificationConfiguration")
}
var r0 *s3.GetBucketNotificationConfigurationOutput
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *s3.GetBucketNotificationConfigurationInput, ...func(*s3.Options)) (*s3.GetBucketNotificationConfigurationOutput, error)); ok {
return rf(ctx, params, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *s3.GetBucketNotificationConfigurationInput, ...func(*s3.Options)) *s3.GetBucketNotificationConfigurationOutput); ok {
r0 = rf(ctx, params, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*s3.GetBucketNotificationConfigurationOutput)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *s3.GetBucketNotificationConfigurationInput, ...func(*s3.Options)) error); ok {
r1 = rf(ctx, params, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockS3Client_GetBucketNotificationConfiguration_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetBucketNotificationConfiguration'
type MockS3Client_GetBucketNotificationConfiguration_Call struct {
*mock.Call
}
// GetBucketNotificationConfiguration is a helper method to define mock.On call
// - ctx context.Context
// - params *s3.GetBucketNotificationConfigurationInput
// - opts ...func(*s3.Options)
func (_e *MockS3Client_Expecter) GetBucketNotificationConfiguration(ctx interface{}, params interface{}, opts ...interface{}) *MockS3Client_GetBucketNotificationConfiguration_Call {
return &MockS3Client_GetBucketNotificationConfiguration_Call{Call: _e.mock.On("GetBucketNotificationConfiguration",
append([]interface{}{ctx, params}, opts...)...)}
}
func (_c *MockS3Client_GetBucketNotificationConfiguration_Call) Run(run func(ctx context.Context, params *s3.GetBucketNotificationConfigurationInput, opts ...func(*s3.Options))) *MockS3Client_GetBucketNotificationConfiguration_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]func(*s3.Options), len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(func(*s3.Options))
}
}
run(args[0].(context.Context), args[1].(*s3.GetBucketNotificationConfigurationInput), variadicArgs...)
})
return _c
}
func (_c *MockS3Client_GetBucketNotificationConfiguration_Call) Return(_a0 *s3.GetBucketNotificationConfigurationOutput, _a1 error) *MockS3Client_GetBucketNotificationConfiguration_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockS3Client_GetBucketNotificationConfiguration_Call) RunAndReturn(run func(context.Context, *s3.GetBucketNotificationConfigurationInput, ...func(*s3.Options)) (*s3.GetBucketNotificationConfigurationOutput, error)) *MockS3Client_GetBucketNotificationConfiguration_Call {
_c.Call.Return(run)
return _c
}
// GetObject provides a mock function with given fields: ctx, params, opts
func (_m *MockS3Client) GetObject(ctx context.Context, params *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) {
_va := make([]interface{}, len(opts))
@@ -1799,6 +1873,80 @@ func (_c *MockS3Client_PutBucketLifecycleConfiguration_Call) RunAndReturn(run fu
return _c
}
// PutBucketNotificationConfiguration provides a mock function with given fields: ctx, params, opts
func (_m *MockS3Client) PutBucketNotificationConfiguration(ctx context.Context, params *s3.PutBucketNotificationConfigurationInput, opts ...func(*s3.Options)) (*s3.PutBucketNotificationConfigurationOutput, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, params)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for PutBucketNotificationConfiguration")
}
var r0 *s3.PutBucketNotificationConfigurationOutput
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *s3.PutBucketNotificationConfigurationInput, ...func(*s3.Options)) (*s3.PutBucketNotificationConfigurationOutput, error)); ok {
return rf(ctx, params, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *s3.PutBucketNotificationConfigurationInput, ...func(*s3.Options)) *s3.PutBucketNotificationConfigurationOutput); ok {
r0 = rf(ctx, params, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*s3.PutBucketNotificationConfigurationOutput)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *s3.PutBucketNotificationConfigurationInput, ...func(*s3.Options)) error); ok {
r1 = rf(ctx, params, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockS3Client_PutBucketNotificationConfiguration_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PutBucketNotificationConfiguration'
type MockS3Client_PutBucketNotificationConfiguration_Call struct {
*mock.Call
}
// PutBucketNotificationConfiguration is a helper method to define mock.On call
// - ctx context.Context
// - params *s3.PutBucketNotificationConfigurationInput
// - opts ...func(*s3.Options)
func (_e *MockS3Client_Expecter) PutBucketNotificationConfiguration(ctx interface{}, params interface{}, opts ...interface{}) *MockS3Client_PutBucketNotificationConfiguration_Call {
return &MockS3Client_PutBucketNotificationConfiguration_Call{Call: _e.mock.On("PutBucketNotificationConfiguration",
append([]interface{}{ctx, params}, opts...)...)}
}
func (_c *MockS3Client_PutBucketNotificationConfiguration_Call) Run(run func(ctx context.Context, params *s3.PutBucketNotificationConfigurationInput, opts ...func(*s3.Options))) *MockS3Client_PutBucketNotificationConfiguration_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]func(*s3.Options), len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(func(*s3.Options))
}
}
run(args[0].(context.Context), args[1].(*s3.PutBucketNotificationConfigurationInput), variadicArgs...)
})
return _c
}
func (_c *MockS3Client_PutBucketNotificationConfiguration_Call) Return(_a0 *s3.PutBucketNotificationConfigurationOutput, _a1 error) *MockS3Client_PutBucketNotificationConfiguration_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockS3Client_PutBucketNotificationConfiguration_Call) RunAndReturn(run func(context.Context, *s3.PutBucketNotificationConfigurationInput, ...func(*s3.Options)) (*s3.PutBucketNotificationConfigurationOutput, error)) *MockS3Client_PutBucketNotificationConfiguration_Call {
_c.Call.Return(run)
return _c
}
// PutObject provides a mock function with given fields: ctx, params, opts
func (_m *MockS3Client) PutObject(ctx context.Context, params *s3.PutObjectInput, opts ...func(*s3.Options)) (*s3.PutObjectOutput, error) {
_va := make([]interface{}, len(opts))
scripts/compose.yml
+21 -15
View File
@@ -21,16 +21,30 @@ tasks:
up:test:
cmds:
- docker compose -f {{.TEST_COMPOSE_FILE}} up --no-recreate -d
refresh:
cmds:
- task: down
- task: up
up:
cmds:
- task docker:build
- docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d
- task: init
init:
cmds:
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_CLEAN
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_INIT
- aws sqs create-queue --queue-name $QNAME_QUERY_RUNNER
- aws s3 mb s3://$BUCKET_IN
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_INIT
- |
aws s3api put-bucket-notification-configuration \
--bucket $BUCKET_IN \
--notification-configuration '{
"QueueConfigurations": [{
"QueueArn": "arn:aws:sqs:us-east-1:000000000000:document_init",
"Events": ["s3:ObjectCreated:*"]
}]
}'
- aws sqs create-queue --queue-name $QNAME_DOCUMENT_CLEAN
- aws sqs create-queue --queue-name $QNAME_QUERY_RUNNER
down:test:
cmds:
- task: down
@@ -51,15 +65,7 @@ tasks:
- docker volume rm deployments_db-data
lint:
cmds:
- task: lint:file
vars:
COMPOSE_FILE: '{{.LOCAL_COMPOSE_FILE}}'
- task: lint:file
vars:
COMPOSE_FILE: '{{.TEST_COMPOSE_FILE}}'
lint:file:
internal: true
vars:
COMPOSE_FILE: '{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}'
cmds:
- docker compose -f {{.COMPOSE_FILE}} config -q --resolve-image-digests
- docker compose -f {{.LOCAL_COMPOSE_FILE}} config -q
- |
docker compose -f {{.TEST_COMPOSE_FILE}} config -q \
--resolve-image-digests
scripts/database.yml
+1 -1
View File
@@ -32,4 +32,4 @@ tasks:
deps:
- deps:up
cmds:
- migrate -path {{.MIGRATIONS}} -database {{.DB_URI}} up
- migrate -path {{.MIGRATIONS}} -database {{.DB_URI_TEST}} up
sqlc.yml
+1 -1
View File
@@ -2,7 +2,7 @@
version: "2"
servers:
- engine: postgresql
uri: "${DB_URI}"
uri: "${DB_URI_TEST}"
sql:
- name: "db"
engine: "postgresql"
test/docInitRunner/docinitrunner_test.go
+18 -12
View File
@@ -2,12 +2,11 @@ package integration_test
import (
"context"
"fmt"
"os"
"path"
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/server/runner"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/serviceconfig/queue"
documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
@@ -16,6 +15,7 @@ import (
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
)
@@ -49,6 +49,21 @@ func TestDocInitRunner(t *testing.T) {
doccleanurl := test.CreateQueue(t, ctx, cfg, "docclean")
bucketName := "docinitbucket"
test.CreateBucket(t, ctx, cfg, bucketName)
arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner)
_, err = cfg.StoreClient.PutBucketNotificationConfiguration(ctx, &s3.PutBucketNotificationConfigurationInput{
Bucket: &bucketName,
NotificationConfiguration: &types.NotificationConfiguration{
QueueConfigurations: []types.QueueConfiguration{
{
QueueArn: &arn,
Events: []types.Event{
types.EventS3ObjectCreated,
},
},
},
},
})
assert.NoError(t, err)
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
Cfg: cfg,
@@ -91,7 +106,7 @@ func TestDocInitRunner(t *testing.T) {
})
assert.NoError(t, err)
location := "/i/am/here"
location := fmt.Sprintf("%s/%s/%s", clientRes.JSON201.Id, jobRes.JSON201.Id, "object_name")
body := strings.NewReader("hello world")
_, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{
Bucket: &bucketName,
@@ -100,15 +115,6 @@ func TestDocInitRunner(t *testing.T) {
})
assert.NoError(t, err)
err = cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: net.Runners[test.DocInitRunner].URI,
Body: documentinit.Create{
JobID: jobRes.JSON201.Id,
Location: location,
},
})
assert.NoError(t, err)
resRegex := regexp.MustCompile(`{"id": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"`)
test.AssertMessageBody(t, ctx, cfg, doccleanurl, resRegex)
}