diff --git a/internal/serviceconfig/database/pool.go b/internal/serviceconfig/database/pool.go index 2aec2038..e866a392 100644 --- a/internal/serviceconfig/database/pool.go +++ b/internal/serviceconfig/database/pool.go @@ -88,7 +88,7 @@ func (b *DBConfig) DBPing(ctx context.Context) error { slog.Info("Successful database ping") return nil } - slog.Info("Attempted database ping", "host", b.DBHost, "port", b.DBPort) + slog.Debug("Attempted database ping", "host", b.DBHost, "port", b.DBPort) case <-ctx.Done(): return ctx.Err() } diff --git a/internal/serviceconfig/objectstore/config.go b/internal/serviceconfig/objectstore/config.go index 591d6b3c..aa97a523 100644 --- a/internal/serviceconfig/objectstore/config.go +++ b/internal/serviceconfig/objectstore/config.go @@ -73,7 +73,7 @@ func (c *ObjectStoreConfig) PingStoreByName(ctx context.Context, name string) er slog.Info("Successful bucket ping", "name", name) return nil } - slog.Info("Attempted bucket ping", "name", name, "address", c.GetS3Endpoint()) + slog.Debug("Attempted bucket ping", "name", name, "address", c.GetS3Endpoint()) case <-ctx.Done(): return ctx.Err() } diff --git a/internal/serviceconfig/queue/config.go b/internal/serviceconfig/queue/config.go index fd0fa291..53adcd6d 100644 --- a/internal/serviceconfig/queue/config.go +++ b/internal/serviceconfig/queue/config.go @@ -58,7 +58,7 @@ func (c *QueueConfig) PingQueueByURL(ctx context.Context, url string) error { slog.Info("Successful queue ping", "url", url, "approx_msgs", r.Attributes[string(types.QueueAttributeNameApproximateNumberOfMessages)]) return nil } - slog.Info("Attempted queue ping", "url", url, "address", c.GetSQSEndpoint()) + slog.Debug("Attempted queue ping", "url", url, "address", c.GetSQSEndpoint()) case <-ctx.Done(): return ctx.Err() } diff --git a/internal/test/aws.go b/internal/test/aws.go index c9209f4d..e0a86586 100644 --- a/internal/test/aws.go +++ b/internal/test/aws.go @@ -3,8 +3,11 @@ package test import ( "context" "fmt" + "io" + "net/http" "queryorchestration/internal/serviceconfig" "testing" + "time" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" @@ -45,9 +48,18 @@ func CreateAWSContainer(t *testing.T, ctx context.Context, cfg *CreateAWSConfig) }, ExposedPorts: []string{port.Port()}, WaitingFor: wait.ForAll( - // wait.ForExposedPort(), wait.ForListeningPort(port), wait.ForLog("Ready."), + wait.ForHTTP("/_localstack/health"). + WithPort(port). + WithStartupTimeout(50*time.Second). + WithPollInterval(5*time.Second). + WithResponseMatcher(func(body io.Reader) bool { + return true + }). + WithStatusCodeMatcher(func(statusCode int) bool { + return statusCode == http.StatusOK + }), ), } diff --git a/internal/test/container.go b/internal/test/container.go index 8dee4238..46305d05 100644 --- a/internal/test/container.go +++ b/internal/test/container.go @@ -39,6 +39,7 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t "AWS_ACCESS_KEY_ID": cfg.Cfg.GetAWSKeyID(), "AWS_SECRET_ACCESS_KEY": cfg.Cfg.GetAWSSecretKey(), "AWS_REGION": cfg.Cfg.GetAWSRegion(), + "AWS_DEFAULT_REGION": cfg.Cfg.GetAWSRegion(), "AWS_ENDPOINT_URL": cfg.Cfg.GetAWSEndpoint(), "AWS_ENDPOINT_URL_SQS": cfg.Cfg.GetAWSEndpoint(), "AWS_ENDPOINT_URL_S3": cfg.Cfg.GetAWSEndpoint(), diff --git a/internal/test/database.go b/internal/test/database.go index 7e1e8199..1216e952 100644 --- a/internal/test/database.go +++ b/internal/test/database.go @@ -7,6 +7,7 @@ import ( "queryorchestration/internal/serviceconfig" "queryorchestration/internal/serviceconfig/database" "testing" + "time" "github.com/docker/go-connections/nat" "github.com/testcontainers/testcontainers-go" @@ -43,6 +44,12 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (tes wait.ForExposedPort(), wait.ForListeningPort(port), wait.ForLog("database system is ready to accept connections"), + wait.ForSQL(port, "postgres", func(host string, port nat.Port) string { + return fmt.Sprintf("postgres://%s:%s@localhost:%s/%s?sslmode=disable", + user, pass, port.Port(), name) + }). + WithStartupTimeout(25*time.Second). + WithPollInterval(10*time.Second), ), } diff --git a/internal/test/ecosystem.go b/internal/test/ecosystem.go index c203f698..120b182b 100644 --- a/internal/test/ecosystem.go +++ b/internal/test/ecosystem.go @@ -68,12 +68,16 @@ func CreateRunnersAndServicesNetwork(t *testing.T, ctx context.Context, ncfg *Ec qs := make(map[string]*Container, len(ncfg.Runners)) qclean := make([]func(), len(ncfg.Runners)) for i, r := range ncfg.Runners { - url := CreateQueue(t, ctx, ncfg.Cfg, r.Name) + url := r.QueueURL + if url == nil { + u := CreateQueue(t, ctx, ncfg.Cfg, r.Name) + url = &u + } c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{ Name: r.Name, Env: r.Env, - QueueURL: url, + QueueURL: *url, Cfg: ncfg.Cfg, Network: network, }) @@ -167,6 +171,7 @@ func CreateServiceNetwork(t *testing.T, ctx context.Context, scfg *ServiceNetwor type RunnerNetworkConfig struct { Network *testcontainers.DockerNetwork AWSContainer *AWSContainerConfig + QueueURL *string Cfg serviceconfig.ConfigProvider Name Runner Env map[string]string @@ -194,7 +199,11 @@ func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkC } } - url := CreateQueue(t, ctx, rcfg.Cfg, rcfg.Name) + url := rcfg.QueueURL + if url == nil { + u := CreateQueue(t, ctx, rcfg.Cfg, rcfg.Name) + url = &u + } _, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{ Network: network, @@ -204,7 +213,7 @@ func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkC c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{ Name: rcfg.Name, Env: rcfg.Env, - QueueURL: url, + QueueURL: *url, Cfg: rcfg.Cfg, Network: network, }) diff --git a/internal/test/runner.go b/internal/test/runner.go index 14baa7c9..2fd31a9b 100644 --- a/internal/test/runner.go +++ b/internal/test/runner.go @@ -8,6 +8,7 @@ import ( doctextrunner "queryorchestration/api/docTextRunner" jobsyncrunner "queryorchestration/api/jobSyncRunner" queryrunner "queryorchestration/api/queryRunner" + querysyncrunner "queryorchestration/api/querySyncRunner" "testing" "queryorchestration/internal/serviceconfig" @@ -22,7 +23,7 @@ const ( DocSyncRunner = docsyncrunner.Name DocCleanRunner = doccleanrunner.Name DocTextRunner = doctextrunner.Name - QuerySyncRunner = queryrunner.Name + QuerySyncRunner = querysyncrunner.Name QueryRunner = queryrunner.Name JobSyncRunner = jobsyncrunner.Name ) diff --git a/scripts/database.yml b/scripts/database.yml index 99e2c4b9..869ab504 100644 --- a/scripts/database.yml +++ b/scripts/database.yml @@ -16,15 +16,9 @@ tasks: cmds: - task compose:clean:generate - task db:generate - deps:up: - internal: true + lint: cmds: - task compose:up:generate - - sleep 3 - lint: - deps: - - deps:up - cmds: - sqlc vet --file sqlc.yml mig:create: cmds: @@ -33,7 +27,6 @@ tasks: test -z "$name" && echo "Error: Name is required." && exit 1 migrate create -ext sql -dir {{.MIGRATIONS}} $name mig: - deps: - - deps:up cmds: + - task compose:up:generate - migrate -path {{.MIGRATIONS}} -database {{.DB_URI_GENERATE}} up diff --git a/scripts/local-deployments.yml b/scripts/local-deployments.yml index dd625262..ef9a292b 100644 --- a/scripts/local-deployments.yml +++ b/scripts/local-deployments.yml @@ -24,24 +24,34 @@ tasks: COMPOSE_FILE: '{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}' cmds: - docker compose -f {{.COMPOSE_FILE}} build + up:cmd: + internal: true + vars: + COMPOSE_FILE: '{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}' + cmds: + - docker compose -f {{.COMPOSE_FILE}} up --no-recreate --wait -d up:test: cmds: - - docker compose -f {{.TEST_COMPOSE_FILE}} up --no-recreate -d + - task: up:cmd + vars: + COMPOSE_FILE: '{{.TEST_COMPOSE_FILE}}' up:generate: cmds: - - docker compose -f {{.GENERATE_COMPOSE_FILE}} up --no-recreate -d + - task: up:cmd + vars: + COMPOSE_FILE: '{{.GENERATE_COMPOSE_FILE}}' up: cmds: - task docker:build - task: build - - docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d + - task: up:cmd - task: init refresh: cmds: - task docker:build - task: build - task: down - - docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d + - task: up:cmd - task: init init: cmds: diff --git a/test/process_test.go b/test/process_test.go index 456b09dc..91713c57 100644 --- a/test/process_test.go +++ b/test/process_test.go @@ -26,7 +26,6 @@ type ProcessConfig struct { } func TestProcess(t *testing.T) { - t.SkipNow() ctx := context.Background() cfg := &ProcessConfig{} @@ -46,6 +45,14 @@ func TestProcess(t *testing.T) { assert.NoError(t, err) test.CreateStoreClient(t, ctx, cfg, acfg.ExternalEndpoint) + dociniturl := test.CreateQueue(t, ctx, cfg, test.DocInitRunner) + docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner) + doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner) + doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner) + querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner) + queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner) + jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner) + bucketName := "docinitbucket" test.CreateBucket(t, ctx, cfg, bucketName) arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner) @@ -64,55 +71,56 @@ func TestProcess(t *testing.T) { }) assert.NoError(t, err) - docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner) - doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner) - doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner) - querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner) - queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner) - jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner) - net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{ - Cfg: cfg, - Network: network, + Cfg: cfg, + Network: network, + AWSContainer: acfg, Runners: []*test.RunnerNetworkConfig{ { - Name: test.DocInitRunner, + Name: test.DocInitRunner, + QueueURL: &dociniturl, Env: map[string]string{ "DOCUMENT_SYNC_URL": docsyncurl, }, }, { - Name: test.DocSyncRunner, + Name: test.DocSyncRunner, + QueueURL: &docsyncurl, Env: map[string]string{ "DOCUMENT_CLEAN_URL": doccleanurl, }, }, { - Name: test.DocCleanRunner, + Name: test.DocCleanRunner, + QueueURL: &doccleanurl, Env: map[string]string{ "DOCUMENT_TEXT_URL": doctexturl, }, }, { - Name: test.DocTextRunner, + Name: test.DocTextRunner, + QueueURL: &doctexturl, Env: map[string]string{ "QUERY_SYNC_URL": querysyncurl, }, }, { - Name: test.QuerySyncRunner, + Name: test.QuerySyncRunner, + QueueURL: &querysyncurl, Env: map[string]string{ "QUERY_URL": queryurl, }, }, { - Name: test.QueryRunner, + Name: test.QueryRunner, + QueueURL: &queryurl, Env: map[string]string{ "QUERY_URL": queryurl, }, }, { - Name: test.JobSyncRunner, + Name: test.JobSyncRunner, + QueueURL: &jobsyncurl, Env: map[string]string{ "DOCUMENT_SYNC_URL": docsyncurl, }, @@ -155,8 +163,10 @@ func TestProcess(t *testing.T) { Type: queryservice.CONTEXTFULL, }) assert.NoError(t, err) + jcfg := `{"path":"id"}` jsonQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ Type: queryservice.JSONEXTRACTOR, + Config: &jcfg, RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id}, }) assert.NoError(t, err) diff --git a/test/queryService/testquery_test.go b/test/queryService/testquery_test.go deleted file mode 100644 index c77de2ee..00000000 --- a/test/queryService/testquery_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package endtoend_test - -import ( - "context" - "queryorchestration/internal/test" - queryservice "queryorchestration/pkg/queryService" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" -) - -func TestQueryServiceTest(t *testing.T) { - ctx := context.Background() - - c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{ - Name: test.QueryService, - Env: map[string]string{ - "JOB_SYNC_URL": "/i/am/here", - }, - }) - defer cleanup() - - client, err := queryservice.NewClientWithResponses(c.URI) - assert.NoError(t, err) - - idRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{ - Type: queryservice.CONTEXTFULL, - }) - assert.NoError(t, err) - id := idRes.JSON201.Id - assert.NotEmpty(t, id) - - docId := uuid.New() - - testRes, err := client.TestQueryWithResponse(ctx, id, queryservice.QueryTestRequest{ - DocumentId: docId, - QueryVersion: int32(1), - }) - assert.NoError(t, err) - assert.NotNil(t, testRes) - // TODO - // assert.NotNil(t, testRes.JSON200) - // assert.Equal(t, "hey", testRes.JSON200.Value) -}