jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/fixe2etests (pull request #64)

Browse Source 
Fix e2e test

* fix
This commit is contained in:
Michael McGuinness
2025-02-13 00:23:13 +00:00
parent c1e9d93037
commit f5bd522faf
12 changed files with 82 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/serviceconfig/database/pool.go
+1 -1
View File
@@ -88,7 +88,7 @@ func (b *DBConfig) DBPing(ctx context.Context) error {
slog.Info("Successful database ping")
return nil
}
slog.Info("Attempted database ping", "host", b.DBHost, "port", b.DBPort)
slog.Debug("Attempted database ping", "host", b.DBHost, "port", b.DBPort)
case <-ctx.Done():
return ctx.Err()
}
internal/serviceconfig/objectstore/config.go
+1 -1
View File
@@ -73,7 +73,7 @@ func (c *ObjectStoreConfig) PingStoreByName(ctx context.Context, name string) er
slog.Info("Successful bucket ping", "name", name)
return nil
}
slog.Info("Attempted bucket ping", "name", name, "address", c.GetS3Endpoint())
slog.Debug("Attempted bucket ping", "name", name, "address", c.GetS3Endpoint())
case <-ctx.Done():
return ctx.Err()
}
internal/serviceconfig/queue/config.go
+1 -1
View File
@@ -58,7 +58,7 @@ func (c *QueueConfig) PingQueueByURL(ctx context.Context, url string) error {
slog.Info("Successful queue ping", "url", url, "approx_msgs", r.Attributes[string(types.QueueAttributeNameApproximateNumberOfMessages)])
return nil
}
slog.Info("Attempted queue ping", "url", url, "address", c.GetSQSEndpoint())
slog.Debug("Attempted queue ping", "url", url, "address", c.GetSQSEndpoint())
case <-ctx.Done():
return ctx.Err()
}
internal/test/aws.go
+13 -1
View File
@@ -3,8 +3,11 @@ package test
import (
"context"
"fmt"
"io"
"net/http"
"queryorchestration/internal/serviceconfig"
"testing"
"time"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
@@ -45,9 +48,18 @@ func CreateAWSContainer(t *testing.T, ctx context.Context, cfg *CreateAWSConfig)
},
ExposedPorts: []string{port.Port()},
WaitingFor: wait.ForAll(
// wait.ForExposedPort(),
wait.ForListeningPort(port),
wait.ForLog("Ready."),
wait.ForHTTP("/_localstack/health").
WithPort(port).
WithStartupTimeout(50*time.Second).
WithPollInterval(5*time.Second).
WithResponseMatcher(func(body io.Reader) bool {
return true
}).
WithStatusCodeMatcher(func(statusCode int) bool {
return statusCode == http.StatusOK
}),
),
}
internal/test/container.go
+1
View File
@@ -39,6 +39,7 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
"AWS_ACCESS_KEY_ID": cfg.Cfg.GetAWSKeyID(),
"AWS_SECRET_ACCESS_KEY": cfg.Cfg.GetAWSSecretKey(),
"AWS_REGION": cfg.Cfg.GetAWSRegion(),
"AWS_DEFAULT_REGION": cfg.Cfg.GetAWSRegion(),
"AWS_ENDPOINT_URL": cfg.Cfg.GetAWSEndpoint(),
"AWS_ENDPOINT_URL_SQS": cfg.Cfg.GetAWSEndpoint(),
"AWS_ENDPOINT_URL_S3": cfg.Cfg.GetAWSEndpoint(),
internal/test/database.go
+7
View File
@@ -7,6 +7,7 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/database"
"testing"
"time"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
@@ -43,6 +44,12 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (tes
wait.ForExposedPort(),
wait.ForListeningPort(port),
wait.ForLog("database system is ready to accept connections"),
wait.ForSQL(port, "postgres", func(host string, port nat.Port) string {
return fmt.Sprintf("postgres://%s:%s@localhost:%s/%s?sslmode=disable",
user, pass, port.Port(), name)
}).
WithStartupTimeout(25*time.Second).
WithPollInterval(10*time.Second),
),
}
internal/test/ecosystem.go
+13 -4
View File
@@ -68,12 +68,16 @@ func CreateRunnersAndServicesNetwork(t *testing.T, ctx context.Context, ncfg *Ec
qs := make(map[string]*Container, len(ncfg.Runners))
qclean := make([]func(), len(ncfg.Runners))
for i, r := range ncfg.Runners {
url := CreateQueue(t, ctx, ncfg.Cfg, r.Name)
url := r.QueueURL
if url == nil {
u := CreateQueue(t, ctx, ncfg.Cfg, r.Name)
url = &u
}
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Name: r.Name,
Env: r.Env,
QueueURL: url,
QueueURL: *url,
Cfg: ncfg.Cfg,
Network: network,
})
@@ -167,6 +171,7 @@ func CreateServiceNetwork(t *testing.T, ctx context.Context, scfg *ServiceNetwor
type RunnerNetworkConfig struct {
Network *testcontainers.DockerNetwork
AWSContainer *AWSContainerConfig
QueueURL *string
Cfg serviceconfig.ConfigProvider
Name Runner
Env map[string]string
@@ -194,7 +199,11 @@ func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkC
}
}
url := CreateQueue(t, ctx, rcfg.Cfg, rcfg.Name)
url := rcfg.QueueURL
if url == nil {
u := CreateQueue(t, ctx, rcfg.Cfg, rcfg.Name)
url = &u
}
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
@@ -204,7 +213,7 @@ func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkC
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Name: rcfg.Name,
Env: rcfg.Env,
QueueURL: url,
QueueURL: *url,
Cfg: rcfg.Cfg,
Network: network,
})
internal/test/runner.go
+2 -1
View File
@@ -8,6 +8,7 @@ import (
doctextrunner "queryorchestration/api/docTextRunner"
jobsyncrunner "queryorchestration/api/jobSyncRunner"
queryrunner "queryorchestration/api/queryRunner"
querysyncrunner "queryorchestration/api/querySyncRunner"
"testing"
"queryorchestration/internal/serviceconfig"
@@ -22,7 +23,7 @@ const (
DocSyncRunner = docsyncrunner.Name
DocCleanRunner = doccleanrunner.Name
DocTextRunner = doctextrunner.Name
QuerySyncRunner = queryrunner.Name
QuerySyncRunner = querysyncrunner.Name
QueryRunner = queryrunner.Name
JobSyncRunner = jobsyncrunner.Name
)
scripts/database.yml
+2 -9
View File
@@ -16,15 +16,9 @@ tasks:
cmds:
- task compose:clean:generate
- task db:generate
deps:up:
internal: true
lint:
cmds:
- task compose:up:generate
- sleep 3
lint:
deps:
- deps:up
cmds:
- sqlc vet --file sqlc.yml
mig:create:
cmds:
@@ -33,7 +27,6 @@ tasks:
test -z "$name" && echo "Error: Name is required." && exit 1
migrate create -ext sql -dir {{.MIGRATIONS}} $name
mig:
deps:
- deps:up
cmds:
- task compose:up:generate
- migrate -path {{.MIGRATIONS}} -database {{.DB_URI_GENERATE}} up
scripts/local-deployments.yml
+14 -4
View File
@@ -24,24 +24,34 @@ tasks:
COMPOSE_FILE: '{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}'
cmds:
- docker compose -f {{.COMPOSE_FILE}} build
up:cmd:
internal: true
vars:
COMPOSE_FILE: '{{.COMPOSE_FILE | default .LOCAL_COMPOSE_FILE}}'
cmds:
- docker compose -f {{.COMPOSE_FILE}} up --no-recreate --wait -d
up:test:
cmds:
- docker compose -f {{.TEST_COMPOSE_FILE}} up --no-recreate -d
- task: up:cmd
vars:
COMPOSE_FILE: '{{.TEST_COMPOSE_FILE}}'
up:generate:
cmds:
- docker compose -f {{.GENERATE_COMPOSE_FILE}} up --no-recreate -d
- task: up:cmd
vars:
COMPOSE_FILE: '{{.GENERATE_COMPOSE_FILE}}'
up:
cmds:
- task docker:build
- task: build
- docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d
- task: up:cmd
- task: init
refresh:
cmds:
- task docker:build
- task: build
- task: down
- docker compose -f {{.LOCAL_COMPOSE_FILE}} up --no-recreate -d
- task: up:cmd
- task: init
init:
cmds:
test/process_test.go
+27 -17
View File
@@ -26,7 +26,6 @@ type ProcessConfig struct {
}
func TestProcess(t *testing.T) {
t.SkipNow()
ctx := context.Background()
cfg := &ProcessConfig{}
@@ -46,6 +45,14 @@ func TestProcess(t *testing.T) {
assert.NoError(t, err)
test.CreateStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
dociniturl := test.CreateQueue(t, ctx, cfg, test.DocInitRunner)
docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner)
doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner)
doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner)
querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner)
queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner)
jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner)
bucketName := "docinitbucket"
test.CreateBucket(t, ctx, cfg, bucketName)
arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner)
@@ -64,55 +71,56 @@ func TestProcess(t *testing.T) {
})
assert.NoError(t, err)
docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner)
doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner)
doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner)
querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner)
queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner)
jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner)
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
Cfg: cfg,
Network: network,
Cfg: cfg,
Network: network,
AWSContainer: acfg,
Runners: []*test.RunnerNetworkConfig{
{
Name: test.DocInitRunner,
Name: test.DocInitRunner,
QueueURL: &dociniturl,
Env: map[string]string{
"DOCUMENT_SYNC_URL": docsyncurl,
},
},
{
Name: test.DocSyncRunner,
Name: test.DocSyncRunner,
QueueURL: &docsyncurl,
Env: map[string]string{
"DOCUMENT_CLEAN_URL": doccleanurl,
},
},
{
Name: test.DocCleanRunner,
Name: test.DocCleanRunner,
QueueURL: &doccleanurl,
Env: map[string]string{
"DOCUMENT_TEXT_URL": doctexturl,
},
},
{
Name: test.DocTextRunner,
Name: test.DocTextRunner,
QueueURL: &doctexturl,
Env: map[string]string{
"QUERY_SYNC_URL": querysyncurl,
},
},
{
Name: test.QuerySyncRunner,
Name: test.QuerySyncRunner,
QueueURL: &querysyncurl,
Env: map[string]string{
"QUERY_URL": queryurl,
},
},
{
Name: test.QueryRunner,
Name: test.QueryRunner,
QueueURL: &queryurl,
Env: map[string]string{
"QUERY_URL": queryurl,
},
},
{
Name: test.JobSyncRunner,
Name: test.JobSyncRunner,
QueueURL: &jobsyncurl,
Env: map[string]string{
"DOCUMENT_SYNC_URL": docsyncurl,
},
@@ -155,8 +163,10 @@ func TestProcess(t *testing.T) {
Type: queryservice.CONTEXTFULL,
})
assert.NoError(t, err)
jcfg := `{"path":"id"}`
jsonQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
Config: &jcfg,
RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
})
assert.NoError(t, err)
test/queryService/testquery_test.go
-45
View File
@@ -1,45 +0,0 @@
package endtoend_test
import (
"context"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
)
func TestQueryServiceTest(t *testing.T) {
ctx := context.Background()
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.NoError(t, err)
id := idRes.JSON201.Id
assert.NotEmpty(t, id)
docId := uuid.New()
testRes, err := client.TestQueryWithResponse(ctx, id, queryservice.QueryTestRequest{
DocumentId: docId,
QueryVersion: int32(1),
})
assert.NoError(t, err)
assert.NotNil(t, testRes)
// TODO
// assert.NotNil(t, testRes.JSON200)
// assert.Equal(t, "hey", testRes.JSON200.Value)
}