jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/testwithlogs (pull request #65)

Browse Source 
Query Version Sync Runner

* testing

* queryversiosyncworking

* update

* tests

* fixtests
This commit is contained in:
Michael McGuinness
2025-02-14 10:56:24 +00:00
parent 477518e5eb
commit 0df3d16976
91 changed files with 1737 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/aws.go
+13 -10
View File
@@ -25,7 +25,7 @@ type CreateAWSConfig struct {
Cfg serviceconfig.ConfigProvider
}
func CreateAWSContainer(t *testing.T, ctx context.Context, cfg *CreateAWSConfig) (*AWSContainerConfig, func()) {
func CreateAWSContainer(t testing.TB, ctx context.Context, cfg *CreateAWSConfig) (*AWSContainerConfig, func()) {
alias := "localstack"
port, err := nat.NewPort("tcp", "4566")
@@ -36,15 +36,18 @@ func CreateAWSContainer(t *testing.T, ctx context.Context, cfg *CreateAWSConfig)
req := testcontainers.ContainerRequest{
Image: "localstack/localstack:4.1.0",
Env: map[string]string{
"AWS_ACCESS_KEY_ID": cfg.Cfg.GetAWSKeyID(),
"AWS_SECRET_ACCESS_KEY": cfg.Cfg.GetAWSSecretKey(),
"AWS_SESSION_TOKEN": cfg.Cfg.GetAWSSessionToken(),
"AWS_REGION": cfg.Cfg.GetAWSRegion(),
"SERVICES": "s3,sqs",
"SKIP_SSL_CERT_DOWNLOAD": "1",
"LOCALSTACK_HOST": alias,
"SQS_ENDPOINT_STRATEGY": "path",
"EAGER_SERVICE_LOADING": "1",
"AWS_ACCESS_KEY_ID": cfg.Cfg.GetAWSKeyID(),
"AWS_SECRET_ACCESS_KEY": cfg.Cfg.GetAWSSecretKey(),
"AWS_SESSION_TOKEN": cfg.Cfg.GetAWSSessionToken(),
"AWS_REGION": cfg.Cfg.GetAWSRegion(),
"SERVICES": "s3,sqs,cloudwatch,logs",
"SKIP_SSL_CERT_DOWNLOAD": "1",
"LOCALSTACK_HOST": alias,
"SQS_ENDPOINT_STRATEGY": "path",
"EAGER_SERVICE_LOADING": "1",
"DEBUG": "1",
"LS_LOG": "trace",
"SQS_CLOUDWATCH_METRICS_REPORT_INTERVAL": "5",
},
ExposedPorts: []string{port.Port()},
WaitingFor: wait.ForAll(
internal/test/container.go
+1 -6
View File
@@ -28,7 +28,7 @@ type containerConfig struct {
ExposedPorts []nat.Port
}
func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (testcontainers.Container, func()) {
func createContainer(t testing.TB, ctx context.Context, cfg *containerConfig) (testcontainers.Container, func()) {
env := map[string]string{
"DB_USER": cfg.Cfg.GetDBUser(),
"DB_PASS": cfg.Cfg.GetDBSecret(),
@@ -65,11 +65,6 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
ports[index] = port.Port()
}
req.ExposedPorts = ports
// req.WaitingFor = wait.ForAll(
// // wait.ForExposedPort(),
// // wait.ForListeningPort(cfg.ExposedPorts[0]),
// wait.ForLog(cfg.WaitForMsg),
// )
}
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
internal/test/database.go
+1 -1
View File
@@ -20,7 +20,7 @@ type CreateDatabaseConfig struct {
RunMigrations bool
}
func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (testcontainers.Container, func()) {
func CreateDB(t testing.TB, ctx context.Context, cfg *CreateDatabaseConfig) (testcontainers.Container, func()) {
alias := "postgres"
port, err := nat.NewPort("tcp", "5432")
internal/test/ecosystem.go
+4 -4
View File
@@ -24,7 +24,7 @@ type EcosystemNetworkConfig struct {
AWSContainer *AWSContainerConfig
}
func CreateRunnersAndServicesNetwork(t *testing.T, ctx context.Context, ncfg *EcosystemNetworkConfig) (*EcosystemConfig, func()) {
func CreateRunnersAndServicesNetwork(t testing.TB, ctx context.Context, ncfg *EcosystemNetworkConfig) (*EcosystemConfig, func()) {
network := ncfg.Network
var ncleanup func()
if network == nil {
@@ -106,7 +106,7 @@ func CreateRunnersAndServicesNetwork(t *testing.T, ctx context.Context, ncfg *Ec
}
}
func SetCfgProvider(t *testing.T, cfg serviceconfig.ConfigProvider) {
func SetCfgProvider(t testing.TB, cfg serviceconfig.ConfigProvider) {
cfg.SetAWSConfig(&aws.AWSConfig{
AWSKeyID: "test",
AWSSecretKey: "test",
@@ -135,7 +135,7 @@ type ServiceNetworkConfig struct {
Env map[string]string
}
func CreateServiceNetwork(t *testing.T, ctx context.Context, scfg *ServiceNetworkConfig) (*Container, func()) {
func CreateServiceNetwork(t testing.TB, ctx context.Context, scfg *ServiceNetworkConfig) (*Container, func()) {
if scfg.Cfg == nil {
scfg.Cfg = &serviceconfig.BaseConfig{}
SetCfgProvider(t, scfg.Cfg)
@@ -177,7 +177,7 @@ type RunnerNetworkConfig struct {
Env map[string]string
}
func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkConfig) (*Container, func()) {
func CreateRunnerNetwork(t testing.TB, ctx context.Context, rcfg *RunnerNetworkConfig) (*Container, func()) {
network := rcfg.Network
var ncleanup func()
if network == nil {
internal/test/ecosystem_test.go
+6 -3
View File
@@ -22,7 +22,8 @@ func TestCreateRunnerNetwork(t *testing.T) {
Cfg: cfg,
Name: QueryRunner,
Env: map[string]string{
"QUERY_URL": "/i/am/here",
"QUERY_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "/here/there/every/where",
},
})
@@ -41,7 +42,8 @@ func TestCreateServiceNetwork(t *testing.T) {
conn, cleanup := CreateServiceNetwork(t, ctx, &ServiceNetworkConfig{
Name: QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
"JOB_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "/here/there/every/where",
},
})
@@ -74,7 +76,8 @@ func TestCreateRunnersAndServicesNetwork(t *testing.T) {
{
Name: QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
"JOB_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "/here/there/every/where",
},
},
},
internal/test/network.go
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/testcontainers/testcontainers-go/network"
)
func CreateNetwork(t *testing.T, ctx context.Context) (*testcontainers.DockerNetwork, func()) {
func CreateNetwork(t testing.TB, ctx context.Context) (*testcontainers.DockerNetwork, func()) {
network, err := network.New(ctx, network.WithDriver("bridge"))
if err != nil {
t.Fatal(err)
internal/test/objectstore.go
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func CreateBucket(t *testing.T, ctx context.Context, cfg objectstore.ConfigProvider, name string) {
func CreateBucket(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, name string) {
_, err := cfg.GetStoreClient().CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(name),
})
@@ -23,7 +23,7 @@ func CreateBucket(t *testing.T, ctx context.Context, cfg objectstore.ConfigProvi
}
}
func CreateStoreClient(t *testing.T, ctx context.Context, cfg objectstore.ConfigProvider, endpoint string) {
func CreateStoreClient(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, endpoint string) {
cfg.SetS3Endpoint(endpoint)
cfg.SetS3UsePathStyle(true)
err := cfg.SetStoreClient(ctx)
internal/test/queue.go
+4 -4
View File
@@ -13,7 +13,7 @@ import (
"github.com/stretchr/testify/assert"
)
func CreateQueue(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, name string) string {
func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, name string) string {
queueM, err := cfg.GetQueueClient().CreateQueue(ctx, &sqs.CreateQueueInput{
QueueName: aws.String(name),
})
@@ -29,7 +29,7 @@ func CreateQueue(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProv
return *queueM.QueueUrl
}
func AssertMessage(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, params *queue.ReceiveParams) types.Message {
func AssertMessage(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, params *queue.ReceiveParams) types.Message {
result, err := cfg.ReceiveFromQueue(ctx, params)
assert.NoError(t, err)
assert.NotNil(t, result.Messages)
@@ -39,7 +39,7 @@ func AssertMessage(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigPr
return result.Messages[0]
}
func AssertMessageBody(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, body *regexp.Regexp) {
func AssertMessageBody(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, body *regexp.Regexp) {
message := AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
QueueURL: url,
})
@@ -47,7 +47,7 @@ func AssertMessageBody(t *testing.T, ctx context.Context, cfg serviceconfig.Conf
assert.Regexp(t, body, *message.Body)
}
func AssertMessageAttr(t *testing.T, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, name string, value *regexp.Regexp) {
func AssertMessageAttr(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, url string, name string, value *regexp.Regexp) {
message := AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
QueueURL: url,
Attributes: []string{name},
internal/test/runner.go
+10 -8
View File
@@ -9,6 +9,7 @@ import (
jobsyncrunner "queryorchestration/api/jobSyncRunner"
queryrunner "queryorchestration/api/queryRunner"
querysyncrunner "queryorchestration/api/querySyncRunner"
queryversionsyncrunner "queryorchestration/api/queryVersionSyncRunner"
"testing"
"queryorchestration/internal/serviceconfig"
@@ -19,13 +20,14 @@ import (
type Runner = string
const (
DocInitRunner = docinitrunner.Name
DocSyncRunner = docsyncrunner.Name
DocCleanRunner = doccleanrunner.Name
DocTextRunner = doctextrunner.Name
QuerySyncRunner = querysyncrunner.Name
QueryRunner = queryrunner.Name
JobSyncRunner = jobsyncrunner.Name
DocInitRunner = docinitrunner.Name
DocSyncRunner = docsyncrunner.Name
DocCleanRunner = doccleanrunner.Name
DocTextRunner = doctextrunner.Name
QuerySyncRunner = querysyncrunner.Name
QueryRunner = queryrunner.Name
JobSyncRunner = jobsyncrunner.Name
QueryVersionSyncRunner = queryversionsyncrunner.Name
)
type RunnerConfig struct {
@@ -36,7 +38,7 @@ type RunnerConfig struct {
Env map[string]string
}
func CreateRunner(t *testing.T, ctx context.Context, config *RunnerConfig) (*Container, func()) {
func CreateRunner(t testing.TB, ctx context.Context, config *RunnerConfig) (*Container, func()) {
if config.Env == nil {
config.Env = map[string]string{}
}
internal/test/service.go
+6 -7
View File
@@ -24,19 +24,18 @@ type ServiceConfig struct {
Env map[string]string
}
func CreateService(t *testing.T, ctx context.Context, config *ServiceConfig) (*Container, func()) {
func CreateService(t testing.TB, ctx context.Context, config *ServiceConfig) (*Container, func()) {
port, err := nat.NewPort("tcp", "8080")
if err != nil {
t.Fatalf("Failed to create port: %v", err)
}
container, cleanup := createContainer(t, ctx, &containerConfig{
Cfg: config.Cfg,
Name: config.Name,
Network: config.Network,
Env: config.Env,
ExposedPorts: []nat.Port{port},
WaitForMsg: "⇨ http server started on [::]:8080",
Cfg: config.Cfg,
Name: config.Name,
Network: config.Network,
Env: config.Env,
WaitForMsg: "⇨ http server started on [::]:8080",
})
host, err := container.Host(ctx)
internal/test/service_test.go
+2 -1
View File
@@ -34,7 +34,8 @@ func TestCreateService(t *testing.T) {
Cfg: cfg,
Network: ncfg,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
"JOB_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "/here/there/every/where",
},
}