jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/docinitialisation (pull request #41)

Browse Source 
Queuing Changes and Cfg Testing

* staarting

* staarting

* startedpush

* note

* save

* mocking

* removederrs

* fixtests

* cleanuperrs

* newenvsetup

* preppingtests

* queue

* mmovetocfgpassunittests

* sortoutconfig

* passinginteg

* deps

* fixtests
This commit is contained in:
Michael McGuinness
2025-02-03 17:30:50 +00:00
parent 15adaebfcd
commit 7001ca854c
135 changed files with 3916 additions and 1523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/api.go
-62
View File
@@ -1,62 +0,0 @@
package test
import (
"context"
"fmt"
"testing"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
)
type APIContainerConfig struct {
ServiceName string
DB *ExternalDatabase
Network *testcontainers.DockerNetwork
}
func CreateAPIContainer(t *testing.T, ctx context.Context, config *APIContainerConfig) (string, func()) {
port, err := nat.NewPort("tcp", "8080")
if err != nil {
t.Fatalf("Failed to create port: %v", err)
}
container, cleanup := createContainer(t, ctx, &containerConfig{
DB: config.DB,
ServiceName: config.ServiceName,
Network: config.Network,
ExposedPorts: []nat.Port{port},
WaitForMsg: "Listening for requests",
})
host, err := container.Host(ctx)
if err != nil {
t.Fatalf("Failed to extract host: %v", err)
}
mappedPort, err := container.MappedPort(ctx, port)
if err != nil {
t.Fatalf("Failed to extract port: %v", err)
}
return fmt.Sprintf("http://%s:%s", host, mappedPort.Port()), cleanup
}
func CreateAPIWithDependencies(t *testing.T, ctx context.Context, serviceName string) (string, func()) {
network, ncleanup := CreateNetwork(t, ctx)
dbconfig, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
})
address, ccleanup := CreateAPIContainer(t, ctx, &APIContainerConfig{
ServiceName: serviceName,
DB: dbconfig.External,
Network: network,
})
return address, func() {
ncleanup()
dbcleanup()
ccleanup()
}
}
internal/test/api_test.go
-56
View File
@@ -1,56 +0,0 @@
package test_test
import (
"context"
"os"
"path"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateAPIContainer(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
ncfg, ncleanup := test.CreateNetwork(t, ctx)
defer ncleanup()
t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../.."))
cfg := &serviceconfig.BaseConfig{}
dbcfg, dbcleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
RunMigrations: true,
})
defer dbcleanup()
acfg := &test.APIContainerConfig{
ServiceName: "queryService",
DB: dbcfg.External,
Network: ncfg,
}
conn, cleanup := test.CreateAPIContainer(t, ctx, acfg)
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateAPIWithDependencies(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
conn, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
internal/test/container.go
+35 -14
View File
@@ -1,8 +1,11 @@
package test
import (
"bufio"
"context"
"fmt"
"log/slog"
"queryorchestration/internal/serviceconfig"
"strconv"
"testing"
@@ -11,9 +14,14 @@ import (
"github.com/testcontainers/testcontainers-go/wait"
)
type Container struct {
Container testcontainers.Container
URI string
}
type containerConfig struct {
ServiceName string
DB *ExternalDatabase
Name string
Cfg *serviceconfig.BaseConfig
Network *testcontainers.DockerNetwork
Env map[string]string
WaitForMsg string
@@ -22,12 +30,16 @@ type containerConfig struct {
func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (testcontainers.Container, func()) {
env := map[string]string{
"DB_USER": cfg.DB.User,
"DB_PASS": cfg.DB.Password,
"DB_HOST": cfg.DB.Host,
"DB_NAME": cfg.DB.Name,
"DB_PORT": strconv.Itoa(cfg.DB.Port),
"DB_NOSSL": "true",
"DB_USER": cfg.Cfg.DBUser,
"DB_PASS": cfg.Cfg.DBSecret,
"DB_HOST": cfg.Cfg.DBHost,
"DB_NAME": cfg.Cfg.DBName,
"DB_PORT": strconv.Itoa(cfg.Cfg.DBPort),
"DB_NOSSL": "true",
"AWS_ACCESS_KEY_ID": cfg.Cfg.AWSKeyID,
"AWS_SECRET_ACCESS_KEY": cfg.Cfg.AWSRegion,
"AWS_REGION": cfg.Cfg.AWSSecretKey,
"AWS_ENDPOINT_URL_SQS": cfg.Cfg.AWSEndpointUrlSQS,
}
if cfg.Env != nil {
for k, v := range cfg.Env {
@@ -40,7 +52,7 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
Env: env,
Networks: []string{cfg.Network.Name},
WaitingFor: wait.ForLog(cfg.WaitForMsg),
Entrypoint: []string{fmt.Sprintf("./%s", cfg.ServiceName)},
Entrypoint: []string{fmt.Sprintf("./%s", cfg.Name)},
}
if len(cfg.ExposedPorts) > 0 {
@@ -49,11 +61,11 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
ports[index] = port.Port()
}
req.ExposedPorts = ports
req.WaitingFor = wait.ForAll(
// wait.ForExposedPort(),
// wait.ForListeningPort(cfg.ExposedPorts[0]),
wait.ForLog(cfg.WaitForMsg),
)
// req.WaitingFor = wait.ForAll(
// // wait.ForExposedPort(),
// // wait.ForListeningPort(cfg.ExposedPorts[0]),
// wait.ForLog(cfg.WaitForMsg),
// )
}
container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
@@ -61,6 +73,15 @@ func createContainer(t *testing.T, ctx context.Context, cfg *containerConfig) (t
Started: true,
})
if err != nil {
logs, _ := container.Logs(ctx)
defer logs.Close()
scanner := bufio.NewScanner(logs)
for scanner.Scan() {
line := scanner.Text()
slog.Error(line)
}
t.Fatalf("Failed to start container: %v", err)
}
internal/test/container_test.go
+8 -12
View File
@@ -2,9 +2,6 @@ package test
import (
"context"
"os"
"path"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/stretchr/testify/assert"
@@ -19,19 +16,18 @@ func TestCreateContainer(t *testing.T) {
ncfg, ncleanup := CreateNetwork(t, ctx)
defer ncleanup()
t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../.."))
cfg := &serviceconfig.BaseConfig{}
dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
RunMigrations: true,
cfg := CreateBaseConfig()
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
})
defer dbcleanup()
ccfg := &containerConfig{
ServiceName: "queryService",
DB: dbcfg.External,
Network: ncfg,
Name: QueryService,
Cfg: cfg,
Network: ncfg,
}
container, cleanup := createContainer(t, ctx, ccfg)
internal/test/database.go
+33 -43
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"queryorchestration/internal/database/migrations"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/database"
"testing"
"github.com/docker/go-connections/nat"
@@ -12,26 +13,13 @@ import (
"github.com/testcontainers/testcontainers-go/wait"
)
type ExternalDatabase struct {
Name string
Host string
Port int
User string
Password string
}
type Database struct {
Container testcontainers.Container
External *ExternalDatabase
}
type CreateDatabaseConfig struct {
Network *testcontainers.DockerNetwork
Cfg serviceconfig.ConfigProvider
RunMigrations bool
}
func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Database, func()) {
func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (testcontainers.Container, func()) {
alias := "postgres"
port, err := nat.NewPort("tcp", "5432")
@@ -82,47 +70,49 @@ func CreateDB(t *testing.T, ctx context.Context, cfg *CreateDatabaseConfig) (*Da
t.Fatalf("Failed to extract port: %v", err)
}
t.Setenv("DB_USER", user)
t.Setenv("DB_PASS", pass)
t.Setenv("DB_HOST", host)
t.Setenv("DB_PORT", fmt.Sprint(mappedPort.Int()))
t.Setenv("DB_NAME", name)
t.Setenv("DB_NOSSL", "true")
dbcfg := &database.DBConfig{
DBName: name,
DBSecret: pass,
DBUser: user,
DBPort: mappedPort.Int(),
DBHost: host,
DBNoSSL: true,
}
t.Setenv("DB_USER", dbcfg.DBUser)
t.Setenv("DB_PASS", dbcfg.DBSecret)
t.Setenv("DB_NAME", dbcfg.DBName)
t.Setenv("DB_NOSSL", fmt.Sprintf("%v", dbcfg.DBNoSSL))
t.Setenv("DB_HOST", dbcfg.DBHost)
t.Setenv("DB_PORT", fmt.Sprint(dbcfg.DBPort))
if cfg.Cfg != nil {
err = serviceconfig.InitializeConfig(cfg.Cfg)
if err != nil {
t.Fatal(err)
}
if cfg.RunMigrations {
cfg.Cfg.SetDBConfig(dbcfg)
err := migrations.Run(ctx, cfg.Cfg)
if err != nil {
t.Fatal(err)
}
err = cfg.Cfg.SetDBPool(ctx)
err = dbcfg.SetDBPool(ctx)
if err != nil {
t.Fatal(err)
}
}
}
external := &ExternalDatabase{
Name: name,
Password: pass,
User: user,
Port: port.Int(),
Host: alias,
}
return &Database{
Container: container,
External: external,
}, func() {
err := container.Terminate(ctx)
if err != nil {
t.Error(err)
}
if cfg.Network != nil {
dbcfg.DBHost = alias
dbcfg.DBPort = port.Int()
}
cfg.Cfg.SetDBConfig(dbcfg)
}
return container, func() {
err := container.Terminate(ctx)
if err != nil {
t.Error(err)
}
}
}
internal/test/database_test.go
+6 -4
View File
@@ -4,7 +4,6 @@ import (
"context"
"os"
"path"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"testing"
@@ -17,7 +16,9 @@ func TestCreateDB(t *testing.T) {
}
ctx := context.Background()
cfg := &serviceconfig.BaseConfig{}
cfg := test.CreateBaseConfig()
cfg.BasePath = path.Join(os.Getenv("PWD"), "../..")
dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{Cfg: cfg})
assert.NotNil(t, dbcfg)
assert.Nil(t, cfg.DBPool)
@@ -32,8 +33,9 @@ func TestCreateDBWithMigrations(t *testing.T) {
}
ctx := context.Background()
t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../.."))
cfg := &serviceconfig.BaseConfig{}
cfg := test.CreateBaseConfig()
cfg.BasePath = path.Join(os.Getenv("PWD"), "../..")
dbcfg, cleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Cfg: cfg,
RunMigrations: true,
internal/test/ecosystem.go
+195
View File
@@ -0,0 +1,195 @@
package test
import (
"context"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/testcontainers/testcontainers-go"
)
type EcosystemConfig struct {
Services map[Service]*Container
Runners map[Runner]*Container
Cfg *serviceconfig.BaseConfig
}
type EcosystemNetworkConfig struct {
Runners []*RunnerNetworkConfig
Services []*ServiceNetworkConfig
Cfg *serviceconfig.BaseConfig
Network *testcontainers.DockerNetwork
}
func CreateRunnersAndServicesNetwork(t *testing.T, ctx context.Context, ncfg *EcosystemNetworkConfig) (*EcosystemConfig, func()) {
if ncfg.Cfg == nil {
ncfg.Cfg = CreateBaseConfig()
}
network := ncfg.Network
var ncleanup func()
if network == nil {
network, ncleanup = CreateNetwork(t, ctx)
}
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
Cfg: ncfg.Cfg,
RunMigrations: true,
})
ss := make(map[string]*Container, len(ncfg.Services))
sclean := make([]func(), len(ncfg.Services))
for i, s := range ncfg.Services {
c, ccleanup := CreateService(t, ctx, &ServiceConfig{
Name: s.Name,
Env: s.Env,
Cfg: ncfg.Cfg,
Network: network,
})
ss[s.Name] = c
sclean[i] = ccleanup
}
var qcleanup func()
if ncfg.Cfg.QueueClient == nil {
qcleanup = CreateQueueClient(t, ctx, &CreateQueueConfig{
Network: network,
Cfg: ncfg.Cfg,
})
}
qs := make(map[string]*Container, len(ncfg.Runners))
qclean := make([]func(), len(ncfg.Runners))
for i, r := range ncfg.Runners {
url := CreateQueue(t, ctx, ncfg.Cfg, r.Name)
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Name: r.Name,
Env: r.Env,
QueueURL: url,
Cfg: ncfg.Cfg,
Network: network,
})
qs[r.Name] = c
qclean[i] = ccleanup
}
return &EcosystemConfig{
Services: ss,
Runners: qs,
}, func() {
for _, c := range sclean {
c()
}
for _, c := range qclean {
c()
}
dbcleanup()
if qcleanup != nil {
qcleanup()
}
if ncleanup != nil {
ncleanup()
}
}
}
func CreateBaseConfig() *serviceconfig.BaseConfig {
cfg := &serviceconfig.BaseConfig{}
cfg.AWSKeyID = "test"
cfg.AWSSecretKey = "test"
cfg.AWSRegion = "us-east-1"
cfg.DBUser = "invalid_user"
cfg.DBSecret = "invalid_pass"
cfg.DBHost = "invalid_host"
cfg.DBPort = 5432
cfg.DBName = "invalid_name"
cfg.DBNoSSL = true
return cfg
}
type ServiceNetworkConfig struct {
Name Runner
Env map[string]string
}
func CreateServiceNetwork(t *testing.T, ctx context.Context, scfg *ServiceNetworkConfig) (*Container, func()) {
cfg := CreateBaseConfig()
network, ncleanup := CreateNetwork(t, ctx)
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
Cfg: cfg,
})
c, ccleanup := CreateService(t, ctx, &ServiceConfig{
Name: scfg.Name,
Env: scfg.Env,
Cfg: cfg,
Network: network,
})
return c, func() {
dbcleanup()
ccleanup()
ncleanup()
}
}
type RunnerNetworkConfig struct {
Network *testcontainers.DockerNetwork
Cfg *serviceconfig.BaseConfig
Name Runner
Env map[string]string
}
func CreateRunnerNetwork(t *testing.T, ctx context.Context, rcfg *RunnerNetworkConfig) (*Container, func()) {
if rcfg.Cfg == nil {
rcfg.Cfg = CreateBaseConfig()
}
network := rcfg.Network
var ncleanup func()
if network == nil {
network, ncleanup = CreateNetwork(t, ctx)
}
var qcleanup func()
if rcfg.Cfg.QueueClient == nil {
qcleanup = CreateQueueClient(t, ctx, &CreateQueueConfig{
Network: network,
Cfg: rcfg.Cfg,
})
}
url := CreateQueue(t, ctx, rcfg.Cfg, rcfg.Name)
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
Cfg: rcfg.Cfg,
})
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Name: rcfg.Name,
Env: rcfg.Env,
QueueURL: url,
Cfg: rcfg.Cfg,
Network: network,
})
return c, func() {
ccleanup()
dbcleanup()
if qcleanup != nil {
qcleanup()
}
if ncleanup != nil {
ncleanup()
}
}
}
internal/test/ecosystem_test.go
+79
View File
@@ -0,0 +1,79 @@
package test
import (
"context"
"os"
"path"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateRunnerNetwork(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
conn, cleanup := CreateRunnerNetwork(t, ctx, &RunnerNetworkConfig{
Name: QueryRunner,
})
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateServiceNetwork(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
conn, cleanup := CreateServiceNetwork(t, ctx, &ServiceNetworkConfig{
Name: QueryService,
})
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateRunnersAndServicesNetwork(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := CreateBaseConfig()
cfg.BasePath = path.Join(os.Getenv("PWD"), "../..")
conn, cleanup := CreateRunnersAndServicesNetwork(t, ctx, &EcosystemNetworkConfig{
Cfg: cfg,
Runners: []*RunnerNetworkConfig{
{Name: QueryRunner},
},
Services: []*ServiceNetworkConfig{
{Name: QueryService},
},
})
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateBaseConfig(t *testing.T) {
cfg := CreateBaseConfig()
assert.Equal(t, "test", cfg.AWSKeyID)
assert.Equal(t, "test", cfg.AWSSecretKey)
assert.Equal(t, "us-east-1", cfg.AWSRegion)
assert.Equal(t, "invalid_user", cfg.DBUser)
assert.Equal(t, "invalid_pass", cfg.DBSecret)
assert.Equal(t, "invalid_host", cfg.DBHost)
assert.Equal(t, 5432, cfg.DBPort)
assert.Equal(t, "invalid_name", cfg.DBName)
assert.True(t, cfg.DBNoSSL)
}
internal/test/queue.go
+50 -44
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"queryorchestration/internal/server/queue"
"queryorchestration/internal/serviceconfig"
"testing"
"time"
@@ -18,29 +19,20 @@ import (
"github.com/testcontainers/testcontainers-go/wait"
)
type QueueConfig struct {
Container testcontainers.Container
Client *sqs.Client
URL string
External *ExternalQueue
}
type ExternalQueue struct {
Endpoint string
Credentials aws.CredentialsProvider
Region string
URL string
type queueContainerConfig struct {
Container testcontainers.Container
ExternalEndpoint string
NetworkEndpoint string
}
type CreateQueueConfig struct {
Network *testcontainers.DockerNetwork
Cfg *serviceconfig.BaseConfig
}
func CreateQueue(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*QueueConfig, func()) {
queueName := "test-queue"
region := "us-east-1"
func createQueueContainer(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*queueContainerConfig, func()) {
alias := "localstack"
provider := credentials.NewStaticCredentialsProvider("test", "test", "")
provider := credentials.NewStaticCredentialsProvider(cfg.Cfg.AWSKeyID, cfg.Cfg.AWSSecretKey, cfg.Cfg.AWSSessionToken)
port, err := nat.NewPort("tcp", "4566")
if err != nil {
@@ -53,8 +45,7 @@ func CreateQueue(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*Qu
"AWS_ACCESS_KEY_ID": provider.Value.AccessKeyID,
"AWS_SECRET_ACCESS_KEY": provider.Value.SecretAccessKey,
"AWS_SESSION_TOKEN": provider.Value.SessionToken,
"AWS_DEFAULT_REGION": region,
"AWS_REGION": region,
"AWS_REGION": cfg.Cfg.AWSRegion,
"SERVICES": "sqs",
"SKIP_SSL_CERT_DOWNLOAD": "1",
"LOCALSTACK_HOST": alias,
@@ -92,32 +83,20 @@ func CreateQueue(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*Qu
t.Fatalf("Failed to extract port: %v", err)
}
endpoint := fmt.Sprintf("http://%s:%s", host, mappedPort.Port())
sqsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region), config.WithCredentialsProvider(provider), config.WithBaseEndpoint(endpoint))
if err != nil {
t.Fatal(err)
extEndpoint := fmt.Sprintf("http://%s:%s", host, mappedPort.Port())
t.Setenv("AWS_ENDPOINT_URL_SQS", extEndpoint)
cfg.Cfg.AWSEndpointUrlSQS = extEndpoint
var endpoint string
if cfg.Network != nil {
endpoint = fmt.Sprintf("http://%s:%s", alias, port.Port())
cfg.Cfg.AWSEndpointUrlSQS = endpoint
}
client := sqs.NewFromConfig(sqsCfg)
queueM, err := client.CreateQueue(ctx, &sqs.CreateQueueInput{
QueueName: aws.String(queueName),
})
if err != nil {
t.Fatal(err)
}
endpoint = fmt.Sprintf("http://%s:%s", alias, port.Port())
return &QueueConfig{
Container: container,
Client: client,
URL: *queueM.QueueUrl,
External: &ExternalQueue{
Region: region,
Credentials: provider,
Endpoint: endpoint,
},
return &queueContainerConfig{
Container: container,
ExternalEndpoint: extEndpoint,
NetworkEndpoint: endpoint,
}, func() {
err := container.Terminate(ctx)
if err != nil {
@@ -126,11 +105,38 @@ func CreateQueue(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) (*Qu
}
}
func CreateQueueClient(t *testing.T, ctx context.Context, cfg *CreateQueueConfig) func() {
qcfg, clean := createQueueContainer(t, ctx, cfg)
provider := credentials.NewStaticCredentialsProvider(cfg.Cfg.AWSKeyID, cfg.Cfg.AWSSecretKey, cfg.Cfg.AWSSessionToken)
sqsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(cfg.Cfg.AWSRegion), config.WithCredentialsProvider(provider), config.WithBaseEndpoint(qcfg.ExternalEndpoint))
if err != nil {
t.Fatal(err)
}
cfg.Cfg.QueueClient = sqs.NewFromConfig(sqsCfg)
return clean
}
func CreateQueue(t *testing.T, ctx context.Context, cfg *serviceconfig.BaseConfig, name string) string {
queueM, err := cfg.QueueClient.CreateQueue(ctx, &sqs.CreateQueueInput{
QueueName: aws.String(name),
})
if err != nil {
t.Fatal(err)
}
return *queueM.QueueUrl
}
func AssertMessageWait(t *testing.T, ctx context.Context, cfg *queue.Config, attrs []string) types.Message {
time.Sleep(time.Second)
time.Sleep(1 * time.Second)
result, err := queue.Receive(ctx, cfg, attrs)
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, result.Messages)
assert.Len(t, result.Messages, 1)
assert.NotNil(t, result.Messages[0])
internal/test/queue_test.go
+80 -30
View File
@@ -1,9 +1,8 @@
package test_test
package test
import (
"context"
"queryorchestration/internal/server/queue"
"queryorchestration/internal/test"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -11,17 +10,53 @@ import (
"github.com/stretchr/testify/assert"
)
func TestCreateQueueContainer(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := CreateBaseConfig()
qcfg, cleanup := createQueueContainer(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
assert.NotNil(t, qcfg)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateQueueClient(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
cfg := CreateBaseConfig()
cleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
assert.NotNil(t, cfg.QueueClient)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateQueue(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
qcfg, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{})
assert.NotNil(t, qcfg)
assert.NotNil(t, cleanup)
cfg := CreateBaseConfig()
cleanup()
cleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
defer cleanup()
url := CreateQueue(t, ctx, cfg, "myname")
assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/myname", url)
}
func TestAssertMessageWait(t *testing.T) {
@@ -30,18 +65,23 @@ func TestAssertMessageWait(t *testing.T) {
}
ctx := context.Background()
qcfg, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{})
defer cleanup()
cfg := CreateBaseConfig()
cfg := &queue.Config{
URL: qcfg.URL,
Client: qcfg.Client,
cleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
defer cleanup()
url := CreateQueue(t, ctx, cfg, "myname")
qucfg := &queue.Config{
URL: url,
Client: cfg.QueueClient,
}
err := queue.Send(ctx, cfg, "body", map[string]types.MessageAttributeValue{})
assert.Nil(t, err)
err := queue.Send(ctx, qucfg, "body", map[string]types.MessageAttributeValue{})
assert.NoError(t, err)
msg := test.AssertMessageWait(t, ctx, cfg, []string{})
msg := AssertMessageWait(t, ctx, qucfg, []string{})
assert.NotNil(t, msg)
}
@@ -51,18 +91,23 @@ func TestAssertMessageBodyWait(t *testing.T) {
}
ctx := context.Background()
qcfg, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{})
defer cleanup()
cfg := CreateBaseConfig()
cfg := &queue.Config{
URL: qcfg.URL,
Client: qcfg.Client,
cleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
defer cleanup()
url := CreateQueue(t, ctx, cfg, "myname")
qucfg := &queue.Config{
URL: url,
Client: cfg.QueueClient,
}
err := queue.Send(ctx, cfg, "body", map[string]types.MessageAttributeValue{})
assert.Nil(t, err)
err := queue.Send(ctx, qucfg, "body", map[string]types.MessageAttributeValue{})
assert.NoError(t, err)
test.AssertMessageBodyWait(t, ctx, cfg, "\"body\"")
AssertMessageBodyWait(t, ctx, qucfg, "\"body\"")
}
func TestAssertMessageAttrWait(t *testing.T) {
@@ -71,24 +116,29 @@ func TestAssertMessageAttrWait(t *testing.T) {
}
ctx := context.Background()
qcfg, cleanup := test.CreateQueue(t, ctx, &test.CreateQueueConfig{})
defer cleanup()
cfg := CreateBaseConfig()
cfg := &queue.Config{
URL: qcfg.URL,
Client: qcfg.Client,
cleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Cfg: cfg,
})
defer cleanup()
url := CreateQueue(t, ctx, cfg, "myname")
qucfg := &queue.Config{
URL: url,
Client: cfg.QueueClient,
}
name := "name"
value := "value"
err := queue.Send(ctx, cfg, "body", map[string]types.MessageAttributeValue{
err := queue.Send(ctx, qucfg, "body", map[string]types.MessageAttributeValue{
name: {
DataType: aws.String("String"),
StringValue: aws.String(value),
},
})
assert.Nil(t, err)
assert.NoError(t, err)
test.AssertMessageAttrWait(t, ctx, cfg, name, value)
AssertMessageAttrWait(t, ctx, qucfg, name, value)
}
internal/test/queuecontainer.go
-81
View File
@@ -1,81 +0,0 @@
package test
import (
"context"
"testing"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/testcontainers/testcontainers-go"
)
type QueueContainerConfig struct {
ServiceName string
Queue *queueConfig
DB *ExternalDatabase
Network *testcontainers.DockerNetwork
}
func CreateQueueContainer(t *testing.T, ctx context.Context, config *QueueContainerConfig) func() {
queueCredentials, err := config.Queue.Credentials.Retrieve(ctx)
if err != nil {
t.Fatal(err)
}
_, cleanup := createContainer(t, ctx, &containerConfig{
Network: config.Network,
ServiceName: config.ServiceName,
DB: config.DB,
WaitForMsg: "Listening to queue",
Env: map[string]string{
"QUEUE_URL": config.Queue.URL,
"AWS_DEFAULT_REGION": config.Queue.Region,
"AWS_REGION": config.Queue.Region,
"AWS_ACCESS_KEY_ID": queueCredentials.AccessKeyID,
"AWS_SECRET_ACCESS_KEY": queueCredentials.SecretAccessKey,
"AWS_SESSION_TOKEN": queueCredentials.SessionToken,
"AWS_ENDPOINT_URL_SQS": config.Queue.Endpoint,
},
})
return cleanup
}
type queueConfig struct {
URL string
Region string
Endpoint string
Credentials aws.CredentialsProvider
}
func CreateQueueWithDependencies(t *testing.T, ctx context.Context, serviceName string) (*QueueConfig, func()) {
network, ncleanup := CreateNetwork(t, ctx)
dbconfig, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
})
qCfg, qcleanup := CreateQueue(t, ctx, &CreateQueueConfig{
Network: network,
})
cfg := &queueConfig{
URL: qCfg.URL,
Region: qCfg.External.Region,
Endpoint: qCfg.External.Endpoint,
Credentials: qCfg.External.Credentials,
}
ccleanup := CreateQueueContainer(t, ctx, &QueueContainerConfig{
ServiceName: serviceName,
Queue: cfg,
DB: dbconfig.External,
Network: network,
})
return qCfg, func() {
ncleanup()
dbcleanup()
qcleanup()
ccleanup()
}
}
internal/test/queuecontainer_test.go
-67
View File
@@ -1,67 +0,0 @@
package test
import (
"context"
"os"
"path"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateQueueContainer(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
ncfg, ncleanup := CreateNetwork(t, ctx)
defer ncleanup()
t.Setenv("BASE_PATH", path.Join(os.Getenv("PWD"), "../.."))
cfg := &serviceconfig.BaseConfig{}
dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
RunMigrations: true,
})
defer dbcleanup()
qcfg, qcleanup := CreateQueue(t, ctx, &CreateQueueConfig{
Network: ncfg,
})
defer qcleanup()
extcfg := &queueConfig{
URL: qcfg.URL,
Region: qcfg.External.Region,
Endpoint: qcfg.External.Endpoint,
Credentials: qcfg.External.Credentials,
}
qccfg := &QueueContainerConfig{
ServiceName: "queryRunner",
DB: dbcfg.External,
Network: ncfg,
Queue: extcfg,
}
cleanup := CreateQueueContainer(t, ctx, qccfg)
assert.NotNil(t, cleanup)
cleanup()
}
func TestCreateQueueWithDependencies(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
conn, cleanup := CreateQueueWithDependencies(t, ctx, "queryRunner")
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}
internal/test/runner.go
+47
View File
@@ -0,0 +1,47 @@
package test
import (
"context"
docinitrunner "queryorchestration/api/docInitRunner"
queryrunner "queryorchestration/api/queryRunner"
"testing"
"queryorchestration/internal/serviceconfig"
"github.com/testcontainers/testcontainers-go"
)
type Runner = string
const (
QueryRunner = queryrunner.Name
DocInitRunner = docinitrunner.Name
)
type RunnerConfig struct {
Name Runner
QueueURL string
Cfg *serviceconfig.BaseConfig
Network *testcontainers.DockerNetwork
Env map[string]string
}
func CreateRunner(t *testing.T, ctx context.Context, config *RunnerConfig) (*Container, func()) {
if config.Env == nil {
config.Env = map[string]string{}
}
config.Env["QUEUE_URL"] = config.QueueURL
c, cleanup := createContainer(t, ctx, &containerConfig{
Network: config.Network,
Name: config.Name,
Cfg: config.Cfg,
Env: config.Env,
WaitForMsg: "Listening to queue",
})
return &Container{
Container: c,
URI: config.QueueURL,
}, cleanup
}
internal/test/runner_test.go
+46
View File
@@ -0,0 +1,46 @@
package test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateRunner(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
ncfg, ncleanup := CreateNetwork(t, ctx)
defer ncleanup()
cfg := CreateBaseConfig()
qcleanup := CreateQueueClient(t, ctx, &CreateQueueConfig{
Network: ncfg,
Cfg: cfg,
})
defer qcleanup()
url := CreateQueue(t, ctx, cfg, "myname")
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
})
defer dbcleanup()
qccfg := &RunnerConfig{
Name: QueryRunner,
Cfg: cfg,
Network: ncfg,
QueueURL: url,
}
c, cleanup := CreateRunner(t, ctx, qccfg)
assert.NotNil(t, cleanup)
assert.NotNil(t, c)
cleanup()
}
internal/test/service.go
+57
View File
@@ -0,0 +1,57 @@
package test
import (
"context"
"fmt"
queryservice "queryorchestration/api/queryService"
"queryorchestration/internal/serviceconfig"
"testing"
"github.com/docker/go-connections/nat"
"github.com/testcontainers/testcontainers-go"
)
type Service = string
const (
QueryService = queryservice.Name
)
type ServiceConfig struct {
Name Service
Cfg *serviceconfig.BaseConfig
Network *testcontainers.DockerNetwork
Env map[string]string
}
func CreateService(t *testing.T, ctx context.Context, config *ServiceConfig) (*Container, func()) {
port, err := nat.NewPort("tcp", "8080")
if err != nil {
t.Fatalf("Failed to create port: %v", err)
}
container, cleanup := createContainer(t, ctx, &containerConfig{
Cfg: config.Cfg,
Name: config.Name,
Network: config.Network,
Env: config.Env,
ExposedPorts: []nat.Port{port},
WaitForMsg: "⇨ http server started on [::]:8080",
})
host, err := container.Host(ctx)
if err != nil {
t.Fatalf("Failed to extract host: %v", err)
}
mappedPort, err := container.MappedPort(ctx, port)
if err != nil {
t.Fatalf("Failed to extract port: %v", err)
}
address := fmt.Sprintf("http://%s:%s", host, mappedPort.Port())
return &Container{
URI: address,
Container: container,
}, cleanup
}
internal/test/service_test.go
+41
View File
@@ -0,0 +1,41 @@
package test_test
import (
"context"
"os"
"path"
"queryorchestration/internal/test"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateService(t *testing.T) {
if testing.Short() {
t.Skip("Skipping long test in short mode")
}
ctx := context.Background()
ncfg, ncleanup := test.CreateNetwork(t, ctx)
defer ncleanup()
cfg := test.CreateBaseConfig()
cfg.BasePath = path.Join(os.Getenv("PWD"), "../..")
_, dbcleanup := test.CreateDB(t, ctx, &test.CreateDatabaseConfig{
Network: ncfg,
Cfg: cfg,
})
defer dbcleanup()
acfg := &test.ServiceConfig{
Name: test.QueryService,
Cfg: cfg,
Network: ncfg,
}
conn, cleanup := test.CreateService(t, ctx, acfg)
assert.NotNil(t, conn)
assert.NotNil(t, cleanup)
cleanup()
}