jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/tests (pull request #107)

Browse Source 
Fix Fullsuite

* save

* foundthefix..

* codeandrequire
This commit is contained in:
Michael McGuinness
2025-03-19 11:54:14 +00:00
parent 3dd0c718cd
commit 1587da9d11
101 changed files with 977 additions and 910 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/container.go
+17 -8
View File
@@ -47,6 +47,7 @@ func createContainer(t testing.TB, ctx context.Context, cfg *containerConfig) (t
"AWS_ENDPOINT_URL_SQS": cfg.Cfg.GetAWSEndpoint(),
"AWS_ENDPOINT_URL_S3": cfg.Cfg.GetAWSEndpoint(),
"AWS_S3_USE_PATH_STYLE": strconv.FormatBool(true),
"LOG_LEVEL": "DEBUG",
}
if cfg.Env != nil {
for k, v := range cfg.Env {
@@ -79,14 +80,7 @@ func createContainer(t testing.TB, ctx context.Context, cfg *containerConfig) (t
Started: true,
})
if err != nil {
logs, _ := container.Logs(ctx)
defer logs.Close()
scanner := bufio.NewScanner(logs)
for scanner.Scan() {
line := scanner.Text()
slog.Error(line)
}
PrintContainerLogs(t, ctx, container)
require.NoError(t, err)
}
@@ -98,3 +92,18 @@ func createContainer(t testing.TB, ctx context.Context, cfg *containerConfig) (t
}
}
}
func PrintContainerLogs(t testing.TB, ctx context.Context, container testcontainers.Container) {
logs, err := container.Logs(ctx)
defer func() {
err := logs.Close()
require.NoError(t, err)
}()
require.NoError(t, err)
scanner := bufio.NewScanner(logs)
for scanner.Scan() {
line := scanner.Text()
slog.Error(line)
}
}
internal/test/container_test.go
+2
View File
@@ -42,5 +42,7 @@ func TestCreateContainer(t *testing.T) {
assert.NotNil(t, container)
assert.NotNil(t, cleanup)
PrintContainerLogs(t, ctx, container)
cleanup()
}
internal/test/ecosystem.go
+105 -103
View File
@@ -16,6 +16,111 @@ import (
"github.com/testcontainers/testcontainers-go"
)
type Network struct {
Dependencies Dependencies
APIs map[string]*Container
Runners map[string]*Container
Client *queryapi.ClientWithResponses
}
func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Network, func()) {
deps, clean := CreateFullDependencies(t, ctx, cfg)
apiContainers := make(map[string]*Container, len(apis))
apiClean := make([]func(), len(apis))
for i, s := range apis {
c, ccleanup := CreateAPI(t, ctx, &APIConfig{
API: s,
Cfg: cfg,
Network: deps.Network,
})
apiContainers[s.Name] = c
apiClean[i] = ccleanup
}
qService, err := queryapi.NewClientWithResponses(apiContainers[QueryAPIName].URI)
require.NoError(t, err)
runnerContainers := make(map[string]*Container, len(runners))
runnerClean := make([]func(), len(runners))
for i, r := range runners {
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Runner: r,
Cfg: cfg,
Network: deps.Network,
})
runnerContainers[r.Name] = c
runnerClean[i] = ccleanup
}
return Network{
Dependencies: deps,
APIs: apiContainers,
Runners: runnerContainers,
Client: qService,
}, func() {
for _, c := range apiClean {
c()
}
for _, c := range runnerClean {
c()
}
clean()
}
}
type FullDependenciesConfig interface {
serviceconfig.ConfigProvider
objectstore.ConfigProvider
}
type Dependencies struct {
BucketName string
QueueURLs map[string]string
Network *testcontainers.DockerNetwork
AWSConfig *AWSContainerConfig
DBConfig testcontainers.Container
}
func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Dependencies, func()) {
network, ncleanup := CreateNetwork(t, ctx)
acfg, awsclean := CreateAWSContainer(t, ctx, &CreateAWSConfig{
Cfg: cfg,
Network: network,
})
dbcfg, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
Cfg: cfg,
RunMigrations: true,
})
SetQueueClient(t, ctx, cfg)
SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
urls := map[string]string{}
for _, runner := range runners {
urls[runner.Name] = CreateQueue(t, ctx, cfg, runner.Name)
}
CreateBucket(t, ctx, cfg, BucketName)
SetBucketNotifs(t, ctx, cfg, BucketName)
return Dependencies{
BucketName: BucketName,
QueueURLs: urls,
Network: network,
AWSConfig: acfg,
DBConfig: dbcfg,
}, func() {
awsclean()
dbcleanup()
ncleanup()
}
}
func SetCfgProviderWithBasePath(t testing.TB, cfg serviceconfig.ConfigProvider, basePath string) {
SetCfgProvider(t, cfg)
@@ -81,106 +186,3 @@ func CreateAPINetwork(t testing.TB, ctx context.Context, scfg *ServiceNetworkCon
}
}
}
type FullDependenciesConfig interface {
serviceconfig.ConfigProvider
objectstore.ConfigProvider
}
type Dependencies struct {
BucketName string
QueueURLs map[string]string
Network *testcontainers.DockerNetwork
AWSConfig *AWSContainerConfig
}
func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Dependencies, func()) {
network, ncleanup := CreateNetwork(t, ctx)
acfg, awsclean := CreateAWSContainer(t, ctx, &CreateAWSConfig{
Cfg: cfg,
Network: network,
})
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{
Network: network,
Cfg: cfg,
RunMigrations: true,
})
SetQueueClient(t, ctx, cfg)
SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
urls := map[string]string{}
for _, runner := range runners {
urls[runner.Name] = CreateQueue(t, ctx, cfg, runner.Name)
}
CreateBucket(t, ctx, cfg, BucketName)
SetBucketNotifs(t, ctx, cfg, BucketName)
return Dependencies{
BucketName: BucketName,
QueueURLs: urls,
Network: network,
AWSConfig: acfg,
}, func() {
awsclean()
dbcleanup()
ncleanup()
}
}
type Network struct {
Dependencies Dependencies
APIs map[string]*Container
Runners map[string]*Container
Client *queryapi.ClientWithResponses
}
func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Network, func()) {
deps, clean := CreateFullDependencies(t, ctx, cfg)
apiContainers := make(map[string]*Container, len(apis))
apiClean := make([]func(), len(apis))
for i, s := range apis {
c, ccleanup := CreateAPI(t, ctx, &APIConfig{
API: s,
Cfg: cfg,
Network: deps.Network,
})
apiContainers[s.Name] = c
apiClean[i] = ccleanup
}
qService, err := queryapi.NewClientWithResponses(apiContainers[QueryAPIName].URI)
require.NoError(t, err)
runnerContainers := make(map[string]*Container, len(runners))
runnerClean := make([]func(), len(runners))
for i, r := range runners {
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Runner: r,
Cfg: cfg,
Network: deps.Network,
})
runnerContainers[r.Name] = c
runnerClean[i] = ccleanup
}
return Network{
Dependencies: deps,
APIs: apiContainers,
Runners: runnerContainers,
Client: qService,
}, func() {
for _, c := range apiClean {
c()
}
for _, c := range runnerClean {
c()
}
clean()
}
}
internal/test/objectstore.go
+4 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"log/slog"
"testing"
objectstore "queryorchestration/internal/serviceconfig/objectstore"
@@ -20,6 +21,7 @@ func CreateBucket(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvi
Bucket: aws.String(name),
})
require.NoError(t, err)
slog.Info("create bucket", "bucket", name)
err = cfg.PingStoreByName(ctx, name)
require.NoError(t, err)
@@ -43,7 +45,7 @@ func SetBucketNotifs(t testing.TB, ctx context.Context, cfg objectstore.ConfigPr
},
})
require.NoError(t, err)
slog.Info("set bucket notifs", "bucket", name, "queueARN", arn)
}
func SetStoreClient(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider, endpoint string) {
@@ -61,4 +63,5 @@ func PutObject(t testing.TB, ctx context.Context, cfg objectstore.ConfigProvider
Body: file,
})
require.NoError(t, err)
slog.Info("put object", "bucket", bucket, "key", location)
}
internal/test/queryAPI/service.go
+6 -2
View File
@@ -3,6 +3,7 @@ package queryapitest
import (
"context"
"fmt"
"log/slog"
"testing"
"time"
@@ -36,7 +37,7 @@ func CreateClientWithSync(t testing.TB, ctx context.Context, client queryapi.Cli
func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.ClientWithResponsesInterface, id string, status queryapi.ClientStatus) {
t.Helper()
timeout := time.After(30 * time.Second)
timeout := time.After(60 * time.Second)
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
@@ -47,13 +48,16 @@ func WaitForClientStatus(t testing.TB, ctx context.Context, service queryapi.Cli
case <-ticker.C:
jRes, err := service.GetStatusByClientIdWithResponse(ctx, id)
if err != nil {
assert.NoError(t, err)
slog.Error("error getting status", "error", err.Error())
require.NoError(t, err)
}
if jRes.JSON200.Status == status {
assert.Equal(t, status, jRes.JSON200.Status)
return
}
slog.Error("unexpected status", "status", jRes.JSON200.Status)
}
}
}
internal/test/queue.go
+3 -1
View File
@@ -3,6 +3,7 @@ package test
import (
"context"
"fmt"
"log/slog"
"regexp"
"testing"
@@ -36,6 +37,7 @@ func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProv
QueueName: aws.String(name),
})
require.NoError(t, err)
slog.Info("create queue", "name", name, "url", *queueM.QueueUrl)
err = cfg.PingQueueByURL(ctx, *queueM.QueueUrl)
require.NoError(t, err)
@@ -45,7 +47,7 @@ func CreateQueue(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProv
func AssertMessage(t testing.TB, ctx context.Context, cfg serviceconfig.ConfigProvider, params *queue.ReceiveParams) types.Message {
result, err := cfg.ReceiveFromQueue(ctx, params)
assert.NoError(t, err)
require.NoError(t, err)
assert.NotNil(t, result.Messages)
assert.Len(t, result.Messages, 1)
assert.NotNil(t, result.Messages[0])
internal/test/queue_test.go
+8 -7
View File
@@ -13,6 +13,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateQueue(t *testing.T) {
@@ -30,7 +31,7 @@ func TestCreateQueue(t *testing.T) {
defer cleanup()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
require.NoError(t, err)
url := CreateQueue(t, ctx, cfg, "myname")
assert.Equal(t, "http://localstack:4566/queue/us-east-1/000000000000/myname", url)
@@ -50,14 +51,14 @@ func TestAssertMessageWait(t *testing.T) {
})
defer cleanup()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
require.NoError(t, err)
url := CreateQueue(t, ctx, cfg, "myname")
err = cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: url,
Body: "body",
})
assert.NoError(t, err)
require.NoError(t, err)
msg := AssertMessage(t, ctx, cfg, &queue.ReceiveParams{
QueueURL: url,
@@ -79,14 +80,14 @@ func TestAssertMessageBodyWait(t *testing.T) {
})
defer cleanup()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
require.NoError(t, err)
url := CreateQueue(t, ctx, cfg, "myname")
err = cfg.SendToQueue(ctx, &queue.SendParams{
QueueURL: url,
Body: "body",
})
assert.NoError(t, err)
require.NoError(t, err)
AssertMessageBody(t, ctx, cfg, url, regexp.MustCompile("\"body\""))
}
@@ -105,7 +106,7 @@ func TestAssertMessageAttrWait(t *testing.T) {
})
defer cleanup()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
require.NoError(t, err)
url := CreateQueue(t, ctx, cfg, "myname")
name := "name"
@@ -121,7 +122,7 @@ func TestAssertMessageAttrWait(t *testing.T) {
},
},
})
assert.NoError(t, err)
require.NoError(t, err)
AssertMessageAttr(t, ctx, cfg, url, name, regexp.MustCompile(value))
}
internal/test/runner_test.go
+2 -1
View File
@@ -7,6 +7,7 @@ import (
"queryorchestration/internal/serviceconfig"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateRunner(t *testing.T) {
@@ -27,7 +28,7 @@ func TestCreateRunner(t *testing.T) {
})
defer qcleanup()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
require.NoError(t, err)
_ = CreateQueue(t, ctx, cfg, QueryRunnerName)
_, dbcleanup := CreateDB(t, ctx, &CreateDatabaseConfig{