jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/clientexternalid (pull request #99)

Browse Source 
Client External ID

* normalizeexternalid

* cleanopenapi

* cleanopenapi

* changingpublic

* noprecommit

* testing

* precommit

* processtest

* nodb

* tests

* tests
This commit is contained in:
Michael McGuinness
2025-03-11 16:31:06 +00:00
parent 8e4d66a998
commit 6648cdf1cb
68 changed files with 5171 additions and 1106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/test/ecosystem.go
+156 -11
View File
@@ -2,11 +2,15 @@ package test
import (
"context"
"os"
"path"
"testing"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/database"
"queryorchestration/internal/serviceconfig/objectstore"
queryservice "queryorchestration/pkg/queryService"
"github.com/stretchr/testify/require"
"github.com/testcontainers/testcontainers-go"
@@ -22,12 +26,15 @@ type EcosystemNetworkConfig struct {
Runners []*RunnerNetworkConfig
Services []*ServiceNetworkConfig
Cfg serviceconfig.ConfigProvider
Network *testcontainers.DockerNetwork
AWSContainer *AWSContainerConfig
Dependencies *Dependencies
}
func CreateRunnersAndServicesNetwork(t testing.TB, ctx context.Context, ncfg *EcosystemNetworkConfig) (*EcosystemConfig, func()) {
network := ncfg.Network
if ncfg.Dependencies == nil {
ncfg.Dependencies = &Dependencies{}
}
network := ncfg.Dependencies.Network
var ncleanup func()
if network == nil {
network, ncleanup = CreateNetwork(t, ctx)
@@ -53,7 +60,7 @@ func CreateRunnersAndServicesNetwork(t testing.TB, ctx context.Context, ncfg *Ec
}
var qcleanup func()
if ncfg.AWSContainer == nil {
if ncfg.Dependencies.AWSConfig == nil {
_, qcleanup = CreateAWSContainer(t, ctx, &CreateAWSConfig{
Network: network,
Cfg: ncfg.Cfg,
@@ -61,23 +68,21 @@ func CreateRunnersAndServicesNetwork(t testing.TB, ctx context.Context, ncfg *Ec
}
if ncfg.Cfg.GetQueueClient() == nil {
err := ncfg.Cfg.SetQueueClient(ctx)
require.NoError(t, err)
SetQueueClient(t, ctx, ncfg.Cfg)
}
qs := make(map[string]*Container, len(ncfg.Runners))
qclean := make([]func(), len(ncfg.Runners))
for i, r := range ncfg.Runners {
url := r.QueueURL
if url == nil {
u := CreateQueue(t, ctx, ncfg.Cfg, r.Name)
url = &u
url := ncfg.Dependencies.QueueURLs[r.Name]
if url == "" {
url = CreateQueue(t, ctx, ncfg.Cfg, r.Name)
}
c, ccleanup := CreateRunner(t, ctx, &RunnerConfig{
Name: r.Name,
Env: r.Env,
QueueURL: *url,
QueueURL: url,
Cfg: ncfg.Cfg,
Network: network,
})
@@ -106,6 +111,12 @@ func CreateRunnersAndServicesNetwork(t testing.TB, ctx context.Context, ncfg *Ec
}
}
func SetCfgProviderWithBasePath(t testing.TB, cfg serviceconfig.ConfigProvider, basePath string) {
SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), basePath))
}
func SetCfgProvider(t testing.TB, cfg serviceconfig.ConfigProvider) {
cfg.SetAWSConfig(&aws.AWSConfig{
AWSKeyID: "test",
@@ -227,3 +238,137 @@ func CreateRunnerNetwork(t testing.TB, ctx context.Context, rcfg *RunnerNetworkC
}
}
}
type FullDependenciesConfig interface {
serviceconfig.ConfigProvider
objectstore.ConfigProvider
}
type Dependencies struct {
BucketName string
QueueURLs map[string]string
Network *testcontainers.DockerNetwork
AWSConfig *AWSContainerConfig
}
func CreateFullDependencies(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Dependencies, func()) {
network, ncleanup := CreateNetwork(t, ctx)
acfg, clean := CreateAWSContainer(t, ctx, &CreateAWSConfig{
Cfg: cfg,
Network: network,
})
SetQueueClient(t, ctx, cfg)
SetStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
urls := map[string]string{}
urls[DocInitRunner] = CreateQueue(t, ctx, cfg, DocInitRunner)
urls[DocSyncRunner] = CreateQueue(t, ctx, cfg, DocSyncRunner)
urls[DocCleanRunner] = CreateQueue(t, ctx, cfg, DocCleanRunner)
urls[DocTextRunner] = CreateQueue(t, ctx, cfg, DocTextRunner)
urls[QuerySyncRunner] = CreateQueue(t, ctx, cfg, QuerySyncRunner)
urls[QueryRunner] = CreateQueue(t, ctx, cfg, QueryRunner)
urls[ClientSyncRunner] = CreateQueue(t, ctx, cfg, ClientSyncRunner)
urls[QueryVersionSyncRunner] = CreateQueue(t, ctx, cfg, QueryVersionSyncRunner)
bucketName := "docinitbucket"
CreateBucket(t, ctx, cfg, bucketName)
SetBucketNotifs(t, ctx, cfg, bucketName)
return Dependencies{
BucketName: bucketName,
QueueURLs: urls,
Network: network,
AWSConfig: acfg,
}, func() {
ncleanup()
clean()
}
}
type Network struct {
Dependencies Dependencies
Network EcosystemConfig
Client *queryservice.ClientWithResponses
}
func CreateFullNetwork(t testing.TB, ctx context.Context, cfg FullDependenciesConfig) (Network, func()) {
deps, clean := CreateFullDependencies(t, ctx, cfg)
net, cleanup := CreateRunnersAndServicesNetwork(t, ctx, &EcosystemNetworkConfig{
Cfg: cfg,
Dependencies: &deps,
Runners: []*RunnerNetworkConfig{
{
Name: DocInitRunner,
Env: map[string]string{
"DOCUMENT_SYNC_URL": deps.QueueURLs[DocSyncRunner],
},
},
{
Name: DocSyncRunner,
Env: map[string]string{
"DOCUMENT_CLEAN_URL": deps.QueueURLs[DocCleanRunner],
},
},
{
Name: DocCleanRunner,
Env: map[string]string{
"DOCUMENT_TEXT_URL": deps.QueueURLs[DocTextRunner],
},
},
{
Name: DocTextRunner,
Env: map[string]string{
"QUERY_SYNC_URL": deps.QueueURLs[QuerySyncRunner],
},
},
{
Name: QuerySyncRunner,
Env: map[string]string{
"QUERY_URL": deps.QueueURLs[QueryRunner],
},
},
{
Name: QueryRunner,
Env: map[string]string{
"QUERY_URL": deps.QueueURLs[QueryRunner],
},
},
{
Name: ClientSyncRunner,
Env: map[string]string{
"DOCUMENT_SYNC_URL": deps.QueueURLs[DocSyncRunner],
},
},
{
Name: QueryVersionSyncRunner,
Env: map[string]string{
"CLIENT_SYNC_URL": deps.QueueURLs[ClientSyncRunner],
},
},
},
Services: []*ServiceNetworkConfig{
{
Name: QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": deps.QueueURLs[ClientSyncRunner],
"QUERY_VERSION_SYNC_URL": deps.QueueURLs[QueryVersionSyncRunner],
},
},
},
})
qService, err := queryservice.NewClientWithResponses(net.Services[QueryService].URI)
require.NoError(t, err)
return Network{
Dependencies: deps,
Network: *net,
Client: qService,
}, func() {
cleanup()
clean()
}
}