Merged in feature/clientexternalid (pull request #99)
Client External ID * normalizeexternalid * cleanopenapi * cleanopenapi * changingpublic * noprecommit * testing * precommit * processtest * nodb * tests * tests
This commit is contained in:
Michael McGuinness
+22
-157
@@ -3,20 +3,16 @@ package endtoend_test
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/serviceconfig/objectstore"
|
||||
documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean"
|
||||
"queryorchestration/internal/test"
|
||||
queryservicetest "queryorchestration/internal/test/queryService"
|
||||
queryservice "queryorchestration/pkg/queryService"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
awstypes "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/oapi-codegen/runtime/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -24,7 +20,6 @@ import (
|
||||
|
||||
type ProcessConfig struct {
|
||||
serviceconfig.BaseConfig
|
||||
documentcleanc.DocCleanConfig
|
||||
objectstore.ObjectStoreConfig
|
||||
}
|
||||
|
||||
@@ -32,150 +27,26 @@ func TestProcess(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
cfg := &ProcessConfig{}
|
||||
test.SetCfgProvider(t, cfg)
|
||||
cfg.SetBasePath(path.Join(os.Getenv("PWD"), ".."))
|
||||
test.SetCfgProviderWithBasePath(t, cfg, "..")
|
||||
|
||||
network, ncleanup := test.CreateNetwork(t, ctx)
|
||||
defer ncleanup()
|
||||
|
||||
acfg, clean := test.CreateAWSContainer(t, ctx, &test.CreateAWSConfig{
|
||||
Cfg: cfg,
|
||||
Network: network,
|
||||
})
|
||||
net, clean := test.CreateFullNetwork(t, ctx, cfg)
|
||||
defer clean()
|
||||
|
||||
err := cfg.SetQueueClient(ctx)
|
||||
require.NoError(t, err)
|
||||
test.CreateStoreClient(t, ctx, cfg, acfg.ExternalEndpoint)
|
||||
client := queryservicetest.CreateClientWithSync(t, ctx, net.Client)
|
||||
|
||||
dociniturl := test.CreateQueue(t, ctx, cfg, test.DocInitRunner)
|
||||
docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner)
|
||||
doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner)
|
||||
doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner)
|
||||
querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner)
|
||||
queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner)
|
||||
clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunner)
|
||||
queryversionsyncurl := test.CreateQueue(t, ctx, cfg, test.QueryVersionSyncRunner)
|
||||
|
||||
bucketName := "docinitbucket"
|
||||
test.CreateBucket(t, ctx, cfg, bucketName)
|
||||
arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner)
|
||||
_, err = cfg.StoreClient.PutBucketNotificationConfiguration(ctx, &s3.PutBucketNotificationConfigurationInput{
|
||||
Bucket: &bucketName,
|
||||
NotificationConfiguration: &awstypes.NotificationConfiguration{
|
||||
QueueConfigurations: []awstypes.QueueConfiguration{
|
||||
{
|
||||
QueueArn: &arn,
|
||||
Events: []awstypes.Event{
|
||||
awstypes.EventS3ObjectCreated,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
|
||||
Cfg: cfg,
|
||||
Network: network,
|
||||
AWSContainer: acfg,
|
||||
Runners: []*test.RunnerNetworkConfig{
|
||||
{
|
||||
Name: test.DocInitRunner,
|
||||
QueueURL: &dociniturl,
|
||||
Env: map[string]string{
|
||||
"DOCUMENT_SYNC_URL": docsyncurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.DocSyncRunner,
|
||||
QueueURL: &docsyncurl,
|
||||
Env: map[string]string{
|
||||
"DOCUMENT_CLEAN_URL": doccleanurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.DocCleanRunner,
|
||||
QueueURL: &doccleanurl,
|
||||
Env: map[string]string{
|
||||
"DOCUMENT_TEXT_URL": doctexturl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.DocTextRunner,
|
||||
QueueURL: &doctexturl,
|
||||
Env: map[string]string{
|
||||
"QUERY_SYNC_URL": querysyncurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.QuerySyncRunner,
|
||||
QueueURL: &querysyncurl,
|
||||
Env: map[string]string{
|
||||
"QUERY_URL": queryurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.QueryRunner,
|
||||
QueueURL: &queryurl,
|
||||
Env: map[string]string{
|
||||
"QUERY_URL": queryurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.ClientSyncRunner,
|
||||
QueueURL: &clientsyncurl,
|
||||
Env: map[string]string{
|
||||
"DOCUMENT_SYNC_URL": docsyncurl,
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: test.QueryVersionSyncRunner,
|
||||
QueueURL: &queryversionsyncurl,
|
||||
Env: map[string]string{
|
||||
"CLIENT_SYNC_URL": clientsyncurl,
|
||||
},
|
||||
},
|
||||
},
|
||||
Services: []*test.ServiceNetworkConfig{
|
||||
{
|
||||
Name: test.QueryService,
|
||||
Env: map[string]string{
|
||||
"CLIENT_SYNC_URL": clientsyncurl,
|
||||
"QUERY_VERSION_SYNC_URL": queryversionsyncurl,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
defer cleanup()
|
||||
|
||||
qService, err := queryservice.NewClientWithResponses(net.Services[test.QueryService].URI)
|
||||
require.NoError(t, err)
|
||||
|
||||
clientRes, err := qService.CreateClientWithResponse(ctx, queryservice.ClientCreate{
|
||||
Name: "example_name",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
canSync := true
|
||||
_, err = qService.UpdateClientWithResponse(ctx, clientRes.JSON201.Id, queryservice.ClientUpdate{
|
||||
CanSync: &canSync,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
contextQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
|
||||
contextQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
|
||||
Type: queryservice.CONTEXTFULL,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
require.NoError(t, err)
|
||||
jcfg := `{"path":"keyone"}`
|
||||
jsonQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
|
||||
jsonQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
|
||||
Type: queryservice.JSONEXTRACTOR,
|
||||
Config: &jcfg,
|
||||
RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
require.NoError(t, err)
|
||||
newActiveVersion := int32(1)
|
||||
_, err = qService.SetCollectorByClientIdWithResponse(ctx, clientRes.JSON201.Id, queryservice.CollectorSet{
|
||||
_, err = net.Client.SetCollectorByClientIdWithResponse(ctx, client.Id, queryservice.CollectorSet{
|
||||
ActiveVersion: &newActiveVersion,
|
||||
Fields: &[]queryservice.CollectorField{
|
||||
{
|
||||
@@ -184,30 +55,24 @@ func TestProcess(t *testing.T) {
|
||||
},
|
||||
},
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
require.NoError(t, err)
|
||||
|
||||
WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC)
|
||||
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
|
||||
|
||||
location := fmt.Sprintf("%s/%s", clientRes.JSON201.Id, "object_name")
|
||||
body := strings.NewReader(pdfHelloWorld)
|
||||
_, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucketName,
|
||||
Key: &location,
|
||||
Body: body,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
location := test.PutObject(t, ctx, cfg, client.Uid, net.Dependencies.BucketName, "object_name", body)
|
||||
|
||||
WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.NOTSYNCED)
|
||||
WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC)
|
||||
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.NOTSYNCED)
|
||||
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
|
||||
|
||||
docs, err := qService.ListDocumentsByClientIdWithResponse(ctx, clientRes.JSON201.Id)
|
||||
docs, err := net.Client.ListDocumentsByClientIdWithResponse(ctx, client.Id)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, *docs.JSON200, 1)
|
||||
doc := (*docs.JSON200)[0]
|
||||
assert.Equal(t, bucketName, doc.Bucket)
|
||||
assert.Equal(t, net.Dependencies.BucketName, doc.Bucket)
|
||||
assert.Equal(t, location, doc.Key)
|
||||
|
||||
testRes, err := qService.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
|
||||
testRes, err := net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
|
||||
QueryVersion: 1,
|
||||
DocumentId: doc.Id,
|
||||
})
|
||||
@@ -216,17 +81,17 @@ func TestProcess(t *testing.T) {
|
||||
|
||||
aV := int32(2)
|
||||
jcfg = `{"path":"keytwo"}`
|
||||
res, err := qService.UpdateQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryUpdate{
|
||||
res, err := net.Client.UpdateQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryUpdate{
|
||||
ActiveVersion: &aV,
|
||||
Config: &jcfg,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res)
|
||||
|
||||
WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.NOTSYNCED)
|
||||
WaitForClientStatus(t, ctx, qService, clientRes.JSON201.Id, queryservice.INSYNC)
|
||||
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.NOTSYNCED)
|
||||
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
|
||||
|
||||
testRes, err = qService.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
|
||||
testRes, err = net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
|
||||
QueryVersion: 2,
|
||||
DocumentId: doc.Id,
|
||||
})
|
||||
@@ -234,7 +99,7 @@ func TestProcess(t *testing.T) {
|
||||
assert.Equal(t, "valuetwo", testRes.JSON200.Value)
|
||||
}
|
||||
|
||||
func WaitForClientStatus(t testing.TB, ctx context.Context, service *queryservice.ClientWithResponses, id types.UUID, status queryservice.ClientStatus) {
|
||||
func WaitForClientStatus(t testing.TB, ctx context.Context, service *queryservice.ClientWithResponses, id string, status queryservice.ClientStatus) {
|
||||
t.Helper()
|
||||
|
||||
timeout := time.After(30 * time.Second)
|
||||
|
||||
Reference in New Issue
Block a user