jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/splitqueryrunning (pull request #57)

Browse Source 
Split Query Running + Debugging Full Flow

* completedquerysyncrunner

* spliitinglogic

* synccomplete

* informdependents

* only push same collector

* deps

* livetesting

* foundissue

* some issues resolved

* activeupdate

* collectorupdatefixes

* fix dbquesries

* tests

* tests

* pollingdebug
This commit is contained in:
Michael McGuinness
2025-02-11 15:22:59 +00:00
parent 24a038ec3d
commit 71f9802e1a
108 changed files with 3013 additions and 1666 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/process_test.go
+40 -9
View File
@@ -10,12 +10,12 @@ import (
documentcleanc "queryorchestration/internal/serviceconfig/queue/documentclean"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
"regexp"
"strings"
"testing"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
awstypes "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
)
@@ -51,12 +51,12 @@ func TestProcess(t *testing.T) {
arn := fmt.Sprintf("arn:aws:sqs:%s:000000000000:%s", cfg.AWSRegion, test.DocInitRunner)
_, err = cfg.StoreClient.PutBucketNotificationConfiguration(ctx, &s3.PutBucketNotificationConfigurationInput{
Bucket: &bucketName,
NotificationConfiguration: &types.NotificationConfiguration{
QueueConfigurations: []types.QueueConfiguration{
NotificationConfiguration: &awstypes.NotificationConfiguration{
QueueConfigurations: []awstypes.QueueConfiguration{
{
QueueArn: &arn,
Events: []types.Event{
types.EventS3ObjectCreated,
Events: []awstypes.Event{
awstypes.EventS3ObjectCreated,
},
},
},
@@ -67,6 +67,7 @@ func TestProcess(t *testing.T) {
doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner)
doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner)
querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner)
queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner)
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
Cfg: cfg,
@@ -90,6 +91,18 @@ func TestProcess(t *testing.T) {
"QUERY_SYNC_URL": querysyncurl,
},
},
{
Name: test.QuerySyncRunner,
Env: map[string]string{
"QUERY_URL": queryurl,
},
},
{
Name: test.QueryRunner,
Env: map[string]string{
"QUERY_URL": queryurl,
},
},
},
Services: []*test.ServiceNetworkConfig{
{
@@ -121,6 +134,27 @@ func TestProcess(t *testing.T) {
})
assert.NoError(t, err)
contextQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.NoError(t, err)
jsonQueryRes, err := qService.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
})
assert.NoError(t, err)
newActiveVersion := int32(2)
_, err = qService.UpdateJobCollectorByJobIdWithResponse(ctx, jobRes.JSON201.Id, queryservice.JobCollectorUpdate{
ActiveVersion: &newActiveVersion,
Fields: &[]queryservice.JobCollectorField{
{
Name: "JSON_QUERY",
QueryId: jsonQueryRes.JSON201.Id,
},
},
})
assert.NoError(t, err)
location := fmt.Sprintf("%s/%s/%s", clientRes.JSON201.Id, jobRes.JSON201.Id, "object_name")
body := strings.NewReader("hello world")
_, err = cfg.StoreClient.PutObject(ctx, &s3.PutObjectInput{
@@ -129,7 +163,4 @@ func TestProcess(t *testing.T) {
Body: body,
})
assert.NoError(t, err)
resRegex := regexp.MustCompile(`{"id": "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"`)
test.AssertMessageBody(t, ctx, cfg, querysyncurl, resRegex)
}