jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/docinitialisation (pull request #41)

Browse Source 
Queuing Changes and Cfg Testing

* staarting

* staarting

* startedpush

* note

* save

* mocking

* removederrs

* fixtests

* cleanuperrs

* newenvsetup

* preppingtests

* queue

* mmovetocfgpassunittests

* sortoutconfig

* passinginteg

* deps

* fixtests
This commit is contained in:
Michael McGuinness
2025-02-03 17:30:50 +00:00
parent 15adaebfcd
commit 7001ca854c
135 changed files with 3916 additions and 1523 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/docInitRunner/docinitrunner_test.go
+91
View File
@@ -0,0 +1,91 @@
package integration_test
import (
"context"
"os"
"path"
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/server/queue"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
"testing"
"github.com/aws/aws-sdk-go-v2/service/sqs/types"
"github.com/stretchr/testify/assert"
)
func TestDocInitRunner(t *testing.T) {
ctx := context.Background()
cfg := test.CreateBaseConfig()
cfg.BasePath = path.Join(os.Getenv("PWD"), "../..")
network, ncleanup := test.CreateNetwork(t, ctx)
defer ncleanup()
qcleanup := test.CreateQueueClient(t, ctx, &test.CreateQueueConfig{
Cfg: cfg,
Network: network,
})
defer qcleanup()
doccleanurl := test.CreateQueue(t, ctx, cfg, "docclean")
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
Cfg: cfg,
Network: network,
Runners: []*test.RunnerNetworkConfig{
{
Name: test.DocInitRunner,
Env: map[string]string{
"DOCUMENT_CLEAN_URL": doccleanurl,
},
},
},
Services: []*test.ServiceNetworkConfig{
{
Name: test.QueryService,
},
},
})
defer cleanup()
qService, err := queryservice.NewClientWithResponses(net.Services[test.QueryService].URI)
assert.NoError(t, err)
clientRes, err := qService.CreateClientWithResponse(ctx, queryservice.ClientCreate{
Name: "example_name",
})
assert.NoError(t, err)
jobRes, err := qService.CreateJobWithResponse(ctx, queryservice.JobCreate{
ClientId: clientRes.JSON201.Id,
})
assert.NoError(t, err)
canSync := true
_, err = qService.UpdateClientWithResponse(ctx, clientRes.JSON201.Id, queryservice.ClientUpdate{
CanSync: &canSync,
})
assert.NoError(t, err)
_, err = qService.UpdateJobWithResponse(ctx, jobRes.JSON201.Id, queryservice.JobUpdate{
CanSync: &canSync,
})
assert.NoError(t, err)
document := documentinit.Create{
JobID: jobRes.JSON201.Id,
Location: "/I/am/here", // TODO
}
qcfg := &queue.Config{
URL: net.Runners[test.DocInitRunner].URI,
Client: cfg.QueueClient,
}
err = queue.Send(ctx, qcfg, document, map[string]types.MessageAttributeValue{})
assert.NoError(t, err)
_ = test.AssertMessageWait(t, ctx, &queue.Config{
URL: doccleanurl,
Client: cfg.GetQueueClient(),
}, []string{})
}
test/queryRunner/queryrunner_test.go
+13 -10
View File
@@ -2,7 +2,6 @@ package integration_test
import (
"context"
"encoding/json"
"queryorchestration/internal/query"
"queryorchestration/internal/server/queue"
"queryorchestration/internal/test"
@@ -16,7 +15,12 @@ import (
func TestQueryRunner(t *testing.T) {
ctx := context.Background()
qCfg, cleanup := test.CreateQueueWithDependencies(t, ctx, "queryRunner")
cfg := test.CreateBaseConfig()
c, cleanup := test.CreateRunnerNetwork(t, ctx, &test.RunnerNetworkConfig{
Cfg: cfg,
Name: test.QueryRunner,
})
defer cleanup()
document := query.Document{
@@ -25,14 +29,13 @@ func TestQueryRunner(t *testing.T) {
CleanVersion: int32(1),
TextVersion: int32(1),
}
docJson, err := json.Marshal(document)
assert.Nil(t, err)
cfg := &queue.Config{
URL: qCfg.URL,
Client: qCfg.Client,
qcfg := &queue.Config{
URL: c.URI,
Client: cfg.QueueClient,
}
err = queue.Send(ctx, cfg, string(docJson), map[string]types.MessageAttributeValue{})
assert.Nil(t, err)
err := queue.Send(ctx, qcfg, document, map[string]types.MessageAttributeValue{})
assert.NoError(t, err)
// TODO - check document output
}
test/queryService/client_test.go
+9 -7
View File
@@ -12,23 +12,25 @@ import (
func TestClient(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{
Name: "example_name",
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
assert.NotNil(t, idRes.JSON201.Id)
id := idRes.JSON201.Id
clientRes, err := client.GetClientWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, clientRes.JSON200.Id)
assert.Equal(t, "example_name", clientRes.JSON200.Name)
assert.False(t, clientRes.JSON200.CanSync)
@@ -39,11 +41,11 @@ func TestClient(t *testing.T) {
Name: &updateName,
CanSync: &updateCanSync,
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, updateRes)
clientRes, err = client.GetClientWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, clientRes.JSON200.Id)
assert.Equal(t, updateName, clientRes.JSON200.Name)
assert.True(t, clientRes.JSON200.CanSync)
test/queryService/exportservice_test.go
+6 -4
View File
@@ -12,13 +12,15 @@ import (
func TestExportService(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.TriggerExportWithResponse(ctx, queryservice.ExportTrigger{})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, idRes)
}
test/queryService/job_test.go
+10 -8
View File
@@ -12,27 +12,29 @@ import (
func TestJob(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
clientRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{
Name: "example_name",
})
assert.Nil(t, err)
assert.NoError(t, err)
idRes, err := client.CreateJobWithResponse(ctx, queryservice.JobCreate{
ClientId: clientRes.JSON201.Id,
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
assert.NotNil(t, idRes.JSON201.Id)
id := idRes.JSON201.Id
jobRes, err := client.GetJobWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, jobRes.JSON200.Id)
assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId)
assert.False(t, jobRes.JSON200.CanSync)
@@ -41,11 +43,11 @@ func TestJob(t *testing.T) {
updateRes, err := client.UpdateJobWithResponse(ctx, id, queryservice.JobUpdate{
CanSync: &updateCanSync,
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, updateRes)
jobRes, err = client.GetJobWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, jobRes.JSON200.Id)
assert.Equal(t, clientRes.JSON201.Id, jobRes.JSON200.ClientId)
assert.False(t, jobRes.JSON200.CanSync)
test/queryService/jobcollectorservice_test.go
+12 -10
View File
@@ -14,16 +14,18 @@ func TestJobCollectorService(t *testing.T) {
t.SkipNow()
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
contextRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.Nil(t, err)
assert.NoError(t, err)
jsoncfg := "{\"path\":\"key\"}"
jsonRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
@@ -33,16 +35,16 @@ func TestJobCollectorService(t *testing.T) {
contextRes.JSON201.Id,
},
})
assert.Nil(t, err)
assert.NoError(t, err)
clientRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{
Name: "example_name",
})
assert.Nil(t, err)
assert.NoError(t, err)
jobRes, err := client.CreateJobWithResponse(ctx, queryservice.JobCreate{
ClientId: clientRes.JSON201.Id,
})
assert.Nil(t, err)
assert.NoError(t, err)
id := jobRes.JSON201.Id
fields := []queryservice.JobCollectorField{
@@ -53,7 +55,7 @@ func TestJobCollectorService(t *testing.T) {
}
collRes, err := client.GetJobCollectorByJobIdWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, collRes.JSON200.JobId)
assert.Equal(t, 1, collRes.JSON200.ActiveVersion)
assert.Equal(t, 1, collRes.JSON200.LatestVersion)
@@ -70,11 +72,11 @@ func TestJobCollectorService(t *testing.T) {
MinimumTextVersion: &minText,
Fields: &fields,
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.Nil(t, uRes)
collRes, err = client.GetJobCollectorByJobIdWithResponse(ctx, id)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, id, collRes.JSON200.JobId)
assert.Equal(t, 2, collRes.JSON200.ActiveVersion)
assert.Equal(t, 2, collRes.JSON200.LatestVersion)
test/queryService/openapi_test.go
+9 -7
View File
@@ -13,21 +13,23 @@ import (
func TestQueryServiceOpenAPI(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
resp, err := http.Get(fmt.Sprintf("%s/swagger/doc.json", address))
assert.Nil(t, err)
resp, err := http.Get(fmt.Sprintf("%s/swagger/doc.json", c.URI))
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = http.Get(fmt.Sprintf("%s/swagger/doc.yaml", address))
assert.Nil(t, err)
resp, err = http.Get(fmt.Sprintf("%s/swagger/doc.yaml", c.URI))
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = http.Get(fmt.Sprintf("%s/swagger/index.html", address))
assert.Nil(t, err)
resp, err = http.Get(fmt.Sprintf("%s/swagger/index.html", c.URI))
assert.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
}
test/queryService/queryservice_test.go
+11 -9
View File
@@ -13,16 +13,18 @@ import (
func TestQueryService(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
contextID := idRes.JSON201.Id
@@ -31,11 +33,11 @@ func TestQueryService(t *testing.T) {
idRes, err = client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
})
assert.Nil(t, err)
assert.NoError(t, err)
jsonID := idRes.JSON201.Id
queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
@@ -44,7 +46,7 @@ func TestQueryService(t *testing.T) {
assert.Nil(t, queryRes.JSON200.RequiredQueries)
queriesRes, err := client.ListQueriesWithResponse(ctx)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Len(t, queriesRes.JSON200.Queries, 2)
aV := int32(2)
@@ -54,11 +56,11 @@ func TestQueryService(t *testing.T) {
contextID,
},
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, res)
queryRes, err = client.GetQueryWithResponse(ctx, jsonID)
assert.Nil(t, err)
assert.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion)
test/queryService/testquery_test.go
+7 -5
View File
@@ -13,16 +13,18 @@ import (
func TestQueryServiceTest(t *testing.T) {
ctx := context.Background()
address, cleanup := test.CreateAPIWithDependencies(t, ctx, "queryService")
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(address)
assert.Nil(t, err)
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
})
assert.Nil(t, err)
assert.NoError(t, err)
id := idRes.JSON201.Id
assert.NotEmpty(t, id)
@@ -32,7 +34,7 @@ func TestQueryServiceTest(t *testing.T) {
DocumentId: docId,
QueryVersion: int32(1),
})
assert.Nil(t, err)
assert.NoError(t, err)
assert.NotNil(t, testRes)
// TODO
// assert.NotNil(t, testRes.JSON200)