jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/jobsync (pull request #63)

Browse Source 
Document Sync and Job Sync

* docsyncfunc

* startedQueryWork

* morefixes

* jobsyncsvcstart

* save

* jobsynctext

* save

* save

* docsync

* shorttest

* jobsync

* jobsyncupdateoncollectorupdate

* passlivetest

* collectortest

* lint
This commit is contained in:
Michael McGuinness
2025-02-12 19:00:25 +00:00
parent 25f41eb931
commit c1e9d93037
94 changed files with 1898 additions and 512 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/process_test.go
+18 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend_test
import (
"context"
@@ -64,10 +64,12 @@ func TestProcess(t *testing.T) {
})
assert.NoError(t, err)
docsyncurl := test.CreateQueue(t, ctx, cfg, test.DocSyncRunner)
doccleanurl := test.CreateQueue(t, ctx, cfg, test.DocCleanRunner)
doctexturl := test.CreateQueue(t, ctx, cfg, test.DocTextRunner)
querysyncurl := test.CreateQueue(t, ctx, cfg, test.QuerySyncRunner)
queryurl := test.CreateQueue(t, ctx, cfg, test.QueryRunner)
jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner)
net, cleanup := test.CreateRunnersAndServicesNetwork(t, ctx, &test.EcosystemNetworkConfig{
Cfg: cfg,
@@ -75,6 +77,12 @@ func TestProcess(t *testing.T) {
Runners: []*test.RunnerNetworkConfig{
{
Name: test.DocInitRunner,
Env: map[string]string{
"DOCUMENT_SYNC_URL": docsyncurl,
},
},
{
Name: test.DocSyncRunner,
Env: map[string]string{
"DOCUMENT_CLEAN_URL": doccleanurl,
},
@@ -103,10 +111,19 @@ func TestProcess(t *testing.T) {
"QUERY_URL": queryurl,
},
},
{
Name: test.JobSyncRunner,
Env: map[string]string{
"DOCUMENT_SYNC_URL": docsyncurl,
},
},
},
Services: []*test.ServiceNetworkConfig{
{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": jobsyncurl,
},
},
},
})
test/queryService/client_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend
import (
"context"
@@ -14,6 +14,9 @@ func TestClient(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
test/queryService/exportservice_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend
import (
"context"
@@ -14,6 +14,9 @@ func TestExportService(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
test/queryService/job_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend
import (
"context"
@@ -14,6 +14,9 @@ func TestJob(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
test/queryService/jobcollectorservice_test.go
+55 -25
View File
@@ -1,21 +1,53 @@
package integration_test
package endtoend
import (
"context"
"log"
"os"
"path"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
"regexp"
"testing"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
)
type CollectorConfig struct {
serviceconfig.BaseConfig
queue.QueueConfig
}
func TestJobCollectorService(t *testing.T) {
t.SkipNow()
ctx := context.Background()
cfg := &CollectorConfig{}
test.SetCfgProvider(t, cfg)
cfg.SetBasePath(path.Join(os.Getenv("PWD"), "../.."))
network, ncleanup := test.CreateNetwork(t, ctx)
defer ncleanup()
_, clean := test.CreateAWSContainer(t, ctx, &test.CreateAWSConfig{
Cfg: cfg,
Network: network,
})
defer clean()
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
jobsyncurl := test.CreateQueue(t, ctx, cfg, test.JobSyncRunner)
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Cfg: cfg,
Network: network,
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": jobsyncurl,
},
})
defer cleanup()
@@ -47,41 +79,39 @@ func TestJobCollectorService(t *testing.T) {
assert.NoError(t, err)
id := jobRes.JSON201.Id
fields := []queryservice.JobCollectorField{
{
Name: "json_output",
QueryId: jsonRes.JSON201.Id,
},
}
collRes, err := client.GetJobCollectorByJobIdWithResponse(ctx, id)
assert.NoError(t, err)
assert.Equal(t, id, collRes.JSON200.JobId)
assert.Equal(t, 1, collRes.JSON200.ActiveVersion)
assert.Equal(t, 1, collRes.JSON200.LatestVersion)
assert.Equal(t, 1, collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, 1, collRes.JSON200.MinimumTextVersion)
assert.Equal(t, int32(1), collRes.JSON200.ActiveVersion)
assert.Equal(t, int32(1), collRes.JSON200.LatestVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 0)
av := int32(2)
minClean := int32(2)
minText := int32(2)
fields := []queryservice.JobCollectorField{
{
Name: "json",
QueryId: jsonRes.JSON201.Id,
},
}
uRes, err := client.UpdateJobCollectorByJobIdWithResponse(ctx, id, queryservice.JobCollectorUpdate{
ActiveVersion: &av,
MinimumCleanerVersion: &minClean,
MinimumTextVersion: &minText,
Fields: &fields,
ActiveVersion: &av,
Fields: &fields,
})
assert.NoError(t, err)
assert.Nil(t, uRes)
log.Print(string(uRes.Body))
assert.Equal(t, 200, uRes.StatusCode())
test.AssertMessageBody(t, ctx, cfg, jobsyncurl, regexp.MustCompile(`{"id":".+"}`))
collRes, err = client.GetJobCollectorByJobIdWithResponse(ctx, id)
assert.NoError(t, err)
assert.Equal(t, id, collRes.JSON200.JobId)
assert.Equal(t, 2, collRes.JSON200.ActiveVersion)
assert.Equal(t, 2, collRes.JSON200.LatestVersion)
assert.Equal(t, 2, collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, 2, collRes.JSON200.MinimumTextVersion)
assert.Equal(t, int32(2), collRes.JSON200.ActiveVersion)
assert.Equal(t, int32(2), collRes.JSON200.LatestVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int32(0), collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 1)
assert.ElementsMatch(t, fields, collRes.JSON200.Fields)
}
test/queryService/openapi_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend_test
import (
"context"
@@ -15,6 +15,9 @@ func TestQueryServiceOpenAPI(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
test/queryService/queryservice_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend_test
import (
"context"
@@ -15,6 +15,9 @@ func TestQueryService(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()
test/queryService/testquery_test.go
+4 -1
View File
@@ -1,4 +1,4 @@
package integration_test
package endtoend_test
import (
"context"
@@ -15,6 +15,9 @@ func TestQueryServiceTest(t *testing.T) {
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"JOB_SYNC_URL": "/i/am/here",
},
})
defer cleanup()