jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/docresult (pull request #105)

Browse Source 
Document Result Endpoint

* testing

* cleantesting

* progress

* query

* lint

* readme

* dockerclient

* api

* passtest

* tests

* test
This commit is contained in:
Michael McGuinness
2025-03-17 18:14:15 +00:00
parent f7c41c4ef3
commit 555b6d420b
105 changed files with 3276 additions and 2484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/process_test.go
+58 -58
View File
@@ -2,16 +2,14 @@ package endtoend_test
import (
"context"
"fmt"
"strings"
"testing"
"time"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/test"
queryservicetest "queryorchestration/internal/test/queryService"
queryservice "queryorchestration/pkg/queryService"
queryapitest "queryorchestration/internal/test/queryAPI"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
@@ -32,23 +30,23 @@ func TestProcess(t *testing.T) {
net, clean := test.CreateFullNetwork(t, ctx, cfg)
defer clean()
client := queryservicetest.CreateClientWithSync(t, ctx, net.Client)
client := queryapitest.CreateClientWithSync(t, ctx, net.Client)
contextQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
contextQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
require.NoError(t, err)
jcfg := `{"path":"keyone"}`
jsonQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
jsonQueryRes, err := net.Client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
RequiredQueries: &[]types.UUID{contextQueryRes.JSON201.Id},
})
require.NoError(t, err)
newActiveVersion := int32(1)
_, err = net.Client.SetCollectorByClientIdWithResponse(ctx, client.Id, queryservice.CollectorSet{
_, err = net.Client.SetCollectorByClientIdWithResponse(ctx, client.Id, queryapi.CollectorSet{
ActiveVersion: &newActiveVersion,
Fields: &[]queryservice.CollectorField{
Fields: &[]queryapi.CollectorField{
{
Name: "JSON_QUERY",
QueryId: jsonQueryRes.JSON201.Id,
@@ -57,71 +55,73 @@ func TestProcess(t *testing.T) {
})
require.NoError(t, err)
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC)
body := strings.NewReader(pdfHelloWorld)
location := test.PutObject(t, ctx, cfg, client.Uid, net.Dependencies.BucketName, "object_name", body)
test.PutObject(t, ctx, cfg, client.Uid, net.Dependencies.BucketName, "object_name", body)
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.NOTSYNCED)
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED)
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC)
docs, err := net.Client.ListDocumentsByClientIdWithResponse(ctx, client.Id)
assert.NoError(t, err)
require.NoError(t, err)
assert.Len(t, *docs.JSON200, 1)
doc := (*docs.JSON200)[0]
assert.Equal(t, net.Dependencies.BucketName, doc.Bucket)
assert.Equal(t, location, doc.Key)
testRes, err := net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
expectedDoc := queryapi.Document{
Id: doc.Id,
Hash: doc.Hash,
ClientId: client.Id,
Fields: map[string]interface{}{
"JSON_QUERY": "valueone",
},
}
docRes, err := net.Client.GetDocumentWithResponse(ctx, doc.Id)
require.NoError(t, err)
require.NotNil(t, docRes)
require.NotNil(t, docRes.JSON200)
fullDoc := *docRes.JSON200
assert.EqualExportedValues(t, expectedDoc, fullDoc)
jcfg = `{"path":"keytwo"}`
av := int32(2)
_, err = net.Client.UpdateQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryUpdate{
ActiveVersion: &av,
Config: &jcfg,
})
require.NoError(t, err)
expectedDoc.Fields = map[string]interface{}{
"JSON_QUERY": "valuetwo",
}
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED)
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC)
docRes, err = net.Client.GetDocumentWithResponse(ctx, doc.Id)
require.NoError(t, err)
require.NotNil(t, docRes)
require.NotNil(t, docRes.JSON200)
assert.EqualExportedValues(t, expectedDoc, *docRes.JSON200)
testRes, err := net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryTestRequest{
QueryVersion: 1,
DocumentId: doc.Id,
})
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, "valueone", testRes.JSON200.Value)
aV := int32(2)
jcfg = `{"path":"keytwo"}`
res, err := net.Client.UpdateQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryUpdate{
ActiveVersion: &aV,
Config: &jcfg,
})
assert.NoError(t, err)
assert.NotNil(t, res)
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.NOTSYNCED)
WaitForClientStatus(t, ctx, net.Client, client.Id, queryservice.INSYNC)
testRes, err = net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryservice.QueryTestRequest{
testRes, err = net.Client.TestQueryWithResponse(ctx, jsonQueryRes.JSON201.Id, queryapi.QueryTestRequest{
QueryVersion: 2,
DocumentId: doc.Id,
})
assert.NoError(t, err)
assert.Equal(t, "valuetwo", testRes.JSON200.Value)
}
func WaitForClientStatus(t testing.TB, ctx context.Context, service *queryservice.ClientWithResponses, id string, status queryservice.ClientStatus) {
t.Helper()
timeout := time.After(30 * time.Second)
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-timeout:
require.NoError(t, fmt.Errorf("Timeout waiting for client status to become %s", status))
case <-ticker.C:
jRes, err := service.GetStatusByClientIdWithResponse(ctx, id)
if err != nil {
assert.NoError(t, err)
}
if jRes.JSON200.Status == status {
assert.Equal(t, status, jRes.JSON200.Status)
return
}
}
}
docRes, err = net.Client.GetDocumentWithResponse(ctx, doc.Id)
require.NoError(t, err)
require.NotNil(t, docRes)
require.NotNil(t, docRes.JSON200)
assert.EqualExportedValues(t, expectedDoc, *docRes.JSON200)
}
const pdfHelloWorld = `%PDF-1.4
test/queryService/accessory_test.go → test/queryAPI/accessory_test.go
+3 -7
View File
@@ -11,15 +11,11 @@ import (
"github.com/stretchr/testify/assert"
)
func TestQueryServiceAccessories(t *testing.T) {
func TestQueryAPIAccessories(t *testing.T) {
ctx := context.Background()
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "iamthere",
},
c, cleanup := test.CreateAPINetwork(t, ctx, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
defer cleanup()
test/queryService/client_test.go → test/queryAPI/client_test.go
+6 -10
View File
@@ -5,7 +5,7 @@ import (
"testing"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/stretchr/testify/assert"
)
@@ -13,19 +13,15 @@ import (
func TestClient(t *testing.T) {
ctx := context.Background()
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "iamthere",
},
c, cleanup := test.CreateAPINetwork(t, ctx, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{
idRes, err := client.CreateClientWithResponse(ctx, queryapi.ClientCreate{
Name: "example_name",
Id: "EXA",
})
@@ -43,7 +39,7 @@ func TestClient(t *testing.T) {
updateName := "update_name"
updateCanSync := true
updateRes, err := client.UpdateClientWithResponse(ctx, id, queryservice.ClientUpdate{
updateRes, err := client.UpdateClientWithResponse(ctx, id, queryapi.ClientUpdate{
Name: &updateName,
CanSync: &updateCanSync,
})
test/queryService/collectorservice_test.go → test/queryAPI/collectorservice_test.go
+12 -16
View File
@@ -8,7 +8,7 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
@@ -35,30 +35,26 @@ func TestCollectorService(t *testing.T) {
defer clean()
test.SetQueueClient(t, ctx, cfg)
clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunner)
clientsyncurl := test.CreateQueue(t, ctx, cfg, test.ClientSyncRunnerName)
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
c, cleanup := test.CreateAPINetwork(t, ctx, &test.ServiceNetworkConfig{
Cfg: cfg,
Network: network,
Name: test.QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": clientsyncurl,
"QUERY_VERSION_SYNC_URL": "iamthere",
},
API: test.QueryAPI,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.URI)
assert.NoError(t, err)
contextRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
contextRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
assert.NoError(t, err)
jsoncfg := "{\"path\":\"key\"}"
jsonRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
jsonRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jsoncfg,
RequiredQueries: &[]types.UUID{
contextRes.JSON201.Id,
@@ -66,7 +62,7 @@ func TestCollectorService(t *testing.T) {
})
assert.NoError(t, err)
clientRes, err := client.CreateClientWithResponse(ctx, queryservice.ClientCreate{
clientRes, err := client.CreateClientWithResponse(ctx, queryapi.ClientCreate{
Name: "example_name",
Id: "ID",
})
@@ -83,14 +79,14 @@ func TestCollectorService(t *testing.T) {
assert.Len(t, collRes.JSON200.Fields, 0)
av := int32(1)
fields := []queryservice.CollectorField{
fields := []queryapi.CollectorField{
{
Name: "json",
QueryId: jsonRes.JSON201.Id,
},
}
minVersion := int64(1000)
uRes, err := client.SetCollectorByClientIdWithResponse(ctx, id, queryservice.CollectorSet{
uRes, err := client.SetCollectorByClientIdWithResponse(ctx, id, queryapi.CollectorSet{
ActiveVersion: &av,
MinimumCleanerVersion: &minVersion,
MinimumTextVersion: &minVersion,
test/queryAPI/exportservice_test.go
+27
View File
@@ -0,0 +1,27 @@
package endtoend
import (
"context"
"testing"
"queryorchestration/internal/test"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/stretchr/testify/assert"
)
func TestExportService(t *testing.T) {
ctx := context.Background()
c, cleanup := test.CreateAPINetwork(t, ctx, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.TriggerExportWithResponse(ctx, "CLIENT_ID", queryapi.ExportTrigger{})
assert.NoError(t, err)
assert.NotNil(t, idRes)
}
test/queryService/queryservice_test.go → test/queryAPI/queryservice_test.go
+13 -17
View File
@@ -10,7 +10,7 @@ import (
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/queue"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
@@ -21,7 +21,7 @@ type QueryConfig struct {
queue.QueueConfig
}
func TestQueryService(t *testing.T) {
func TestQueryAPI(t *testing.T) {
ctx := context.Background()
cfg := &QueryConfig{}
@@ -39,24 +39,20 @@ func TestQueryService(t *testing.T) {
err := cfg.SetQueueClient(ctx)
assert.NoError(t, err)
queryversionsyncurl := test.CreateQueue(t, ctx, cfg, test.QueryVersionSyncRunner)
queryversionsyncurl := test.CreateQueue(t, ctx, cfg, test.QueryVersionSyncRunnerName)
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
c, cleanup := test.CreateAPINetwork(t, ctx, &test.ServiceNetworkConfig{
Cfg: cfg,
Network: network,
Name: test.QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": queryversionsyncurl,
},
API: test.QueryAPI,
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.CONTEXTFULL,
idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
assert.NoError(t, err)
assert.NotNil(t, idRes)
@@ -65,8 +61,8 @@ func TestQueryService(t *testing.T) {
assert.NotEmpty(t, contextID)
jcfg := "{}"
idRes, err = client.CreateQueryWithResponse(ctx, queryservice.QueryCreate{
Type: queryservice.JSONEXTRACTOR,
idRes, err = client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
})
assert.NoError(t, err)
@@ -75,7 +71,7 @@ func TestQueryService(t *testing.T) {
queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
assert.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion)
assert.Equal(t, jcfg, *queryRes.JSON200.Config)
@@ -86,7 +82,7 @@ func TestQueryService(t *testing.T) {
assert.Len(t, queriesRes.JSON200.Queries, 2)
aV := int32(2)
res, err := client.UpdateQueryWithResponse(ctx, jsonID, queryservice.QueryUpdate{
res, err := client.UpdateQueryWithResponse(ctx, jsonID, queryapi.QueryUpdate{
ActiveVersion: &aV,
RequiredQueries: &[]types.UUID{
contextID,
@@ -100,7 +96,7 @@ func TestQueryService(t *testing.T) {
queryRes, err = client.GetQueryWithResponse(ctx, jsonID)
assert.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryservice.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion)
assert.Equal(t, int32(2), queryRes.JSON200.LatestVersion)
assert.Equal(t, jcfg, *queryRes.JSON200.Config)
test/queryService/exportservice_test.go
-31
View File
@@ -1,31 +0,0 @@
package endtoend
import (
"context"
"testing"
"queryorchestration/internal/test"
queryservice "queryorchestration/pkg/queryService"
"github.com/stretchr/testify/assert"
)
func TestExportService(t *testing.T) {
ctx := context.Background()
c, cleanup := test.CreateServiceNetwork(t, ctx, &test.ServiceNetworkConfig{
Name: test.QueryService,
Env: map[string]string{
"CLIENT_SYNC_URL": "/i/am/here",
"QUERY_VERSION_SYNC_URL": "iamthere",
},
})
defer cleanup()
client, err := queryservice.NewClientWithResponses(c.URI)
assert.NoError(t, err)
idRes, err := client.TriggerExportWithResponse(ctx, "CLIENT_ID", queryservice.ExportTrigger{})
assert.NoError(t, err)
assert.NotNil(t, idRes)
}