jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/postprocessing (pull request #114)

Browse Source 
Feature/postprocessing

* tests

* passtest

* fixshorttests

* mosttests

* improvingbasedockerfile

* testspeeds

* testing

* host

* canparallel

* clean

* passfullsuite

* singlepagemax

* test

* findfeatures

* findstables

* tbls

* tablestoo

* tablestoo

* lateraltests

* tableloc

* cleanup

* inlinetable

* childids

* cleanup

* tests
This commit is contained in:
Michael McGuinness
2025-04-22 14:40:16 +00:00
parent edc50c7510
commit fee71e7740
404 changed files with 211048 additions and 2820 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/process_test.go
+22 -11
View File
@@ -65,6 +65,13 @@ func TestProcess(t *testing.T) {
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC)
textractBody := "Hello World"
textractExpectation := test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
part := uint16(0)
importKey := objectstore.BucketKey{
ClientID: client.Id,
@@ -73,24 +80,28 @@ func TestProcess(t *testing.T) {
CreatedAt: time.Now().UTC(),
Part: &part,
}
expectation := test.CreateStartDocTextDetectionExpectation(t, net.Dependencies.MockServer, test.StartDocTextDetectionExpectationParams{
Bucket: net.Dependencies.BucketName,
JobID: "textractId",
Key: importKey,
})
body := strings.NewReader(pdfHelloWorld)
inputFile := strings.NewReader(pdfHelloWorld)
test.PutObject(t, ctx, cfg, test.PutObjectParams{
Bucket: net.Dependencies.BucketName,
File: body,
File: inputFile,
Key: importKey,
})
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.NOTSYNCED)
test.WaitForMockTextractOutput(t, ctx, cfg, net.Dependencies.MockServer, test.TextractOutputParams{
Request: expectation.Request,
File: body,
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
textKey := objectstore.BucketKey{
ClientID: client.Id,
EntityID: uuid.New(),
Location: objectstore.Text,
CreatedAt: time.Now().UTC(),
Part: &part,
}
inputFile = strings.NewReader(textractBody)
test.PutObject(t, ctx, cfg, test.PutObjectParams{
Bucket: net.Dependencies.BucketName,
File: inputFile,
Key: textKey,
})
queryapitest.WaitForClientStatus(t, ctx, net.Client, client.Id, queryapi.INSYNC)
test/queryAPI/accessory_test.go
+5 -7
View File
@@ -19,27 +19,25 @@ func TestQueryAPIAccessories(t *testing.T) {
cfg := &Config{}
test.SetCfgProvider(t, cfg)
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, test.QueryAPI)
defer cleanup()
resp, err := http.Get(fmt.Sprintf("%s/swagger/doc.json", c.URI))
resp, err := http.Get(fmt.Sprintf("%s/swagger/doc.json", c.API.URI))
require.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = http.Get(fmt.Sprintf("%s/swagger/doc.yaml", c.URI))
resp, err = http.Get(fmt.Sprintf("%s/swagger/doc.yaml", c.API.URI))
require.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = http.Get(fmt.Sprintf("%s/swagger/index.html", c.URI))
resp, err = http.Get(fmt.Sprintf("%s/swagger/index.html", c.API.URI))
require.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
resp, err = http.Get(fmt.Sprintf("%s/metrics", c.URI))
resp, err = http.Get(fmt.Sprintf("%s/metrics", c.API.URI))
require.NoError(t, err)
assert.NotNil(t, resp)
assert.Equal(t, http.StatusOK, resp.StatusCode)
test/queryAPI/client_test.go
+2 -4
View File
@@ -18,12 +18,10 @@ func TestClient(t *testing.T) {
cfg := &Config{}
test.SetCfgProvider(t, cfg)
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, test.QueryAPI)
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
clientErrRes, err := client.GetClientWithResponse(ctx, "INVALID")
test/queryAPI/collectorservice_test.go
+3 -9
View File
@@ -31,16 +31,10 @@ func TestCollectorService(t *testing.T) {
cfg := &Config{}
test.SetCfgProvider(t, cfg)
deps, clean := test.CreateFullDependencies(t, ctx, cfg)
defer clean()
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, &test.ServiceNetworkConfig{
Network: deps.Network,
API: test.QueryAPI,
})
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, test.QueryAPI)
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
contextRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
@@ -101,5 +95,5 @@ func TestCollectorService(t *testing.T) {
assert.Len(t, collRes.JSON200.Fields, 1)
assert.ElementsMatch(t, fields, collRes.JSON200.Fields)
test.AssertMessageBody(t, ctx, cfg, deps.QueueURLs[test.ClientSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
test.AssertMessageBody(t, ctx, cfg, c.Dependencies.QueueURLs[test.ClientSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
}
test/queryAPI/exportservice_test.go
+2 -4
View File
@@ -18,12 +18,10 @@ func TestExportService(t *testing.T) {
cfg := &Config{}
test.SetCfgProvider(t, cfg)
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, &test.ServiceNetworkConfig{
API: test.QueryAPI,
})
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, test.QueryAPI)
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
idRes, err := client.TriggerExportWithResponse(ctx, "CLIENT_ID", queryapi.ExportTrigger{})
test/queryAPI/queryservice_test.go
+2 -5
View File
@@ -23,13 +23,10 @@ func TestQueryAPI(t *testing.T) {
deps, clean := test.CreateFullDependencies(t, ctx, cfg)
defer clean()
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, &test.ServiceNetworkConfig{
Network: deps.Network,
API: test.QueryAPI,
})
c, cleanup := test.CreateAPINetwork(t, ctx, cfg, test.QueryAPI)
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.URI)
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{