jacob-mathison/query-orchestration
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in feature/remove-query (pull request #201)

Browse Source 
remove query from codebase part 1

* remove query

* fix localstack run
This commit is contained in:
Jay Brown
2026-01-14 17:59:04 +00:00
parent ebf47c6013
commit 0ddae4f91e
167 changed files with 1059 additions and 20641 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/README.md
+78
View File
@@ -0,0 +1,78 @@
# Integration Tests
This directory contains two integration test implementations that test the same core API workflow but serve different purposes.
## Test Files
### `text_extraction_integration_test.go` - Automated CI/CD Testing
**Purpose**: Automated integration testing for CI/CD pipelines.
**How it works**:
- Self-contained using testcontainers (spins up PostgreSQL, S3 mock, and API server automatically)
- Uses the generated Go API client for type-safe requests
- Runs as part of `task test:functional` or `task fullsuite:ci`
**When to use**:
- Local development verification before committing
- CI/CD pipeline testing
- Automated regression testing
**Run with**:
```bash
go test -v ./test/... -run TestE2EDocWorkflow
# Or via task
task test:functional
```
---
### `text_extraction_integration_test.sh` - Manual Testing
**Purpose**: Manual end-to-end testing against a running API server.
**How it works**:
- Uses curl/jq to make REST API calls
- Requires an external API server already running (auth bypass mode)
- Creates comprehensive test data with realistic healthcare contract values
- Demonstrates API usage for UI developers
**When to use**:
- Testing against local docker-compose stack
- Testing against AWS dev/UAT deployments
- Populating a development environment with sample data
- Demonstrating API endpoints to UI developers
**Run with**:
```bash
# Against local stack
./test/text_extraction_integration_test.sh http://localhost:8080
# Against AWS deployment
./test/text_extraction_integration_test.sh http://your-alb-url.us-east-2.elb.amazonaws.com
```
**Requirements**:
- bash 4.0+
- curl
- jq
- zip
- Target API must be running in auth bypass mode
---
## Test Coverage
Both tests exercise the same core workflow:
1. **Client Setup**: Create, verify, list, update client
2. **Document Upload**: Batch ZIP upload, single document upload
3. **Label Operations**: Apply labels, verify, query by label
4. **Folder Metrics**: List folders, get metrics per folder
5. **Field Extractions**: Create, version history, retrieve specific versions
The Go test includes additional scenarios:
- Deep folder hierarchy (12 levels)
- Large array field extraction (100 items)
- Label reapplication history
- Cross-feature integration (folders + labels + field extractions)
test/process_test.go
-362
View File
@@ -1,362 +0,0 @@
package endtoend_test
import (
"testing"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/serviceconfig/aws"
"queryorchestration/internal/serviceconfig/objectstore"
"queryorchestration/internal/test"
queryapitest "queryorchestration/internal/test/queryAPI"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
)
type Config struct {
serviceconfig.BaseConfig
aws.AWSConfig
objectstore.ObjectStoreConfig
}
func TestProcess(t *testing.T) {
t.Run("basic upload", func(t *testing.T) {
cfg := &Config{}
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
// Use errgroup to properly propagate errors and avoid indefinite hangs
// If any goroutine fails, g.Wait() returns the error immediately
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
g = new(errgroup.Group)
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
return nil
})
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
require.NoError(t, err)
assert.Len(t, *docs.JSON200, 1)
doc := (*docs.JSON200)[0]
docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
require.NoError(t, err)
require.NotNil(t, docRes)
require.NotNil(t, docRes.JSON200)
// Verify core document fields
assert.Equal(t, doc.Id, docRes.JSON200.Id)
assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
assert.Equal(t, clientId, docRes.JSON200.ClientId)
assert.Equal(t, map[string]any{"JSON_QUERY": "valueone"}, docRes.JSON200.Fields)
assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
// Verify enriched fields are populated (values depend on upload)
assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
// Test with textRecord=true parameter
includeTextRecord := true
docResWithText, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, &queryapi.GetDocumentParams{
TextRecord: &includeTextRecord,
})
require.NoError(t, err)
require.NotNil(t, docResWithText.JSON200)
// Document doesn't have a field extraction in this test, so hasTextRecord should be false
assert.False(t, docResWithText.JSON200.HasTextRecord)
assert.Nil(t, docResWithText.JSON200.TextRecord)
})
t.Run("update config", func(t *testing.T) {
cfg := &Config{}
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
g = new(errgroup.Group)
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
return nil
})
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
jcfg := `{"path":"keytwo"}`
av := int32(2)
_, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
ActiveVersion: &av,
Config: &jcfg,
})
require.NoError(t, err)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
require.NoError(t, err)
assert.Len(t, *docs.JSON200, 1)
doc := (*docs.JSON200)[0]
docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
require.NoError(t, err)
require.NotNil(t, docRes)
require.NotNil(t, docRes.JSON200)
// Verify core document fields
assert.Equal(t, doc.Id, docRes.JSON200.Id)
assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
assert.Equal(t, clientId, docRes.JSON200.ClientId)
assert.Equal(t, map[string]any{"JSON_QUERY": "valuetwo"}, docRes.JSON200.Fields)
assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
// Verify enriched fields are populated (values depend on upload)
assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
})
t.Run("test multiple versions", func(t *testing.T) {
cfg := &Config{}
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
defer clean()
var clientId queryapi.ClientID
var jsonId queryapi.QueryID
var textractExpectation test.MockExpectation
textractBody := "Hello World"
g := new(errgroup.Group)
g.Go(func() error {
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
jcfg := `{"path":"keytwo"}`
_, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
Config: &jcfg,
})
require.NoError(t, err)
return nil
})
g.Go(func() error {
clientId = queryapitest.CreateClientWithSync(t, net.Client)
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 1 failed: %v", err)
}
g = new(errgroup.Group)
g.Go(func() error {
textractExpectation = test.CreateDetectDocumentTextExpectation(
t,
net.Dependencies.MockServer,
textractBody,
)
return nil
})
g.Go(func() error {
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
return nil
})
g.Go(func() error {
queryapitest.CreateFile(t, net.Client, queryapitest.File{
Filename: "helloworld.pdf",
Content: []byte(pdfHelloWorld),
ClientID: clientId,
})
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("setup phase 2 failed: %v", err)
}
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
require.NoError(t, err)
assert.Len(t, *docs.JSON200, 1)
doc := (*docs.JSON200)[0]
g = new(errgroup.Group)
// Capture doc.Id to avoid closure issues
docId := doc.Id
g.Go(func() error {
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
QueryVersion: 1,
DocumentId: docId,
})
require.NoError(t, err)
assert.Equal(t, "valueone", testRes.JSON200.Value)
return nil
})
g.Go(func() error {
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
QueryVersion: 2,
DocumentId: docId,
})
require.NoError(t, err)
assert.Equal(t, "valuetwo", testRes.JSON200.Value)
return nil
})
if err := g.Wait(); err != nil {
t.Fatalf("test phase failed: %v", err)
}
})
}
const pdfHelloWorld = `%PDF-1.4
%����
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
/Version /1.4
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
/Contents 4 0 R
>>
endobj
4 0 obj
<<
/Length
44
>>
stream
BT
/F1 24 Tf
100 700 Td
(Hello World!) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000015 00000 n
0000000086 00000 n
0000000151 00000 n
0000000376 00000 n
trailer
<<
/Size 5
/Root 1 0 R
>>
startxref
472
%%EOF`
test/queryAPI/collectorservice_test.go
+2 -26
View File
@@ -14,7 +14,6 @@ import (
"github.com/stretchr/testify/require"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
)
@@ -25,6 +24,8 @@ type Config struct {
objectstore.ObjectStoreConfig
}
// TestCollectorService tests the collector API endpoints.
// Note: Query functionality has been removed. See remove_query_plan.md for details.
func TestCollectorService(t *testing.T) {
ctx := t.Context()
@@ -36,21 +37,6 @@ func TestCollectorService(t *testing.T) {
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
contextRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
require.NoError(t, err)
jsoncfg := "{\"path\":\"key\"}"
jsonRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jsoncfg,
RequiredQueries: &[]types.UUID{
contextRes.JSON201.Id,
},
})
require.NoError(t, err)
clientRes, err := client.CreateClientWithResponse(ctx, queryapi.ClientCreate{
Name: "example_name",
Id: "ID",
@@ -66,21 +52,13 @@ func TestCollectorService(t *testing.T) {
assert.Equal(t, int32(0), collRes.JSON200.LatestVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, int64(0), collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 0)
av := int32(1)
fields := []queryapi.CollectorField{
{
Name: "json",
QueryId: jsonRes.JSON201.Id,
},
}
minVersion := int64(1000)
uRes, err := client.SetCollectorByClientIdWithResponse(ctx, id, queryapi.CollectorSet{
ActiveVersion: &av,
MinimumCleanerVersion: &minVersion,
MinimumTextVersion: &minVersion,
Fields: &fields,
})
require.NoError(t, err)
test.AssertStatus(t, http.StatusOK, uRes.HTTPResponse)
@@ -93,8 +71,6 @@ func TestCollectorService(t *testing.T) {
assert.Equal(t, int32(1), collRes.JSON200.LatestVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumCleanerVersion)
assert.Equal(t, minVersion, collRes.JSON200.MinimumTextVersion)
assert.Len(t, collRes.JSON200.Fields, 1)
assert.ElementsMatch(t, fields, collRes.JSON200.Fields)
test.AssertMessageBody(t, cfg, c.Dependencies.QueueURLs[test.ClientSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
}
test/queryAPI/queryservice_test.go
-160
View File
@@ -1,160 +0,0 @@
package endtoend_test
import (
"regexp"
"testing"
"queryorchestration/internal/test"
queryapi "queryorchestration/pkg/queryAPI"
"github.com/stretchr/testify/require"
"github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
)
func TestQueryAPI(t *testing.T) {
t.Run("list no queries", func(t *testing.T) {
cfg := &Config{}
c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
NoObjectStore: true,
Runners: &[]test.RunnerName{},
})
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
queriesRes, err := client.ListQueriesWithResponse(t.Context())
require.NoError(t, err)
assert.Len(t, queriesRes.JSON200.Queries, 0)
})
t.Run("create and get query", func(t *testing.T) {
ctx := t.Context()
cfg := &Config{}
c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
NoObjectStore: true,
Runners: &[]test.RunnerName{},
})
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
jcfg := `{"path": "createkey"}`
idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
})
require.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
jsonID := idRes.JSON201.Id
queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
require.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion)
assert.Equal(t, jcfg, *queryRes.JSON200.Config)
assert.Nil(t, queryRes.JSON200.RequiredQueries)
queriesRes, err := client.ListQueriesWithResponse(ctx)
require.NoError(t, err)
assert.Len(t, queriesRes.JSON200.Queries, 1)
assert.Equal(t, jcfg, *queriesRes.JSON200.Queries[0].Config)
})
t.Run("update query", func(t *testing.T) {
ctx := t.Context()
cfg := &Config{}
c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
NoObjectStore: true,
Runners: &[]test.RunnerName{test.QueryVersionSyncRunnerName},
})
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
jcfg := "{}"
idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
})
require.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
jsonID := idRes.JSON201.Id
aV := int32(2)
newJcfg := `{"path": "keyone"}`
res, err := client.UpdateQueryWithResponse(ctx, jsonID, queryapi.QueryUpdate{
ActiveVersion: &aV,
Config: &newJcfg,
})
require.NoError(t, err)
assert.NotNil(t, res)
test.AssertMessageBody(t, cfg, c.Dependencies.QueueURLs[test.QueryVersionSyncRunnerName], regexp.MustCompile(`{"id":".+"}`))
queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
require.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(2), queryRes.JSON200.ActiveVersion)
assert.Equal(t, int32(2), queryRes.JSON200.LatestVersion)
assert.Equal(t, newJcfg, *queryRes.JSON200.Config)
assert.Nil(t, queryRes.JSON200.RequiredQueries)
})
t.Run("multiple dependent queries", func(t *testing.T) {
ctx := t.Context()
cfg := &Config{}
c, cleanup := test.CreateAPINetworkWithParams(t, cfg, test.QueryAPI, &test.FullDependenciesParams{
NoObjectStore: true,
Runners: &[]test.RunnerName{},
})
defer cleanup()
client, err := queryapi.NewClientWithResponses(c.API.URI)
require.NoError(t, err)
idRes, err := client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.CONTEXTFULL,
})
require.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
contextID := idRes.JSON201.Id
assert.NotEmpty(t, contextID)
jcfg := "{}"
idRes, err = client.CreateQueryWithResponse(ctx, queryapi.QueryCreate{
Type: queryapi.JSONEXTRACTOR,
Config: &jcfg,
RequiredQueries: &[]types.UUID{
contextID,
},
})
require.NoError(t, err)
assert.NotNil(t, idRes)
assert.NotNil(t, idRes.JSON201)
jsonID := idRes.JSON201.Id
queryRes, err := client.GetQueryWithResponse(ctx, jsonID)
require.NoError(t, err)
assert.Equal(t, jsonID, queryRes.JSON200.Id)
assert.Equal(t, queryapi.JSONEXTRACTOR, queryRes.JSON200.Type)
assert.Equal(t, int32(1), queryRes.JSON200.ActiveVersion)
assert.Equal(t, int32(1), queryRes.JSON200.LatestVersion)
assert.Equal(t, jcfg, *queryRes.JSON200.Config)
assert.ElementsMatch(t, []types.UUID{contextID}, *queryRes.JSON200.RequiredQueries)
})
}