Jay Brown
ebf47c6013
track file sizes for all documents in system * feature complete needs dev testing
363 lines
9.1 KiB
Go
363 lines
9.1 KiB
Go
package endtoend_test
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"queryorchestration/internal/serviceconfig"
|
|
"queryorchestration/internal/serviceconfig/aws"
|
|
"queryorchestration/internal/serviceconfig/objectstore"
|
|
"queryorchestration/internal/test"
|
|
|
|
queryapitest "queryorchestration/internal/test/queryAPI"
|
|
queryapi "queryorchestration/pkg/queryAPI"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
type Config struct {
|
|
serviceconfig.BaseConfig
|
|
aws.AWSConfig
|
|
objectstore.ObjectStoreConfig
|
|
}
|
|
|
|
func TestProcess(t *testing.T) {
|
|
t.Run("basic upload", func(t *testing.T) {
|
|
cfg := &Config{}
|
|
|
|
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
|
|
defer clean()
|
|
|
|
var clientId queryapi.ClientID
|
|
var jsonId queryapi.QueryID
|
|
var textractExpectation test.MockExpectation
|
|
textractBody := "Hello World"
|
|
|
|
// Use errgroup to properly propagate errors and avoid indefinite hangs
|
|
// If any goroutine fails, g.Wait() returns the error immediately
|
|
g := new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
clientId = queryapitest.CreateClientWithSync(t, net.Client)
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 1 failed: %v", err)
|
|
}
|
|
|
|
g = new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
textractExpectation = test.CreateDetectDocumentTextExpectation(
|
|
t,
|
|
net.Dependencies.MockServer,
|
|
textractBody,
|
|
)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
queryapitest.CreateFile(t, net.Client, queryapitest.File{
|
|
Filename: "helloworld.pdf",
|
|
Content: []byte(pdfHelloWorld),
|
|
ClientID: clientId,
|
|
})
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 2 failed: %v", err)
|
|
}
|
|
|
|
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
|
|
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
|
|
|
|
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
|
|
require.NoError(t, err)
|
|
assert.Len(t, *docs.JSON200, 1)
|
|
doc := (*docs.JSON200)[0]
|
|
|
|
docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, docRes)
|
|
require.NotNil(t, docRes.JSON200)
|
|
|
|
// Verify core document fields
|
|
assert.Equal(t, doc.Id, docRes.JSON200.Id)
|
|
assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
|
|
assert.Equal(t, clientId, docRes.JSON200.ClientId)
|
|
assert.Equal(t, map[string]any{"JSON_QUERY": "valueone"}, docRes.JSON200.Fields)
|
|
assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
|
|
|
|
// Verify enriched fields are populated (values depend on upload)
|
|
assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
|
|
assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
|
|
assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
|
|
|
|
// Test with textRecord=true parameter
|
|
includeTextRecord := true
|
|
docResWithText, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, &queryapi.GetDocumentParams{
|
|
TextRecord: &includeTextRecord,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotNil(t, docResWithText.JSON200)
|
|
// Document doesn't have a field extraction in this test, so hasTextRecord should be false
|
|
assert.False(t, docResWithText.JSON200.HasTextRecord)
|
|
assert.Nil(t, docResWithText.JSON200.TextRecord)
|
|
})
|
|
t.Run("update config", func(t *testing.T) {
|
|
cfg := &Config{}
|
|
|
|
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
|
|
defer clean()
|
|
|
|
var clientId queryapi.ClientID
|
|
var jsonId queryapi.QueryID
|
|
var textractExpectation test.MockExpectation
|
|
textractBody := "Hello World"
|
|
|
|
g := new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
clientId = queryapitest.CreateClientWithSync(t, net.Client)
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 1 failed: %v", err)
|
|
}
|
|
|
|
g = new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
queryapitest.CreateFile(t, net.Client, queryapitest.File{
|
|
Filename: "helloworld.pdf",
|
|
Content: []byte(pdfHelloWorld),
|
|
ClientID: clientId,
|
|
})
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
textractExpectation = test.CreateDetectDocumentTextExpectation(
|
|
t,
|
|
net.Dependencies.MockServer,
|
|
textractBody,
|
|
)
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 2 failed: %v", err)
|
|
}
|
|
|
|
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
|
|
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
|
|
|
|
jcfg := `{"path":"keytwo"}`
|
|
av := int32(2)
|
|
_, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
|
|
ActiveVersion: &av,
|
|
Config: &jcfg,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
|
|
|
|
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
|
|
require.NoError(t, err)
|
|
assert.Len(t, *docs.JSON200, 1)
|
|
doc := (*docs.JSON200)[0]
|
|
|
|
docRes, err := net.Client.GetDocumentWithResponse(t.Context(), doc.Id, nil)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, docRes)
|
|
require.NotNil(t, docRes.JSON200)
|
|
|
|
// Verify core document fields
|
|
assert.Equal(t, doc.Id, docRes.JSON200.Id)
|
|
assert.Equal(t, doc.Hash, docRes.JSON200.Hash)
|
|
assert.Equal(t, clientId, docRes.JSON200.ClientId)
|
|
assert.Equal(t, map[string]any{"JSON_QUERY": "valuetwo"}, docRes.JSON200.Fields)
|
|
assert.Equal(t, []queryapi.LabelRecord{}, docRes.JSON200.Labels)
|
|
|
|
// Verify enriched fields are populated (values depend on upload)
|
|
assert.NotNil(t, docRes.JSON200.Filename, "Filename should be populated")
|
|
assert.Equal(t, "helloworld.pdf", *docRes.JSON200.Filename)
|
|
assert.NotNil(t, docRes.JSON200.OriginalPath, "OriginalPath should be populated")
|
|
})
|
|
t.Run("test multiple versions", func(t *testing.T) {
|
|
cfg := &Config{}
|
|
|
|
net, clean := test.CreateFullNetwork(t, t.Context(), cfg)
|
|
defer clean()
|
|
|
|
var clientId queryapi.ClientID
|
|
var jsonId queryapi.QueryID
|
|
var textractExpectation test.MockExpectation
|
|
textractBody := "Hello World"
|
|
|
|
g := new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
_, jsonId = queryapitest.CreateDependentQueries(t, net.Client)
|
|
|
|
jcfg := `{"path":"keytwo"}`
|
|
_, err := net.Client.UpdateQueryWithResponse(t.Context(), jsonId, queryapi.QueryUpdate{
|
|
Config: &jcfg,
|
|
})
|
|
require.NoError(t, err)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
clientId = queryapitest.CreateClientWithSync(t, net.Client)
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 1 failed: %v", err)
|
|
}
|
|
|
|
g = new(errgroup.Group)
|
|
|
|
g.Go(func() error {
|
|
textractExpectation = test.CreateDetectDocumentTextExpectation(
|
|
t,
|
|
net.Dependencies.MockServer,
|
|
textractBody,
|
|
)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
queryapitest.SetQueryForClient(t, net.Client, clientId, jsonId)
|
|
return nil
|
|
})
|
|
g.Go(func() error {
|
|
queryapitest.CreateFile(t, net.Client, queryapitest.File{
|
|
Filename: "helloworld.pdf",
|
|
Content: []byte(pdfHelloWorld),
|
|
ClientID: clientId,
|
|
})
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("setup phase 2 failed: %v", err)
|
|
}
|
|
|
|
test.WaitForMockEndpoint(t, net.Dependencies.MockServer, textractExpectation.Request)
|
|
queryapitest.WaitForClientStatus(t, t.Context(), net.Client, clientId, queryapi.INSYNC)
|
|
|
|
docs, err := net.Client.ListDocumentsByClientIdWithResponse(t.Context(), clientId)
|
|
require.NoError(t, err)
|
|
assert.Len(t, *docs.JSON200, 1)
|
|
doc := (*docs.JSON200)[0]
|
|
|
|
g = new(errgroup.Group)
|
|
|
|
// Capture doc.Id to avoid closure issues
|
|
docId := doc.Id
|
|
g.Go(func() error {
|
|
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
|
|
QueryVersion: 1,
|
|
DocumentId: docId,
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "valueone", testRes.JSON200.Value)
|
|
return nil
|
|
})
|
|
|
|
g.Go(func() error {
|
|
testRes, err := net.Client.TestQueryWithResponse(t.Context(), jsonId, queryapi.QueryTestRequest{
|
|
QueryVersion: 2,
|
|
DocumentId: docId,
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "valuetwo", testRes.JSON200.Value)
|
|
return nil
|
|
})
|
|
|
|
if err := g.Wait(); err != nil {
|
|
t.Fatalf("test phase failed: %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
const pdfHelloWorld = `%PDF-1.4
|
|
%����
|
|
1 0 obj
|
|
<<
|
|
/Type /Catalog
|
|
/Pages 2 0 R
|
|
/Version /1.4
|
|
>>
|
|
endobj
|
|
2 0 obj
|
|
<<
|
|
/Type /Pages
|
|
/Kids [3 0 R]
|
|
/Count 1
|
|
>>
|
|
endobj
|
|
3 0 obj
|
|
<<
|
|
/Type /Page
|
|
/Parent 2 0 R
|
|
/MediaBox [0 0 612 792]
|
|
/Resources <<
|
|
/Font <<
|
|
/F1 <<
|
|
/Type /Font
|
|
/Subtype /Type1
|
|
/BaseFont /Helvetica
|
|
>>
|
|
>>
|
|
>>
|
|
/Contents 4 0 R
|
|
>>
|
|
endobj
|
|
4 0 obj
|
|
<<
|
|
/Length
|
|
44
|
|
>>
|
|
stream
|
|
BT
|
|
/F1 24 Tf
|
|
100 700 Td
|
|
(Hello World!) Tj
|
|
ET
|
|
endstream
|
|
endobj
|
|
xref
|
|
0 5
|
|
0000000000 65535 f
|
|
0000000015 00000 n
|
|
0000000086 00000 n
|
|
0000000151 00000 n
|
|
0000000376 00000 n
|
|
trailer
|
|
<<
|
|
/Size 5
|
|
/Root 1 0 R
|
|
>>
|
|
startxref
|
|
472
|
|
%%EOF`
|