Merged in feature/textExtractionsPart1 (pull request #192)
all schema and rest apis for text extraction support * in progress * stage 8 complete * phase 9 completed * phase 9 complete * ongoing - s3 path fix * working * optimize ci build * e2e tests * missing test
This commit is contained in:
Jay Brown
@@ -0,0 +1,481 @@
|
||||
package fieldextraction_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"queryorchestration/internal/database/repository"
|
||||
"queryorchestration/internal/fieldextraction"
|
||||
"queryorchestration/internal/serviceconfig"
|
||||
"queryorchestration/internal/test"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type TestConfig struct {
|
||||
serviceconfig.BaseConfig
|
||||
}
|
||||
|
||||
func TestCreateFieldExtraction(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
svc := fieldextraction.New(cfg)
|
||||
|
||||
// Create test client and document
|
||||
clientID := "test-client-extractions"
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Extractions",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := "contract.pdf"
|
||||
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "testhash123",
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("create field extraction successfully", func(t *testing.T) {
|
||||
contractTitle := "Provider Agreement 2024"
|
||||
clientName := "Acme Corp"
|
||||
amendmentNum := int32(1)
|
||||
autoRenewal := true
|
||||
autoRenewalTerm := "1 year"
|
||||
|
||||
// Create single fields
|
||||
singleFields := &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle,
|
||||
Aaretederivedamendmentnum: &amendmentNum,
|
||||
Clientname: &clientName,
|
||||
Aaretederivedeffectivedt: pgtype.Date{
|
||||
Time: test.MustParseDate("2024-01-01"),
|
||||
Valid: true,
|
||||
},
|
||||
Aaretederivedterminationdt: pgtype.Date{
|
||||
Time: test.MustParseDate("2025-12-31"),
|
||||
Valid: true,
|
||||
},
|
||||
Autorenewalind: &autoRenewal,
|
||||
Autorenewalterm: &autoRenewalTerm,
|
||||
Createdby: "user123",
|
||||
}
|
||||
|
||||
// Create array fields (2 rows)
|
||||
exhibitTitle1 := "Exhibit A"
|
||||
exhibitPage1 := "1"
|
||||
provTin1 := "123456789"
|
||||
provNpi1 := "9876543210"
|
||||
provName1 := "Test Provider 1"
|
||||
|
||||
exhibitTitle2 := "Exhibit B"
|
||||
exhibitPage2 := "2"
|
||||
provTin2 := "987654321"
|
||||
provNpi2 := "0123456789"
|
||||
provName2 := "Test Provider 2"
|
||||
|
||||
arrayFields := []*repository.AddFieldExtractionArrayFieldParams{
|
||||
{
|
||||
Exhibittitle: &exhibitTitle1,
|
||||
Exhibitpage: &exhibitPage1,
|
||||
Reimbprovtin: &provTin1,
|
||||
Reimbprovnpi: &provNpi1,
|
||||
Reimbprovname: &provName1,
|
||||
Reimbeffectivedt: pgtype.Date{
|
||||
Time: test.MustParseDate("2024-01-01"),
|
||||
Valid: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
Exhibittitle: &exhibitTitle2,
|
||||
Exhibitpage: &exhibitPage2,
|
||||
Reimbprovtin: &provTin2,
|
||||
Reimbprovnpi: &provNpi2,
|
||||
Reimbprovname: &provName2,
|
||||
Reimbeffectivedt: pgtype.Date{
|
||||
Time: test.MustParseDate("2024-06-01"),
|
||||
Valid: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: singleFields,
|
||||
ArrayFields: arrayFields,
|
||||
}
|
||||
|
||||
result, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, documentID, result.Documentid)
|
||||
assert.Equal(t, contractTitle, *result.Contracttitle)
|
||||
assert.Equal(t, clientName, *result.Clientname)
|
||||
assert.Equal(t, amendmentNum, *result.Aaretederivedamendmentnum)
|
||||
assert.Equal(t, "user123", result.Createdby)
|
||||
|
||||
// Verify array fields were created
|
||||
arrayFieldsResult, err := svc.GetFieldExtractionArrayFields(ctx, result.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, arrayFieldsResult, 2)
|
||||
assert.Equal(t, int16(0), arrayFieldsResult[0].Arrayindex)
|
||||
assert.Equal(t, exhibitTitle1, *arrayFieldsResult[0].Exhibittitle)
|
||||
assert.Equal(t, int16(1), arrayFieldsResult[1].Arrayindex)
|
||||
assert.Equal(t, exhibitTitle2, *arrayFieldsResult[1].Exhibittitle)
|
||||
})
|
||||
|
||||
t.Run("reject nil input", func(t *testing.T) {
|
||||
_, err := svc.CreateFieldExtraction(ctx, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "input cannot be nil")
|
||||
})
|
||||
|
||||
t.Run("reject nil documentID", func(t *testing.T) {
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: uuid.Nil,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Createdby: "user123",
|
||||
},
|
||||
}
|
||||
_, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "documentID cannot be nil")
|
||||
})
|
||||
|
||||
t.Run("reject empty createdBy", func(t *testing.T) {
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Createdby: "",
|
||||
},
|
||||
}
|
||||
_, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "createdBy cannot be empty")
|
||||
})
|
||||
|
||||
t.Run("create with empty array fields", func(t *testing.T) {
|
||||
// Create new document for this test
|
||||
filename2 := "contract2.pdf"
|
||||
documentID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "testhash456",
|
||||
Filename: &filename2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
contractTitle := "Simple Contract"
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID2,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID2,
|
||||
Filename: &filename2,
|
||||
Contracttitle: &contractTitle,
|
||||
Createdby: "user123",
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{}, // Empty array
|
||||
}
|
||||
|
||||
result, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, result)
|
||||
assert.Equal(t, documentID2, result.Documentid)
|
||||
|
||||
// Verify no array fields
|
||||
arrayFieldsResult, err := svc.GetFieldExtractionArrayFields(ctx, result.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, arrayFieldsResult, 0)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetCurrentFieldExtraction(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
svc := fieldextraction.New(cfg)
|
||||
|
||||
// Create test client and document
|
||||
clientID := "test-client-get-current"
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Get Current",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := "current.pdf"
|
||||
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "currenthash",
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("get current extraction", func(t *testing.T) {
|
||||
// Create first version
|
||||
contractTitle1 := "Version 1"
|
||||
input1 := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle1,
|
||||
Createdby: "user123",
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
|
||||
}
|
||||
_, err := svc.CreateFieldExtraction(ctx, input1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create second version
|
||||
contractTitle2 := "Version 2"
|
||||
input2 := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle2,
|
||||
Createdby: "user456",
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
|
||||
}
|
||||
version2, err := svc.CreateFieldExtraction(ctx, input2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get current should return version 2
|
||||
current, err := svc.GetCurrentFieldExtraction(ctx, documentID)
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, current)
|
||||
assert.Equal(t, version2.ID, current.ID)
|
||||
assert.Equal(t, contractTitle2, *current.Contracttitle)
|
||||
assert.Equal(t, int64(2), current.Version)
|
||||
})
|
||||
|
||||
t.Run("return nil for non-existent document", func(t *testing.T) {
|
||||
nonExistentID := uuid.New()
|
||||
result, err := svc.GetCurrentFieldExtraction(ctx, nonExistentID)
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, result)
|
||||
})
|
||||
|
||||
t.Run("reject nil documentID", func(t *testing.T) {
|
||||
_, err := svc.GetCurrentFieldExtraction(ctx, uuid.Nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "documentID cannot be nil")
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetFieldExtractionHistory(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
svc := fieldextraction.New(cfg)
|
||||
|
||||
// Create test client and document
|
||||
clientID := "test-client-history"
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client History",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := "history.pdf"
|
||||
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "historyhash",
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("get history with multiple versions", func(t *testing.T) {
|
||||
// Create 3 versions
|
||||
for i := 1; i <= 3; i++ {
|
||||
contractTitle := fmt.Sprintf("Version %d", i)
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle,
|
||||
Createdby: fmt.Sprintf("user%d", i),
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
|
||||
}
|
||||
_, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Get history
|
||||
history, err := svc.GetFieldExtractionHistory(ctx, documentID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, history, 3)
|
||||
|
||||
// Should be ordered by version DESC (most recent first)
|
||||
assert.Equal(t, int64(3), history[0].Version)
|
||||
assert.Equal(t, "user3", history[0].Createdby)
|
||||
assert.Equal(t, int64(2), history[1].Version)
|
||||
assert.Equal(t, "user2", history[1].Createdby)
|
||||
assert.Equal(t, int64(1), history[2].Version)
|
||||
assert.Equal(t, "user1", history[2].Createdby)
|
||||
})
|
||||
|
||||
t.Run("return empty for document with no extractions", func(t *testing.T) {
|
||||
// Create new document without extractions
|
||||
filename2 := "nohistory.pdf"
|
||||
documentID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "nohistoryhash",
|
||||
Filename: &filename2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
history, err := svc.GetFieldExtractionHistory(ctx, documentID2)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, history, 0)
|
||||
})
|
||||
|
||||
t.Run("reject nil documentID", func(t *testing.T) {
|
||||
_, err := svc.GetFieldExtractionHistory(ctx, uuid.Nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "documentID cannot be nil")
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetFieldExtractionArrayFields(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
svc := fieldextraction.New(cfg)
|
||||
|
||||
// Create test client and document
|
||||
clientID := "test-client-array-fields"
|
||||
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
|
||||
Clientid: clientID,
|
||||
Name: "Test Client Array Fields",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
filename := "arraytest.pdf"
|
||||
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "arrayhash",
|
||||
Filename: &filename,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("get array fields ordered by index", func(t *testing.T) {
|
||||
// Create extraction with 5 array rows
|
||||
arrayFields := make([]*repository.AddFieldExtractionArrayFieldParams, 5)
|
||||
for i := 0; i < 5; i++ {
|
||||
title := fmt.Sprintf("Exhibit %d", i)
|
||||
page := fmt.Sprintf("%d", i+1)
|
||||
arrayFields[i] = &repository.AddFieldExtractionArrayFieldParams{
|
||||
Exhibittitle: &title,
|
||||
Exhibitpage: &page,
|
||||
}
|
||||
}
|
||||
|
||||
contractTitle := "Array Test Contract"
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle,
|
||||
Createdby: "user123",
|
||||
},
|
||||
ArrayFields: arrayFields,
|
||||
}
|
||||
|
||||
extraction, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get array fields
|
||||
result, err := svc.GetFieldExtractionArrayFields(ctx, extraction.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result, 5)
|
||||
|
||||
// Verify order and content
|
||||
for i := 0; i < 5; i++ {
|
||||
//nolint:gosec // Test index is small, int16 is sufficient
|
||||
assert.Equal(t, int16(i), result[i].Arrayindex)
|
||||
assert.Equal(t, fmt.Sprintf("Exhibit %d", i), *result[i].Exhibittitle)
|
||||
assert.Equal(t, fmt.Sprintf("%d", i+1), *result[i].Exhibitpage)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("return empty for extraction with no array fields", func(t *testing.T) {
|
||||
// Create new document
|
||||
filename2 := "noarray.pdf"
|
||||
documentID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
|
||||
Clientid: clientID,
|
||||
Hash: "noarrayhash",
|
||||
Filename: &filename2,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
contractTitle := "No Array Contract"
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: documentID2,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: documentID2,
|
||||
Filename: &filename2,
|
||||
Contracttitle: &contractTitle,
|
||||
Createdby: "user123",
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
|
||||
}
|
||||
|
||||
extraction, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.NoError(t, err)
|
||||
|
||||
result, err := svc.GetFieldExtractionArrayFields(ctx, extraction.ID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, result, 0)
|
||||
})
|
||||
|
||||
t.Run("reject nil fieldExtractionID", func(t *testing.T) {
|
||||
_, err := svc.GetFieldExtractionArrayFields(ctx, uuid.Nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "fieldExtractionID cannot be nil")
|
||||
})
|
||||
}
|
||||
|
||||
func TestTransactionRollback(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
cfg := &TestConfig{}
|
||||
test.CreateDB(t, cfg)
|
||||
svc := fieldextraction.New(cfg)
|
||||
|
||||
t.Run("invalid document reference fails transaction", func(t *testing.T) {
|
||||
// Try to create extraction for non-existent document
|
||||
nonExistentDocID := uuid.New()
|
||||
filename := "rollback.pdf"
|
||||
contractTitle := "Should Fail"
|
||||
input := &fieldextraction.CreateFieldExtractionInput{
|
||||
DocumentID: nonExistentDocID,
|
||||
SingleFields: &repository.AddFieldExtractionParams{
|
||||
Documentid: nonExistentDocID,
|
||||
Filename: &filename,
|
||||
Contracttitle: &contractTitle,
|
||||
Createdby: "user123",
|
||||
},
|
||||
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
|
||||
}
|
||||
|
||||
_, err := svc.CreateFieldExtraction(ctx, input)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "failed to create field extraction")
|
||||
|
||||
// Verify nothing was created
|
||||
history, err := svc.GetFieldExtractionHistory(ctx, nonExistentDocID)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, history, 0)
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user