Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support

* in progress

* stage 8 complete

* phase 9 completed

* phase 9 complete

* ongoing - s3 path fix

* working

* optimize ci build

* e2e tests

* missing test
This commit is contained in:
Jay Brown
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
+16 -8
View File
@@ -9,6 +9,8 @@ import (
"queryorchestration/internal/database/repository"
documentinit "queryorchestration/internal/document/init"
"queryorchestration/internal/serviceconfig/objectstore"
"github.com/google/uuid"
)
const Name = "docInitRunner"
@@ -41,8 +43,9 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
return false
}
// Get filename from documentUploads table
// Get filename and folder_id from documentUploads table
var filename *string
var folderID *uuid.UUID
upload, err := s.svc.Queries.GetDocumentUploadByKey(ctx, &repository.GetDocumentUploadByKeyParams{
Bucket: body.Bucket,
Key: body.Key,
@@ -51,20 +54,25 @@ func (s Runner) Process(ctx context.Context, body Body) bool {
slog.Error("unable to get document upload", "bucket", body.Bucket, "key", body.Key, "error", err)
return false
}
if err == nil && upload.Filename != nil {
filename = upload.Filename
if err == nil {
if upload.Filename != nil {
filename = upload.Filename
}
folderID = upload.FolderID
}
id, err := s.svc.Document.Create(ctx, &documentinit.Create{
Key: key,
Bucket: body.Bucket,
Hash: body.Hash,
Filename: filename,
Key: key,
Bucket: body.Bucket,
Hash: body.Hash,
Filename: filename,
OriginalPath: filename, // For batch uploads, filename contains the full original path
FolderID: folderID,
})
if err != nil {
return false
}
slog.Debug("created document", "id", id, "filename", filename)
slog.Debug("created document", "id", id, "filename", filename, "folder_id", folderID)
return true
}
+746 -135
View File
File diff suppressed because it is too large Load Diff
+17 -11
View File
@@ -10,6 +10,9 @@ import (
documentbatch "queryorchestration/internal/document/batch"
documentupload "queryorchestration/internal/document/upload"
"queryorchestration/internal/export"
"queryorchestration/internal/fieldextraction"
"queryorchestration/internal/folder"
"queryorchestration/internal/label"
"queryorchestration/internal/query"
querytest "queryorchestration/internal/query/test"
queryupdate "queryorchestration/internal/query/update"
@@ -20,17 +23,20 @@ import (
const Name = "queryAPI"
type Services struct {
Export *export.Service
Collector *collector.Service
CollectorSet *collectorset.Service
Query *query.Service
QueryUpdate *queryupdate.Service
QueryTest *querytest.Service
Client *client.Service
ClientUpdate *clientupdate.Service
Document *document.Service
DocumentUpload *documentupload.Service
DocumentBatch *documentbatch.Service
Export *export.Service
Collector *collector.Service
CollectorSet *collectorset.Service
Query *query.Service
QueryUpdate *queryupdate.Service
QueryTest *querytest.Service
Client *client.Service
ClientUpdate *clientupdate.Service
Document *document.Service
DocumentUpload *documentupload.Service
DocumentBatch *documentbatch.Service
FieldExtraction *fieldextraction.Service
Folder *folder.Service
Label *label.Service
}
// ConfigProvider combines auth and objectstore interfaces for the Controllers
+20 -57
View File
@@ -36,11 +36,8 @@ func TestUploadDocument(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: "client_id",
Name: "client_name",
})
require.NoError(t, err)
// Use test helper to create client with root folder
test.CreateTestClient(t, cfg, "client_id", "client_name")
var buf bytes.Buffer
writer := multipart.NewWriter(&buf)
@@ -71,11 +68,8 @@ func TestListDocumentsByClientId(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: "client_id",
Name: "client_name",
})
require.NoError(t, err)
// Use test helper to create client with root folder
test.CreateTestClient(t, cfg, "client_id", "client_name")
docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
Clientid: "client_id",
Hash: "hash",
@@ -104,11 +98,8 @@ func TestGetDocument(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: "client_id",
Name: "client_name",
})
require.NoError(t, err)
// Use test helper to create client with root folder
test.CreateTestClient(t, cfg, "client_id", "client_name")
docId, err := cfg.GetDBQueries().CreateDocument(t.Context(), &repository.CreateDocumentParams{
Clientid: "client_id",
Hash: "hash",
@@ -137,13 +128,9 @@ func TestListDocumentBatches(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client
// Create test client with root folder
clientID := "test_client_list"
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Batch List",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Batch List")
// Create test batch
batchID, err := cfg.GetDBQueries().CreateBatchUpload(t.Context(), &repository.CreateBatchUploadParams{
@@ -188,13 +175,9 @@ func TestUploadDocumentBatch(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client
// Create test client with root folder
clientID := "test_client_upload"
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Batch Upload",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Batch Upload")
// Create multipart form with ZIP file
var buf bytes.Buffer
@@ -227,13 +210,9 @@ func TestGetDocumentBatch(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client
// Create test client with root folder
clientID := "test_client_get"
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Batch Get",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Batch Get")
// Create test batch
batchID, err := cfg.GetDBQueries().CreateBatchUpload(t.Context(), &repository.CreateBatchUploadParams{
@@ -262,13 +241,9 @@ func TestCancelDocumentBatch(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client
// Create test client with root folder
clientID := "test_client_cancel"
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Batch Cancel",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Batch Cancel")
// Create test batch
batchID, err := cfg.GetDBQueries().CreateBatchUpload(t.Context(), &repository.CreateBatchUploadParams{
@@ -303,13 +278,9 @@ func TestUploadDocumentBatch_Part1_Success(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client with unique ID
// Create test client with unique ID and root folder
clientID := fmt.Sprintf("test_client_part1_%d", time.Now().UnixNano())
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Part1",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Part1")
// Create test ZIP file with 3 PDFs (test1.pdf, test2.pdf, test3.pdf)
zipContent := createTestZIPFile(t, 3)
@@ -385,13 +356,9 @@ func TestUploadDocumentBatch_Part1_InvalidFile(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client with unique ID
// Create test client with unique ID and root folder
clientID := fmt.Sprintf("test_client_invalid_%d", time.Now().UnixNano())
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client Invalid",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client Invalid")
// Create invalid file (not ZIP)
body := &bytes.Buffer{}
@@ -431,13 +398,9 @@ func TestUploadDocumentBatch_Part1_TooLarge(t *testing.T) {
svc := createControllerServices(cfg)
cons := queryapi.NewControllers(svc, cfg)
// Create test client with unique ID
// Create test client with unique ID and root folder
clientID := fmt.Sprintf("test_client_toolarge_%d", time.Now().UnixNano())
err := cfg.GetDBQueries().CreateClient(t.Context(), &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client TooLarge",
})
require.NoError(t, err)
test.CreateTestClient(t, cfg, clientID, "Test Client TooLarge")
// Create oversized ZIP file
body := &bytes.Buffer{}
+461
View File
@@ -0,0 +1,461 @@
package queryapi
import (
"fmt"
"log/slog"
"net/http"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/fieldextraction"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
)
// CreateFieldExtraction creates a new field extraction with single-value and array fields.
// This endpoint creates a versioned field extraction record for a document.
func (s *Controllers) CreateFieldExtraction(ctx echo.Context) error {
var req FieldExtractionRequest
if err := ctx.Bind(&req); err != nil {
slog.Error("failed to parse field extraction request", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "invalid request body")
}
// Validate required fields
documentID := uuid.UUID(req.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
createdBy := string(req.CreatedBy)
if createdBy == "" {
return echo.NewHTTPError(http.StatusBadRequest, "createdBy is required")
}
// Convert API request to service input
input := &fieldextraction.CreateFieldExtractionInput{
DocumentID: documentID,
SingleFields: convertAPISingleFieldsToRepo(&req.SingleFields, createdBy),
ArrayFields: convertAPIArrayFieldsToRepo(req.ArrayFields),
}
// Create field extraction via service
extraction, err := s.svc.FieldExtraction.CreateFieldExtraction(ctx.Request().Context(), input)
if err != nil {
slog.Error("failed to create field extraction", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to create field extraction")
}
// Fetch the current extraction to get version info
current, err := s.svc.FieldExtraction.GetCurrentFieldExtraction(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get current field extraction after creation", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get created field extraction")
}
// Fetch array fields for response
arrayFields, err := s.svc.FieldExtraction.GetFieldExtractionArrayFields(ctx.Request().Context(), extraction.ID)
if err != nil {
slog.Error("failed to get array fields after creation", "error", err, "extraction_id", extraction.ID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get array fields")
}
// Build response
response := buildFieldExtractionResponse(current, arrayFields)
return ctx.JSON(http.StatusCreated, response)
}
// GetCurrentFieldExtraction retrieves the most recent field extraction for a document.
// This returns the current (latest version) field extraction including all single and array fields.
func (s *Controllers) GetCurrentFieldExtraction(ctx echo.Context, params GetCurrentFieldExtractionParams) error {
documentID := uuid.UUID(params.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
// Get current field extraction
current, err := s.svc.FieldExtraction.GetCurrentFieldExtraction(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get current field extraction", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get field extraction")
}
if current == nil {
return echo.NewHTTPError(http.StatusNotFound, "no field extraction found for document")
}
// Fetch array fields
arrayFields, err := s.svc.FieldExtraction.GetFieldExtractionArrayFields(ctx.Request().Context(), current.ID)
if err != nil {
slog.Error("failed to get array fields", "error", err, "extraction_id", current.ID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get array fields")
}
// Build response
response := buildFieldExtractionResponse(current, arrayFields)
return ctx.JSON(http.StatusOK, response)
}
// GetFieldExtractionHistory retrieves all versions of field extractions for a document.
// This returns version metadata without the full field data for each version.
func (s *Controllers) GetFieldExtractionHistory(ctx echo.Context, params GetFieldExtractionHistoryParams) error {
documentID := uuid.UUID(params.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
// Get version history
history, err := s.svc.FieldExtraction.GetFieldExtractionHistory(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get field extraction history", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get field extraction history")
}
// Convert to API response
versions := make([]FieldExtractionVersion, len(history))
for i, h := range history {
createdAt := time.Time{}
if h.Createdat.Valid {
createdAt = h.Createdat.Time
}
versions[i] = FieldExtractionVersion{
Id: openapi_types.UUID(h.ID),
DocumentId: DocumentID(h.Documentid),
//nolint:gosec // Version is a small number, int32 is sufficient
Version: int32(h.Version), // Already 1-based in database
CreatedAt: createdAt,
CreatedBy: openapi_types.Email(h.Createdby),
}
}
return ctx.JSON(http.StatusOK, struct {
Versions []FieldExtractionVersion `json:"versions"`
}{
Versions: versions,
})
}
// buildFieldExtractionResponse builds the API response from the repository types
func buildFieldExtractionResponse(current *repository.Currentfieldextraction, arrayFields []*repository.Documentfieldextractionarrayfield) *FieldExtractionResponse {
createdAt := time.Time{}
if current.Createdat.Valid {
createdAt = current.Createdat.Time
}
return &FieldExtractionResponse{
Id: openapi_types.UUID(current.ID),
DocumentId: DocumentID(current.Documentid),
//nolint:gosec // Version is a small number, int32 is sufficient
Version: int32(current.Version), // Already 1-based in database
SingleFields: convertRepoSingleFieldsToAPI(current),
ArrayFields: convertRepoArrayFieldsToAPI(arrayFields),
CreatedAt: createdAt,
CreatedBy: openapi_types.Email(current.Createdby),
}
}
// convertAPISingleFieldsToRepo converts API SingleFields to repository AddFieldExtractionParams
func convertAPISingleFieldsToRepo(sf *SingleFields, createdBy string) *repository.AddFieldExtractionParams {
params := &repository.AddFieldExtractionParams{
Createdby: createdBy,
}
if sf == nil {
return params
}
params.Filename = sf.FileName
params.Contracttitle = sf.ContractTitle
params.Aaretederivedamendmentnum = sf.AareteDerivedAmendmentNum
params.Clientname = sf.ClientName
params.Payername = sf.PayerName
params.Payerstate = sf.PayerState
params.Providerstate = sf.ProviderState
params.Filenametin = sf.FilenameTin
params.Provgrouptin = sf.ProvGroupTin
params.Provgroupnpi = sf.ProvGroupNpi
params.Provgroupnamefull = sf.ProvGroupNameFull
params.Provothertin = sf.ProvOtherTin
params.Provothernpi = sf.ProvOtherNpi
params.Provothernamefull = sf.ProvOtherNameFull
params.Autorenewalind = sf.AutoRenewalInd
params.Autorenewalterm = sf.AutoRenewalTerm
// Convert dates
if sf.AareteDerivedEffectiveDt != nil {
params.Aaretederivedeffectivedt = dateToPgDate(*sf.AareteDerivedEffectiveDt)
}
if sf.AareteDerivedTerminationDt != nil {
params.Aaretederivedterminationdt = dateToPgDate(*sf.AareteDerivedTerminationDt)
}
return params
}
// convertRepoSingleFieldsToAPI converts repository Currentfieldextraction to API SingleFields
func convertRepoSingleFieldsToAPI(cf *repository.Currentfieldextraction) SingleFields {
sf := SingleFields{
FileName: cf.Filename,
ContractTitle: cf.Contracttitle,
AareteDerivedAmendmentNum: cf.Aaretederivedamendmentnum,
ClientName: cf.Clientname,
PayerName: cf.Payername,
PayerState: cf.Payerstate,
ProviderState: cf.Providerstate,
FilenameTin: cf.Filenametin,
ProvGroupTin: cf.Provgrouptin,
ProvGroupNpi: cf.Provgroupnpi,
ProvGroupNameFull: cf.Provgroupnamefull,
ProvOtherTin: cf.Provothertin,
ProvOtherNpi: cf.Provothernpi,
ProvOtherNameFull: cf.Provothernamefull,
AutoRenewalInd: cf.Autorenewalind,
AutoRenewalTerm: cf.Autorenewalterm,
}
// Convert dates
if cf.Aaretederivedeffectivedt.Valid {
d := pgDateToAPIDate(cf.Aaretederivedeffectivedt)
sf.AareteDerivedEffectiveDt = &d
}
if cf.Aaretederivedterminationdt.Valid {
d := pgDateToAPIDate(cf.Aaretederivedterminationdt)
sf.AareteDerivedTerminationDt = &d
}
return sf
}
// convertAPIArrayFieldsToRepo converts API ArrayFieldItem slice to repository AddFieldExtractionArrayFieldParams slice
func convertAPIArrayFieldsToRepo(items []ArrayFieldItem) []*repository.AddFieldExtractionArrayFieldParams {
result := make([]*repository.AddFieldExtractionArrayFieldParams, len(items))
for i, item := range items {
result[i] = convertAPIArrayFieldItemToRepo(&item)
}
return result
}
// convertAPIArrayFieldItemToRepo converts a single API ArrayFieldItem to repository params
func convertAPIArrayFieldItemToRepo(item *ArrayFieldItem) *repository.AddFieldExtractionArrayFieldParams {
params := &repository.AddFieldExtractionArrayFieldParams{}
if item == nil {
return params
}
// String fields
params.Exhibittitle = item.ExhibitTitle
params.Exhibitpage = item.ExhibitPage
params.Reimbprovtin = item.ReimbProvTin
params.Reimbprovnpi = item.ReimbProvNpi
params.Reimbprovname = item.ReimbProvName
params.Aaretederivedclaimtypecd = item.AareteDerivedClaimTypeCd
params.Aaretederivedproduct = item.AareteDerivedProduct
params.Aaretederivedlob = item.AareteDerivedLob
params.Aaretederivedprogram = item.AareteDerivedProgram
params.Aaretederivednetwork = item.AareteDerivedNetwork
params.Aaretederivedprovtype = item.AareteDerivedProvType
params.Provtaxonomycd = item.ProvTaxonomyCd
params.Provtaxonomycddesc = item.ProvTaxonomyCdDesc
params.Provspecialtycd = item.ProvSpecialtyCd
params.Provspecialtycddesc = item.ProvSpecialtyCdDesc
params.Placeofservicecd = item.PlaceOfServiceCd
params.Placeofservicecddesc = item.PlaceOfServiceCdDesc
params.Billtypecd = item.BillTypeCd
params.Billtypecddesc = item.BillTypeCdDesc
params.Patientagemin = item.PatientAgeMin
params.Patientagemax = item.PatientAgeMax
params.Reimbterm = item.ReimbTerm
params.Lobprogramrelationship = item.LobProgramRelationship
params.Lobproductrelationship = item.LobProductRelationship
params.Aaretederivedreimbmethod = item.AareteDerivedReimbMethod
params.Unitofmeasure = item.UnitOfMeasure
params.Additiondesc = item.AdditionDesc
params.Aaretederivedadditionratechangetimeline = item.AareteDerivedAdditionRateChangeTimeline
params.Aaretederivedfeeschedule = item.AareteDerivedFeeSchedule
params.Aaretederivedfeescheduleversion = item.AareteDerivedFeeScheduleVersion
params.Serviceterm = item.ServiceTerm
params.Cpt4proccd = item.Cpt4ProcCd
params.Cpt4proccddesc = item.Cpt4ProcCdDesc
params.Cpt4procmod = item.Cpt4ProcMod
params.Cpt4procmoddesc = item.Cpt4ProcModDesc
params.Revenuecd = item.RevenueCd
params.Revenuecddesc = item.RevenueCdDesc
params.Diagcd = item.DiagCd
params.Diagcddesc = item.DiagCdDesc
params.Ndccd = item.NdcCd
params.Ndccddesc = item.NdcCdDesc
params.Claimadmittypecd = item.ClaimAdmitTypeCd
params.Authadmittypedesc = item.AuthAdmitTypeDesc
params.Claimstatuscd = item.ClaimStatusCd
params.Claimstatuscddesc = item.ClaimStatusCdDesc
params.Groupertype = item.GrouperType
params.Groupercd = item.GrouperCd
params.Groupercddesc = item.GrouperCdDesc
params.Carveoutcd = item.CarveoutCd
// Boolean fields
params.Carveoutind = item.CarveoutInd
params.Lesserofind = item.LesserOfInd
params.Greaterofind = item.GreaterOfInd
params.Defaultind = item.DefaultInd
// Numeric fields
params.Reimbpctrate = float64ToPgNumeric(item.ReimbPctRate)
params.Reimbfeerate = float64ToPgNumeric(item.ReimbFeeRate)
params.Reimbconversionfactor = float64ToPgNumeric(item.ReimbConversionFactor)
params.Triggercapthresholdamt = float64ToPgNumeric(item.TriggerCapThresholdAmt)
params.Triggerbasethreshold = float64ToPgNumeric(item.TriggerBaseThreshold)
params.Additionmaxfeerateinc = float64ToPgNumeric(item.AdditionMaxFeeRateInc)
params.Additionmaxpctrateinc = float64ToPgNumeric(item.AdditionMaxPctRateInc)
params.Grouperpctrate = float64ToPgNumeric(item.GrouperPctRate)
params.Grouperbaserate = float64ToPgNumeric(item.GrouperBaseRate)
// Date fields
if item.ReimbEffectiveDt != nil {
params.Reimbeffectivedt = dateToPgDate(*item.ReimbEffectiveDt)
}
if item.ReimbTerminationDt != nil {
params.Reimbterminationdt = dateToPgDate(*item.ReimbTerminationDt)
}
return params
}
// convertRepoArrayFieldsToAPI converts repository array fields to API ArrayFieldItem slice
func convertRepoArrayFieldsToAPI(fields []*repository.Documentfieldextractionarrayfield) []ArrayFieldItem {
result := make([]ArrayFieldItem, len(fields))
for i, f := range fields {
result[i] = convertRepoArrayFieldToAPI(f)
}
return result
}
// convertRepoArrayFieldToAPI converts a single repository array field to API ArrayFieldItem
func convertRepoArrayFieldToAPI(f *repository.Documentfieldextractionarrayfield) ArrayFieldItem {
item := ArrayFieldItem{
// String fields
ExhibitTitle: f.Exhibittitle,
ExhibitPage: f.Exhibitpage,
ReimbProvTin: f.Reimbprovtin,
ReimbProvNpi: f.Reimbprovnpi,
ReimbProvName: f.Reimbprovname,
AareteDerivedClaimTypeCd: f.Aaretederivedclaimtypecd,
AareteDerivedProduct: f.Aaretederivedproduct,
AareteDerivedLob: f.Aaretederivedlob,
AareteDerivedProgram: f.Aaretederivedprogram,
AareteDerivedNetwork: f.Aaretederivednetwork,
AareteDerivedProvType: f.Aaretederivedprovtype,
ProvTaxonomyCd: f.Provtaxonomycd,
ProvTaxonomyCdDesc: f.Provtaxonomycddesc,
ProvSpecialtyCd: f.Provspecialtycd,
ProvSpecialtyCdDesc: f.Provspecialtycddesc,
PlaceOfServiceCd: f.Placeofservicecd,
PlaceOfServiceCdDesc: f.Placeofservicecddesc,
BillTypeCd: f.Billtypecd,
BillTypeCdDesc: f.Billtypecddesc,
PatientAgeMin: f.Patientagemin,
PatientAgeMax: f.Patientagemax,
ReimbTerm: f.Reimbterm,
LobProgramRelationship: f.Lobprogramrelationship,
LobProductRelationship: f.Lobproductrelationship,
AareteDerivedReimbMethod: f.Aaretederivedreimbmethod,
UnitOfMeasure: f.Unitofmeasure,
AdditionDesc: f.Additiondesc,
AareteDerivedAdditionRateChangeTimeline: f.Aaretederivedadditionratechangetimeline,
AareteDerivedFeeSchedule: f.Aaretederivedfeeschedule,
AareteDerivedFeeScheduleVersion: f.Aaretederivedfeescheduleversion,
ServiceTerm: f.Serviceterm,
Cpt4ProcCd: f.Cpt4proccd,
Cpt4ProcCdDesc: f.Cpt4proccddesc,
Cpt4ProcMod: f.Cpt4procmod,
Cpt4ProcModDesc: f.Cpt4procmoddesc,
RevenueCd: f.Revenuecd,
RevenueCdDesc: f.Revenuecddesc,
DiagCd: f.Diagcd,
DiagCdDesc: f.Diagcddesc,
NdcCd: f.Ndccd,
NdcCdDesc: f.Ndccddesc,
ClaimAdmitTypeCd: f.Claimadmittypecd,
AuthAdmitTypeDesc: f.Authadmittypedesc,
ClaimStatusCd: f.Claimstatuscd,
ClaimStatusCdDesc: f.Claimstatuscddesc,
GrouperType: f.Groupertype,
GrouperCd: f.Groupercd,
GrouperCdDesc: f.Groupercddesc,
CarveoutCd: f.Carveoutcd,
// Boolean fields
CarveoutInd: f.Carveoutind,
LesserOfInd: f.Lesserofind,
GreaterOfInd: f.Greaterofind,
DefaultInd: f.Defaultind,
}
// Numeric fields
item.ReimbPctRate = pgNumericToFloat64(f.Reimbpctrate)
item.ReimbFeeRate = pgNumericToFloat64(f.Reimbfeerate)
item.ReimbConversionFactor = pgNumericToFloat64(f.Reimbconversionfactor)
item.TriggerCapThresholdAmt = pgNumericToFloat64(f.Triggercapthresholdamt)
item.TriggerBaseThreshold = pgNumericToFloat64(f.Triggerbasethreshold)
item.AdditionMaxFeeRateInc = pgNumericToFloat64(f.Additionmaxfeerateinc)
item.AdditionMaxPctRateInc = pgNumericToFloat64(f.Additionmaxpctrateinc)
item.GrouperPctRate = pgNumericToFloat64(f.Grouperpctrate)
item.GrouperBaseRate = pgNumericToFloat64(f.Grouperbaserate)
// Date fields
if f.Reimbeffectivedt.Valid {
d := pgDateToAPIDate(f.Reimbeffectivedt)
item.ReimbEffectiveDt = &d
}
if f.Reimbterminationdt.Valid {
d := pgDateToAPIDate(f.Reimbterminationdt)
item.ReimbTerminationDt = &d
}
return item
}
// dateToPgDate converts openapi_types.Date to pgtype.Date
func dateToPgDate(d openapi_types.Date) pgtype.Date {
return pgtype.Date{
Time: d.Time,
Valid: true,
}
}
// pgDateToAPIDate converts pgtype.Date to openapi_types.Date
func pgDateToAPIDate(d pgtype.Date) openapi_types.Date {
return openapi_types.Date{
Time: d.Time,
}
}
// float64ToPgNumeric converts *float64 to pgtype.Numeric
func float64ToPgNumeric(f *float64) pgtype.Numeric {
if f == nil {
return pgtype.Numeric{}
}
var num pgtype.Numeric
// Scan doesn't accept float64 directly, need to convert to string
_ = num.Scan(fmt.Sprintf("%v", *f))
return num
}
// pgNumericToFloat64 converts pgtype.Numeric to *float64
func pgNumericToFloat64(n pgtype.Numeric) *float64 {
if !n.Valid {
return nil
}
f, err := n.Float64Value()
if err != nil || !f.Valid {
return nil
}
return &f.Float64
}
+645
View File
@@ -0,0 +1,645 @@
package queryapi_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
queryapi "queryorchestration/api/queryAPI"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/fieldextraction"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type FieldExtractionsTestConfig struct {
serviceconfig.BaseConfig
}
func (c *FieldExtractionsTestConfig) GetBackgroundRunner() any { return nil }
func setupFieldExtractionsTestController(t *testing.T) (*queryapi.Controllers, *FieldExtractionsTestConfig, uuid.UUID, string) {
t.Helper()
cfg := &FieldExtractionsTestConfig{}
test.CreateDB(t, cfg)
// Create services
fieldExtractionSvc := fieldextraction.New(cfg)
services := &queryapi.Services{
FieldExtraction: fieldExtractionSvc,
}
// Create test client and document with unique IDs
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID)
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: "Test Client",
})
require.NoError(t, err)
filename := "test.pdf"
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("testhash-%s", uniqueID), // Unique hash per test
Filename: &filename,
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
return ctrl, cfg, documentID, clientID
}
func TestCreateFieldExtraction_Success(t *testing.T) {
ctrl, _, documentID, _ := setupFieldExtractionsTestController(t)
// Create request body
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("test.pdf"),
ContractTitle: ptr("Test Contract"),
ClientName: ptr("Test Client Inc"),
},
ArrayFields: []queryapi.ArrayFieldItem{
{
ExhibitTitle: ptr("Exhibit A"),
ExhibitPage: ptr("1"),
ReimbProvTin: ptr("123456789"),
},
{
ExhibitTitle: ptr("Exhibit B"),
ExhibitPage: ptr("2"),
ReimbProvTin: ptr("987654321"),
},
},
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, documentID, uuid.UUID(resp.DocumentId))
assert.Equal(t, int32(1), resp.Version) // First version is 1
assert.Equal(t, "test@example.com", string(resp.CreatedBy))
assert.Equal(t, "test.pdf", *resp.SingleFields.FileName)
assert.Equal(t, "Test Contract", *resp.SingleFields.ContractTitle)
assert.Len(t, resp.ArrayFields, 2)
assert.Equal(t, "Exhibit A", *resp.ArrayFields[0].ExhibitTitle)
assert.Equal(t, "Exhibit B", *resp.ArrayFields[1].ExhibitTitle)
}
func TestCreateFieldExtraction_InvalidRequest(t *testing.T) {
ctrl, _, _, _ := setupFieldExtractionsTestController(t)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader([]byte("invalid json")))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.CreateFieldExtraction(c)
// Handler returns an echo.HTTPError which should be treated as an error
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestCreateFieldExtraction_MissingDocumentID(t *testing.T) {
ctrl, _, _, _ := setupFieldExtractionsTestController(t)
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(uuid.Nil),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("test.pdf"),
},
ArrayFields: []queryapi.ArrayFieldItem{},
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestCreateFieldExtraction_MissingCreatedBy(t *testing.T) {
// Use a simple controller without database since we're just testing validation
ctrl := &queryapi.Controllers{}
// Use a valid document ID but empty createdBy
documentID := uuid.New()
body := []byte(fmt.Sprintf(`{
"documentId": "%s",
"createdBy": "",
"singleFields": {"fileName": "test.pdf"},
"arrayFields": []
}`, documentID))
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.CreateFieldExtraction(c)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestGetCurrentFieldExtraction_Success(t *testing.T) {
ctrl, cfg, documentID, _ := setupFieldExtractionsTestController(t)
ctx := t.Context()
// First create a field extraction using the service directly
svc := fieldextraction.New(cfg)
filename := "test.pdf"
contractTitle := "Test Contract"
_, err := svc.CreateFieldExtraction(ctx, &fieldextraction.CreateFieldExtractionInput{
DocumentID: documentID,
SingleFields: &repository.AddFieldExtractionParams{
Documentid: documentID,
Filename: &filename,
Contracttitle: &contractTitle,
Createdby: "user@example.com",
},
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
})
require.NoError(t, err)
// Now test the API
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/field-extractions?documentId=%s", documentID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetCurrentFieldExtractionParams{
DocumentId: queryapi.DocumentID(documentID),
}
err = ctrl.GetCurrentFieldExtraction(c, params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, documentID, uuid.UUID(resp.DocumentId))
assert.Equal(t, int32(1), resp.Version)
assert.Equal(t, "user@example.com", string(resp.CreatedBy))
assert.Equal(t, "Test Contract", *resp.SingleFields.ContractTitle)
}
func TestGetCurrentFieldExtraction_NotFound(t *testing.T) {
ctrl, _, _, _ := setupFieldExtractionsTestController(t)
// Use a random document ID that doesn't have any extractions
randomDocID := uuid.New()
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/field-extractions?documentId=%s", randomDocID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetCurrentFieldExtractionParams{
DocumentId: queryapi.DocumentID(randomDocID),
}
err := ctrl.GetCurrentFieldExtraction(c, params)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusNotFound, httpErr.Code)
}
func TestGetCurrentFieldExtraction_InvalidDocumentID(t *testing.T) {
ctrl, _, _, _ := setupFieldExtractionsTestController(t)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/field-extractions?documentId=", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetCurrentFieldExtractionParams{
DocumentId: queryapi.DocumentID(uuid.Nil),
}
err := ctrl.GetCurrentFieldExtraction(c, params)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestGetFieldExtractionHistory_Success(t *testing.T) {
ctrl, cfg, documentID, _ := setupFieldExtractionsTestController(t)
ctx := t.Context()
// Create multiple versions
svc := fieldextraction.New(cfg)
filename := "test.pdf"
for i := 1; i <= 3; i++ {
contractTitle := fmt.Sprintf("Contract Version %d", i)
_, err := svc.CreateFieldExtraction(ctx, &fieldextraction.CreateFieldExtractionInput{
DocumentID: documentID,
SingleFields: &repository.AddFieldExtractionParams{
Documentid: documentID,
Filename: &filename,
Contracttitle: &contractTitle,
Createdby: fmt.Sprintf("user%d@example.com", i),
},
ArrayFields: []*repository.AddFieldExtractionArrayFieldParams{},
})
require.NoError(t, err)
}
// Now test the API
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/field-extractions/history?documentId=%s", documentID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetFieldExtractionHistoryParams{
DocumentId: queryapi.DocumentID(documentID),
}
err := ctrl.GetFieldExtractionHistory(c, params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
// Parse response
var resp struct {
Versions []queryapi.FieldExtractionVersion `json:"versions"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Should have 3 versions, most recent first
assert.Len(t, resp.Versions, 3)
assert.Equal(t, int32(3), resp.Versions[0].Version)
assert.Equal(t, "user3@example.com", string(resp.Versions[0].CreatedBy))
assert.Equal(t, int32(2), resp.Versions[1].Version)
assert.Equal(t, "user2@example.com", string(resp.Versions[1].CreatedBy))
assert.Equal(t, int32(1), resp.Versions[2].Version)
assert.Equal(t, "user1@example.com", string(resp.Versions[2].CreatedBy))
}
func TestGetFieldExtractionHistory_NoHistory(t *testing.T) {
ctrl, cfg, _, clientID := setupFieldExtractionsTestController(t)
ctx := t.Context()
// Create a new document with no extractions
filename := "nohistory.pdf"
newDocID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: "nohistoryhash",
Filename: &filename,
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/field-extractions/history?documentId=%s", newDocID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetFieldExtractionHistoryParams{
DocumentId: queryapi.DocumentID(newDocID),
}
err = ctrl.GetFieldExtractionHistory(c, params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
// Parse response
var resp struct {
Versions []queryapi.FieldExtractionVersion `json:"versions"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Versions, 0)
}
func TestGetFieldExtractionHistory_InvalidDocumentID(t *testing.T) {
ctrl, _, _, _ := setupFieldExtractionsTestController(t)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/field-extractions/history?documentId=", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetFieldExtractionHistoryParams{
DocumentId: queryapi.DocumentID(uuid.Nil),
}
err := ctrl.GetFieldExtractionHistory(c, params)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestCreateFieldExtraction_WithDates(t *testing.T) {
ctrl, _, documentID, _ := setupFieldExtractionsTestController(t)
effectiveDate := openapi_types.Date{Time: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)}
terminationDate := openapi_types.Date{Time: time.Date(2025, 12, 31, 0, 0, 0, 0, time.UTC)}
reimbEffectiveDate := openapi_types.Date{Time: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)}
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("contract.pdf"),
ContractTitle: ptr("Test Contract with Dates"),
AareteDerivedEffectiveDt: &effectiveDate,
AareteDerivedTerminationDt: &terminationDate,
},
ArrayFields: []queryapi.ArrayFieldItem{
{
ExhibitTitle: ptr("Exhibit A"),
ReimbEffectiveDt: &reimbEffectiveDate,
},
},
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Verify dates are returned correctly
require.NotNil(t, resp.SingleFields.AareteDerivedEffectiveDt)
assert.Equal(t, 2024, resp.SingleFields.AareteDerivedEffectiveDt.Year())
assert.Equal(t, time.January, resp.SingleFields.AareteDerivedEffectiveDt.Month())
assert.Equal(t, 1, resp.SingleFields.AareteDerivedEffectiveDt.Day())
require.NotNil(t, resp.SingleFields.AareteDerivedTerminationDt)
assert.Equal(t, 2025, resp.SingleFields.AareteDerivedTerminationDt.Year())
assert.Equal(t, time.December, resp.SingleFields.AareteDerivedTerminationDt.Month())
assert.Equal(t, 31, resp.SingleFields.AareteDerivedTerminationDt.Day())
require.NotNil(t, resp.ArrayFields[0].ReimbEffectiveDt)
assert.Equal(t, 2024, resp.ArrayFields[0].ReimbEffectiveDt.Year())
assert.Equal(t, time.June, resp.ArrayFields[0].ReimbEffectiveDt.Month())
}
func TestCreateFieldExtraction_WithNumericFields(t *testing.T) {
ctrl, _, documentID, _ := setupFieldExtractionsTestController(t)
reimbPctRate := 10.5
reimbFeeRate := 150.75
grouperBaseRate := 1000.00
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("numeric_test.pdf"),
},
ArrayFields: []queryapi.ArrayFieldItem{
{
ExhibitTitle: ptr("Exhibit A"),
ReimbPctRate: &reimbPctRate,
ReimbFeeRate: &reimbFeeRate,
GrouperBaseRate: &grouperBaseRate,
},
},
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Verify numeric fields
require.Len(t, resp.ArrayFields, 1)
require.NotNil(t, resp.ArrayFields[0].ReimbPctRate)
assert.InDelta(t, 10.5, *resp.ArrayFields[0].ReimbPctRate, 0.001)
require.NotNil(t, resp.ArrayFields[0].ReimbFeeRate)
assert.InDelta(t, 150.75, *resp.ArrayFields[0].ReimbFeeRate, 0.001)
require.NotNil(t, resp.ArrayFields[0].GrouperBaseRate)
assert.InDelta(t, 1000.00, *resp.ArrayFields[0].GrouperBaseRate, 0.001)
}
func TestCreateFieldExtraction_EmptyArrayFields(t *testing.T) {
ctrl, _, documentID, _ := setupFieldExtractionsTestController(t)
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("simple.pdf"),
ContractTitle: ptr("Simple Contract"),
},
ArrayFields: []queryapi.ArrayFieldItem{},
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.ArrayFields, 0)
}
func TestCreateFieldExtraction_LargeArrayFields(t *testing.T) {
ctrl, _, documentID, _ := setupFieldExtractionsTestController(t)
// Create 100 array field items
arrayFields := make([]queryapi.ArrayFieldItem, 100)
for i := range arrayFields {
title := fmt.Sprintf("Exhibit %d", i)
page := fmt.Sprintf("%d", i+1)
arrayFields[i] = queryapi.ArrayFieldItem{
ExhibitTitle: &title,
ExhibitPage: &page,
}
}
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("test@example.com"),
SingleFields: queryapi.SingleFields{
FileName: ptr("large_array.pdf"),
ContractTitle: ptr("Large Array Contract"),
},
ArrayFields: arrayFields,
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.FieldExtractionResponse
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.ArrayFields, 100)
assert.Equal(t, "Exhibit 0", *resp.ArrayFields[0].ExhibitTitle)
assert.Equal(t, "Exhibit 99", *resp.ArrayFields[99].ExhibitTitle)
}
func TestCreateFieldExtraction_VersionIncrement(t *testing.T) {
ctrl, cfg, documentID, _ := setupFieldExtractionsTestController(t)
// Create first version via API
reqBody := queryapi.FieldExtractionRequest{
DocumentId: queryapi.DocumentID(documentID),
CreatedBy: openapi_types.Email("user1@example.com"),
SingleFields: queryapi.SingleFields{
ContractTitle: ptr("Version 1"),
},
ArrayFields: []queryapi.ArrayFieldItem{},
}
body, _ := json.Marshal(reqBody)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp1 queryapi.FieldExtractionResponse
_ = json.Unmarshal(rec.Body.Bytes(), &resp1)
assert.Equal(t, int32(1), resp1.Version)
// Create second version
reqBody.CreatedBy = openapi_types.Email("user2@example.com")
reqBody.SingleFields.ContractTitle = ptr("Version 2")
body, _ = json.Marshal(reqBody)
req = httptest.NewRequest(http.MethodPost, "/field-extractions", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec = httptest.NewRecorder()
c = e.NewContext(req, rec)
err = ctrl.CreateFieldExtraction(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp2 queryapi.FieldExtractionResponse
_ = json.Unmarshal(rec.Body.Bytes(), &resp2)
assert.Equal(t, int32(2), resp2.Version)
// GetCurrentFieldExtraction should return version 2
req = httptest.NewRequest(http.MethodGet, fmt.Sprintf("/field-extractions?documentId=%s", documentID), nil)
rec = httptest.NewRecorder()
c = e.NewContext(req, rec)
_ = cfg // Keep cfg used for test context
params := queryapi.GetCurrentFieldExtractionParams{
DocumentId: queryapi.DocumentID(documentID),
}
err = ctrl.GetCurrentFieldExtraction(c, params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var currentResp queryapi.FieldExtractionResponse
_ = json.Unmarshal(rec.Body.Bytes(), &currentResp)
assert.Equal(t, int32(2), currentResp.Version)
assert.Equal(t, "Version 2", *currentResp.SingleFields.ContractTitle)
}
// Helper function to create string pointer
func ptr(s string) *string {
return &s
}
+154
View File
@@ -0,0 +1,154 @@
package queryapi
import (
"net/http"
"time"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
"github.com/oapi-codegen/nullable"
openapi_types "github.com/oapi-codegen/runtime/types"
)
// CreateFolder creates a new folder for organizing documents.
// POST /folders
// Takes a FolderCreate request body containing path, clientId, parentId (optional), and createdBy.
// Returns the created Folder on success, or an error response.
func (s *Controllers) CreateFolder(ctx echo.Context) error {
var req CreateFolderJSONRequestBody
if err := ctx.Bind(&req); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, "invalid request body")
}
// Validate required fields
if req.Path == "" {
return echo.NewHTTPError(http.StatusBadRequest, "path is required")
}
if req.ClientId == "" {
return echo.NewHTTPError(http.StatusBadRequest, "clientId is required")
}
if req.CreatedBy == "" {
return echo.NewHTTPError(http.StatusBadRequest, "createdBy is required")
}
// Convert parent ID if provided
var parentID *uuid.UUID
if req.ParentId != nil {
id := uuid.UUID(*req.ParentId)
parentID = &id
}
folder, err := s.svc.Folder.CreateFolder(ctx.Request().Context(), req.Path, parentID, string(req.ClientId), string(req.CreatedBy))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
return ctx.JSON(http.StatusCreated, convertRepoFolderToAPI(folder))
}
// RenameFolder updates the path of an existing folder.
// PATCH /folders/{folderId}
// Takes a FolderRename request body containing the new path.
// Returns 204 No Content on success, or an error response.
func (s *Controllers) RenameFolder(ctx echo.Context, folderId openapi_types.UUID) error {
var req RenameFolderJSONRequestBody
if err := ctx.Bind(&req); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, "invalid request body")
}
if req.Path == "" {
return echo.NewHTTPError(http.StatusBadRequest, "path is required")
}
err := s.svc.Folder.RenameFolder(ctx.Request().Context(), uuid.UUID(folderId), req.Path)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
return ctx.NoContent(http.StatusNoContent)
}
// GetFolderDocuments retrieves all documents in a specific folder.
// GET /folders/{folderId}/documents
// Returns a list of documents in the specified folder.
func (s *Controllers) GetFolderDocuments(ctx echo.Context, folderId openapi_types.UUID) error {
documents, err := s.svc.Folder.GetDocumentsByFolder(ctx.Request().Context(), uuid.UUID(folderId))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Convert to API response format
apiDocs := make([]DocumentSummary, len(documents))
for i, doc := range documents {
apiDocs[i] = DocumentSummary{
Hash: doc.Hash,
Id: openapi_types.UUID(doc.ID),
}
}
return ctx.JSON(http.StatusOK, map[string]interface{}{
"documents": apiDocs,
})
}
// ListClientFolders retrieves all folders for a client.
// GET /client/{id}/folders
// Returns a FolderList containing all folders for the client.
// Each folder includes its ID, path, and parentId, allowing clients to reconstruct the folder hierarchy.
// Root-level folders will have parentId set to null.
func (s *Controllers) ListClientFolders(ctx echo.Context, id ClientID) error {
folders, err := s.svc.Folder.GetFoldersByClient(ctx.Request().Context(), string(id))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Convert repository folders to API folders
apiFolders := make([]Folder, len(folders))
for i, f := range folders {
apiFolders[i] = convertRepoFolderToAPI(f)
}
return ctx.JSON(http.StatusOK, FolderList{
Folders: apiFolders,
})
}
// GetFolderMetrics retrieves processing metrics for a folder including document counts by label.
// GET /folders/{folderId}/metrics
// Returns FolderMetrics containing total document count and breakdown by label.
func (s *Controllers) GetFolderMetrics(ctx echo.Context, folderId openapi_types.UUID) error {
metrics, err := s.svc.Folder.GetFolderMetrics(ctx.Request().Context(), uuid.UUID(folderId))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
return ctx.JSON(http.StatusOK, FolderMetrics{
FolderId: folderId,
TotalDocuments: metrics.TotalDocuments,
ByLabel: metrics.ByLabel,
})
}
// convertRepoFolderToAPI converts a repository Folder to the API Folder type.
func convertRepoFolderToAPI(folder *repository.Folder) Folder {
var createdAt time.Time
if folder.Createdat.Valid {
createdAt = folder.Createdat.Time
}
result := Folder{
Id: openapi_types.UUID(folder.ID),
Path: folder.Path,
ClientId: ClientID(folder.Clientid),
CreatedBy: openapi_types.Email(folder.Createdby),
CreatedAt: createdAt,
}
if folder.Parentid != nil {
result.ParentId = nullable.NewNullableWithValue(openapi_types.UUID(*folder.Parentid))
}
return result
}
+533
View File
@@ -0,0 +1,533 @@
package queryapi_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
queryapi "queryorchestration/api/queryAPI"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/folder"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type FoldersTestConfig struct {
serviceconfig.BaseConfig
}
func (c *FoldersTestConfig) GetBackgroundRunner() any { return nil }
func setupFoldersTestController(t *testing.T) (*queryapi.Controllers, *FoldersTestConfig, string) {
t.Helper()
cfg := &FoldersTestConfig{}
test.CreateDB(t, cfg)
// Create services
folderSvc := folder.New(cfg)
services := &queryapi.Services{
Folder: folderSvc,
}
// Create test client with unique ID
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID)
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: fmt.Sprintf("Test Client %s", uniqueID),
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
return ctrl, cfg, clientID
}
func TestCreateFolder_Success(t *testing.T) {
ctrl, _, clientID := setupFoldersTestController(t)
reqBody := queryapi.FolderCreate{
Path: "/documents/contracts",
ClientId: queryapi.ClientID(clientID),
CreatedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFolder(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
// Parse response
var resp queryapi.Folder
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, "/documents/contracts", resp.Path)
assert.Equal(t, clientID, string(resp.ClientId))
assert.Equal(t, "user@example.com", string(resp.CreatedBy))
assert.NotEqual(t, uuid.Nil, uuid.UUID(resp.Id))
}
func TestCreateFolder_WithParent(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create parent folder first
parentFolder, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/documents",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
reqBody := queryapi.FolderCreate{
Path: "/documents/contracts",
ClientId: queryapi.ClientID(clientID),
CreatedBy: openapi_types.Email("user@example.com"),
ParentId: (*openapi_types.UUID)(&parentFolder.ID),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFolder(c)
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp queryapi.Folder
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Verify parent ID is set
assert.True(t, resp.ParentId.IsSpecified())
parentID, err := resp.ParentId.Get()
require.NoError(t, err)
assert.Equal(t, parentFolder.ID, uuid.UUID(parentID))
}
func TestCreateFolder_MissingPath(t *testing.T) {
ctrl, _, clientID := setupFoldersTestController(t)
reqBody := queryapi.FolderCreate{
Path: "",
ClientId: queryapi.ClientID(clientID),
CreatedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFolder(c)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestCreateFolder_MissingClientId(t *testing.T) {
ctrl, _, _ := setupFoldersTestController(t)
reqBody := queryapi.FolderCreate{
Path: "/documents",
ClientId: "",
CreatedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.CreateFolder(c)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestCreateFolder_MissingCreatedBy(t *testing.T) {
ctrl, _, clientID := setupFoldersTestController(t)
// Send raw JSON with empty createdBy field
body := []byte(fmt.Sprintf(`{"path":"/documents","clientId":"%s","createdBy":""}`, clientID))
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/folders", bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.CreateFolder(c)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestRenameFolder_Success(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder first
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/old-path",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
reqBody := queryapi.FolderRename{
Path: "/new-path",
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPatch, fmt.Sprintf("/folders/%s", folderRec.ID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.RenameFolder(c, openapi_types.UUID(folderRec.ID))
require.NoError(t, err)
assert.Equal(t, http.StatusNoContent, rec.Code)
// Verify the folder was renamed
updatedFolder, err := cfg.GetDBQueries().GetFolderByID(ctx, folderRec.ID)
require.NoError(t, err)
assert.Equal(t, "/new-path", updatedFolder.Path)
}
func TestRenameFolder_MissingPath(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder first
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/old-path",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
reqBody := queryapi.FolderRename{
Path: "",
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPatch, fmt.Sprintf("/folders/%s", folderRec.ID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.RenameFolder(c, openapi_types.UUID(folderRec.ID))
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestGetFolderDocuments_Success(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/documents",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
// Create documents in the folder
filename1 := "doc1.pdf"
docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("hash1-%s", uuid.New().String()[:8]),
Filename: &filename1,
Folderid: &folderRec.ID,
})
require.NoError(t, err)
filename2 := "doc2.pdf"
docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("hash2-%s", uuid.New().String()[:8]),
Filename: &filename2,
Folderid: &folderRec.ID,
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentSummary `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Documents, 2)
// Verify both documents are returned
docIDs := make(map[uuid.UUID]bool)
for _, doc := range resp.Documents {
docIDs[uuid.UUID(doc.Id)] = true
}
assert.True(t, docIDs[docID1])
assert.True(t, docIDs[docID2])
}
func TestGetFolderDocuments_EmptyFolder(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder with no documents
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/empty-folder",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/documents", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderDocuments(c, openapi_types.UUID(folderRec.ID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentSummary `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Documents, 0)
}
func TestGetFolderMetrics_Success(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/documents",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
// Create documents in the folder
filename1 := "doc1.pdf"
docID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("metricshash1-%s", uuid.New().String()[:8]),
Filename: &filename1,
Folderid: &folderRec.ID,
})
require.NoError(t, err)
filename2 := "doc2.pdf"
docID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("metricshash2-%s", uuid.New().String()[:8]),
Filename: &filename2,
Folderid: &folderRec.ID,
})
require.NoError(t, err)
filename3 := "doc3.pdf"
_, err = cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("metricshash3-%s", uuid.New().String()[:8]),
Filename: &filename3,
Folderid: &folderRec.ID,
})
require.NoError(t, err)
// Apply labels to documents (using labels from the lookup table)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: docID1,
Label: "Ingested",
Appliedby: "user@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: docID2,
Label: "Ingested",
Appliedby: "user@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: docID2,
Label: "OCR_Processed",
Appliedby: "user@example.com",
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/metrics", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderMetrics(c, openapi_types.UUID(folderRec.ID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp queryapi.FolderMetrics
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, folderRec.ID, uuid.UUID(resp.FolderId))
assert.Equal(t, int32(3), resp.TotalDocuments)
assert.Equal(t, int32(2), resp.ByLabel["Ingested"])
assert.Equal(t, int32(1), resp.ByLabel["OCR_Processed"])
}
func TestGetFolderMetrics_EmptyFolder(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create folder with no documents
folderRec, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/empty-metrics",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/folders/%s/metrics", folderRec.ID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetFolderMetrics(c, openapi_types.UUID(folderRec.ID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp queryapi.FolderMetrics
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, int32(0), resp.TotalDocuments)
assert.Len(t, resp.ByLabel, 0)
}
func TestListClientFolders_Success(t *testing.T) {
ctrl, cfg, clientID := setupFoldersTestController(t)
ctx := t.Context()
// Create some folders for the client
folder1, err := cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/folder1",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/folder1/subfolder",
Clientid: clientID,
Parentid: &folder1.ID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().CreateFolder(ctx, &repository.CreateFolderParams{
Path: "/folder2",
Clientid: clientID,
Createdby: "admin@example.com",
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/client/%s/folders", clientID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.ListClientFolders(c, queryapi.ClientID(clientID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp queryapi.FolderList
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Should have 3 folders
assert.Len(t, resp.Folders, 3)
// Folders should be ordered by path
assert.Equal(t, "/folder1", resp.Folders[0].Path)
assert.Equal(t, "/folder1/subfolder", resp.Folders[1].Path)
assert.Equal(t, "/folder2", resp.Folders[2].Path)
}
func TestListClientFolders_EmptyResult(t *testing.T) {
ctrl, _, clientID := setupFoldersTestController(t)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/client/%s/folders", clientID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.ListClientFolders(c, queryapi.ClientID(clientID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp queryapi.FolderList
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
// Should have empty folders array
assert.Len(t, resp.Folders, 0)
}
+101
View File
@@ -0,0 +1,101 @@
package queryapi
import (
"net/http"
"time"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
)
// ApplyLabel applies a workflow label to a document.
// POST /documents/{documentId}/labels
// Takes a LabelApplication request body containing label and appliedBy.
// Returns the created LabelRecord on success, or an error response.
func (s *Controllers) ApplyLabel(ctx echo.Context, documentId openapi_types.UUID) error {
var req ApplyLabelJSONRequestBody
if err := ctx.Bind(&req); err != nil {
return echo.NewHTTPError(http.StatusBadRequest, "invalid request body")
}
// Validate required fields
if req.Label == "" {
return echo.NewHTTPError(http.StatusBadRequest, "label is required")
}
if req.AppliedBy == "" {
return echo.NewHTTPError(http.StatusBadRequest, "appliedBy is required")
}
label, err := s.svc.Label.ApplyLabel(ctx.Request().Context(), uuid.UUID(documentId), req.Label, string(req.AppliedBy))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
return ctx.JSON(http.StatusCreated, convertRepoLabelToAPI(label))
}
// GetDocumentLabels retrieves all labels applied to a specific document.
// GET /documents/{documentId}/labels
// Returns a list of LabelRecord objects ordered by most recent first.
func (s *Controllers) GetDocumentLabels(ctx echo.Context, documentId openapi_types.UUID) error {
labels, err := s.svc.Label.GetDocumentLabels(ctx.Request().Context(), uuid.UUID(documentId))
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Convert to API response format
apiLabels := make([]LabelRecord, len(labels))
for i, lbl := range labels {
apiLabels[i] = convertRepoLabelToAPI(lbl)
}
return ctx.JSON(http.StatusOK, map[string]interface{}{
"labels": apiLabels,
})
}
// GetDocumentsByLabel retrieves all documents with a specific label for a client.
// GET /labels/{labelName}/documents?clientId={clientId}
// Returns a list of document summaries that have the specified label.
func (s *Controllers) GetDocumentsByLabel(ctx echo.Context, labelName string, params GetDocumentsByLabelParams) error {
if labelName == "" {
return echo.NewHTTPError(http.StatusBadRequest, "labelName is required")
}
documents, err := s.svc.Label.GetDocumentsByLabel(ctx.Request().Context(), string(params.ClientId), labelName)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}
// Convert to API response format
apiDocs := make([]DocumentSummary, len(documents))
for i, doc := range documents {
apiDocs[i] = DocumentSummary{
Hash: doc.Hash,
Id: openapi_types.UUID(doc.ID),
}
}
return ctx.JSON(http.StatusOK, map[string]interface{}{
"documents": apiDocs,
})
}
// convertRepoLabelToAPI converts a repository Documentlabel to the API LabelRecord type.
func convertRepoLabelToAPI(label *repository.Documentlabel) LabelRecord {
var appliedAt time.Time
if label.Appliedat.Valid {
appliedAt = label.Appliedat.Time
}
return LabelRecord{
Id: openapi_types.UUID(label.ID),
DocumentId: openapi_types.UUID(label.Documentid),
Label: label.Label,
AppliedBy: openapi_types.Email(label.Appliedby),
AppliedAt: appliedAt,
}
}
+446
View File
@@ -0,0 +1,446 @@
package queryapi_test
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
queryapi "queryorchestration/api/queryAPI"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/label"
"queryorchestration/internal/serviceconfig"
"queryorchestration/internal/test"
"github.com/google/uuid"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type LabelsTestConfig struct {
serviceconfig.BaseConfig
}
func (c *LabelsTestConfig) GetBackgroundRunner() any { return nil }
func setupLabelsTestController(t *testing.T) (*queryapi.Controllers, *LabelsTestConfig, string, uuid.UUID) {
t.Helper()
cfg := &LabelsTestConfig{}
test.CreateDB(t, cfg)
// Create services
labelSvc := label.New(cfg)
services := &queryapi.Services{
Label: labelSvc,
}
// Create test client with unique ID
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID)
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: fmt.Sprintf("Test Client %s", uniqueID),
})
require.NoError(t, err)
// Create a test document
filename := "test.pdf"
documentID, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("labeltesthash-%s", uniqueID),
Filename: &filename,
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
return ctrl, cfg, clientID, documentID
}
func TestApplyLabel_Success(t *testing.T) {
ctrl, _, _, documentID := setupLabelsTestController(t)
// Use a label that exists in the labels lookup table (seeded in migration)
reqBody := queryapi.LabelApplication{
Label: "Ingested",
AppliedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/documents/%s/labels", documentID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.ApplyLabel(c, openapi_types.UUID(documentID))
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp queryapi.LabelRecord
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Equal(t, documentID, uuid.UUID(resp.DocumentId))
assert.Equal(t, "Ingested", resp.Label)
assert.Equal(t, "user@example.com", string(resp.AppliedBy))
assert.NotEqual(t, uuid.Nil, uuid.UUID(resp.Id))
}
func TestApplyLabel_MissingLabel(t *testing.T) {
ctrl, _, _, documentID := setupLabelsTestController(t)
reqBody := queryapi.LabelApplication{
Label: "",
AppliedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/documents/%s/labels", documentID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.ApplyLabel(c, openapi_types.UUID(documentID))
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestApplyLabel_MissingAppliedBy(t *testing.T) {
ctrl, _, _, documentID := setupLabelsTestController(t)
// Send raw JSON with empty appliedBy field
body := []byte(`{"label":"Ingested","appliedBy":""}`)
e := echo.New()
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/documents/%s/labels", documentID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.ApplyLabel(c, openapi_types.UUID(documentID))
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}
func TestApplyLabel_SameLabelMultipleTimes(t *testing.T) {
ctrl, _, _, documentID := setupLabelsTestController(t)
// Use a label that exists in the labels lookup table (seeded in migration)
reqBody := queryapi.LabelApplication{
Label: "Ingested",
AppliedBy: openapi_types.Email("user@example.com"),
}
body, err := json.Marshal(reqBody)
require.NoError(t, err)
// Apply label first time
e := echo.New()
req := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/documents/%s/labels", documentID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.ApplyLabel(c, openapi_types.UUID(documentID))
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp1 queryapi.LabelRecord
_ = json.Unmarshal(rec.Body.Bytes(), &resp1)
// Apply same label second time - should create another record for history
req = httptest.NewRequest(http.MethodPost, fmt.Sprintf("/documents/%s/labels", documentID), bytes.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec = httptest.NewRecorder()
c = e.NewContext(req, rec)
err = ctrl.ApplyLabel(c, openapi_types.UUID(documentID))
require.NoError(t, err)
assert.Equal(t, http.StatusCreated, rec.Code)
var resp2 queryapi.LabelRecord
_ = json.Unmarshal(rec.Body.Bytes(), &resp2)
// Both should have the same label but different record IDs
assert.Equal(t, "Ingested", resp1.Label)
assert.Equal(t, "Ingested", resp2.Label)
assert.NotEqual(t, resp1.Id, resp2.Id)
}
func TestGetDocumentLabels_Success(t *testing.T) {
ctrl, cfg, _, documentID := setupLabelsTestController(t)
ctx := t.Context()
// Apply multiple labels to the document (using labels from the lookup table)
_, err := cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID,
Label: "Ingested",
Appliedby: "user1@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID,
Label: "OCR_Processed",
Appliedby: "user2@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID,
Label: "GenAI_Processed",
Appliedby: "user3@example.com",
})
require.NoError(t, err)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/documents/%s/labels", documentID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err = ctrl.GetDocumentLabels(c, openapi_types.UUID(documentID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Labels []queryapi.LabelRecord `json:"labels"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Labels, 3)
// Verify all labels are present
labelNames := make(map[string]bool)
for _, lbl := range resp.Labels {
labelNames[lbl.Label] = true
}
assert.True(t, labelNames["Ingested"])
assert.True(t, labelNames["OCR_Processed"])
assert.True(t, labelNames["GenAI_Processed"])
}
func TestGetDocumentLabels_NoLabels(t *testing.T) {
ctrl, _, _, documentID := setupLabelsTestController(t)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/documents/%s/labels", documentID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
err := ctrl.GetDocumentLabels(c, openapi_types.UUID(documentID))
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Labels []queryapi.LabelRecord `json:"labels"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Labels, 0)
}
func TestGetDocumentsByLabel_Success(t *testing.T) {
cfg := &LabelsTestConfig{}
test.CreateDB(t, cfg)
// Create services
labelSvc := label.New(cfg)
services := &queryapi.Services{
Label: labelSvc,
}
// Create test client
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID) // Use normal string clientId
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: fmt.Sprintf("Test Client %s", clientID[:8]),
})
require.NoError(t, err)
// Create test documents
filename1 := "doc1.pdf"
documentID1, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("labelhash1-%s", uuid.New().String()[:8]),
Filename: &filename1,
})
require.NoError(t, err)
filename2 := "doc2.pdf"
documentID2, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("labelhash2-%s", uuid.New().String()[:8]),
Filename: &filename2,
})
require.NoError(t, err)
// Create a third document
filename3 := "doc3.pdf"
documentID3, err := cfg.GetDBQueries().CreateDocument(ctx, &repository.CreateDocumentParams{
Clientid: clientID,
Hash: fmt.Sprintf("labelhash3-%s", uuid.New().String()[:8]),
Filename: &filename3,
})
require.NoError(t, err)
// Apply "Ingested" label to doc1 and doc2, but not doc3 (using labels from lookup table)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID1,
Label: "Ingested",
Appliedby: "user@example.com",
})
require.NoError(t, err)
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID2,
Label: "Ingested",
Appliedby: "user@example.com",
})
require.NoError(t, err)
// Apply different label to doc3
_, err = cfg.GetDBQueries().ApplyLabel(ctx, &repository.ApplyLabelParams{
Documentid: documentID3,
Label: "OCR_Processed",
Appliedby: "user@example.com",
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
// Get documents with "Ingested" label
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/labels/Ingested/documents?clientId=%s", clientID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetDocumentsByLabelParams{
ClientId: queryapi.ClientID(clientID),
}
err = ctrl.GetDocumentsByLabel(c, "Ingested", params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentSummary `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Documents, 2)
// Verify the correct documents are returned
docIDs := make(map[uuid.UUID]bool)
for _, doc := range resp.Documents {
docIDs[uuid.UUID(doc.Id)] = true
}
assert.True(t, docIDs[documentID1])
assert.True(t, docIDs[documentID2])
assert.False(t, docIDs[documentID3])
}
func TestGetDocumentsByLabel_NoDocuments(t *testing.T) {
cfg := &LabelsTestConfig{}
test.CreateDB(t, cfg)
// Create services
labelSvc := label.New(cfg)
services := &queryapi.Services{
Label: labelSvc,
}
// Create test client
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID)
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: fmt.Sprintf("Test Client %s", uniqueID),
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/labels/nonexistent/documents?clientId=%s", clientID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetDocumentsByLabelParams{
ClientId: queryapi.ClientID(clientID),
}
err = ctrl.GetDocumentsByLabel(c, "nonexistent", params)
require.NoError(t, err)
assert.Equal(t, http.StatusOK, rec.Code)
var resp struct {
Documents []queryapi.DocumentSummary `json:"documents"`
}
err = json.Unmarshal(rec.Body.Bytes(), &resp)
require.NoError(t, err)
assert.Len(t, resp.Documents, 0)
}
func TestGetDocumentsByLabel_MissingLabelName(t *testing.T) {
cfg := &LabelsTestConfig{}
test.CreateDB(t, cfg)
// Create services
labelSvc := label.New(cfg)
services := &queryapi.Services{
Label: labelSvc,
}
// Create test client
ctx := t.Context()
uniqueID := uuid.New().String()[:8]
clientID := fmt.Sprintf("test-client-%s", uniqueID)
err := cfg.GetDBQueries().CreateClient(ctx, &repository.CreateClientParams{
Clientid: clientID,
Name: fmt.Sprintf("Test Client %s", uniqueID),
})
require.NoError(t, err)
ctrl := queryapi.NewControllers(services, nil)
e := echo.New()
req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/labels//documents?clientId=%s", clientID), nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
params := queryapi.GetDocumentsByLabelParams{
ClientId: queryapi.ClientID(clientID),
}
err = ctrl.GetDocumentsByLabel(c, "", params)
require.Error(t, err)
httpErr, ok := err.(*echo.HTTPError)
require.True(t, ok)
assert.Equal(t, http.StatusBadRequest, httpErr.Code)
}