Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support

* in progress

* stage 8 complete

* phase 9 completed

* phase 9 complete

* ongoing - s3 path fix

* working

* optimize ci build

* e2e tests

* missing test
This commit is contained in:
Jay Brown
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
+461
View File
@@ -0,0 +1,461 @@
package queryapi
import (
"fmt"
"log/slog"
"net/http"
"time"
"queryorchestration/internal/database/repository"
"queryorchestration/internal/fieldextraction"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
"github.com/labstack/echo/v4"
openapi_types "github.com/oapi-codegen/runtime/types"
)
// CreateFieldExtraction creates a new field extraction with single-value and array fields.
// This endpoint creates a versioned field extraction record for a document.
func (s *Controllers) CreateFieldExtraction(ctx echo.Context) error {
var req FieldExtractionRequest
if err := ctx.Bind(&req); err != nil {
slog.Error("failed to parse field extraction request", "error", err)
return echo.NewHTTPError(http.StatusBadRequest, "invalid request body")
}
// Validate required fields
documentID := uuid.UUID(req.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
createdBy := string(req.CreatedBy)
if createdBy == "" {
return echo.NewHTTPError(http.StatusBadRequest, "createdBy is required")
}
// Convert API request to service input
input := &fieldextraction.CreateFieldExtractionInput{
DocumentID: documentID,
SingleFields: convertAPISingleFieldsToRepo(&req.SingleFields, createdBy),
ArrayFields: convertAPIArrayFieldsToRepo(req.ArrayFields),
}
// Create field extraction via service
extraction, err := s.svc.FieldExtraction.CreateFieldExtraction(ctx.Request().Context(), input)
if err != nil {
slog.Error("failed to create field extraction", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to create field extraction")
}
// Fetch the current extraction to get version info
current, err := s.svc.FieldExtraction.GetCurrentFieldExtraction(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get current field extraction after creation", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get created field extraction")
}
// Fetch array fields for response
arrayFields, err := s.svc.FieldExtraction.GetFieldExtractionArrayFields(ctx.Request().Context(), extraction.ID)
if err != nil {
slog.Error("failed to get array fields after creation", "error", err, "extraction_id", extraction.ID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get array fields")
}
// Build response
response := buildFieldExtractionResponse(current, arrayFields)
return ctx.JSON(http.StatusCreated, response)
}
// GetCurrentFieldExtraction retrieves the most recent field extraction for a document.
// This returns the current (latest version) field extraction including all single and array fields.
func (s *Controllers) GetCurrentFieldExtraction(ctx echo.Context, params GetCurrentFieldExtractionParams) error {
documentID := uuid.UUID(params.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
// Get current field extraction
current, err := s.svc.FieldExtraction.GetCurrentFieldExtraction(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get current field extraction", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get field extraction")
}
if current == nil {
return echo.NewHTTPError(http.StatusNotFound, "no field extraction found for document")
}
// Fetch array fields
arrayFields, err := s.svc.FieldExtraction.GetFieldExtractionArrayFields(ctx.Request().Context(), current.ID)
if err != nil {
slog.Error("failed to get array fields", "error", err, "extraction_id", current.ID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get array fields")
}
// Build response
response := buildFieldExtractionResponse(current, arrayFields)
return ctx.JSON(http.StatusOK, response)
}
// GetFieldExtractionHistory retrieves all versions of field extractions for a document.
// This returns version metadata without the full field data for each version.
func (s *Controllers) GetFieldExtractionHistory(ctx echo.Context, params GetFieldExtractionHistoryParams) error {
documentID := uuid.UUID(params.DocumentId)
if documentID == uuid.Nil {
return echo.NewHTTPError(http.StatusBadRequest, "documentId is required")
}
// Get version history
history, err := s.svc.FieldExtraction.GetFieldExtractionHistory(ctx.Request().Context(), documentID)
if err != nil {
slog.Error("failed to get field extraction history", "error", err, "document_id", documentID)
return echo.NewHTTPError(http.StatusInternalServerError, "failed to get field extraction history")
}
// Convert to API response
versions := make([]FieldExtractionVersion, len(history))
for i, h := range history {
createdAt := time.Time{}
if h.Createdat.Valid {
createdAt = h.Createdat.Time
}
versions[i] = FieldExtractionVersion{
Id: openapi_types.UUID(h.ID),
DocumentId: DocumentID(h.Documentid),
//nolint:gosec // Version is a small number, int32 is sufficient
Version: int32(h.Version), // Already 1-based in database
CreatedAt: createdAt,
CreatedBy: openapi_types.Email(h.Createdby),
}
}
return ctx.JSON(http.StatusOK, struct {
Versions []FieldExtractionVersion `json:"versions"`
}{
Versions: versions,
})
}
// buildFieldExtractionResponse builds the API response from the repository types
func buildFieldExtractionResponse(current *repository.Currentfieldextraction, arrayFields []*repository.Documentfieldextractionarrayfield) *FieldExtractionResponse {
createdAt := time.Time{}
if current.Createdat.Valid {
createdAt = current.Createdat.Time
}
return &FieldExtractionResponse{
Id: openapi_types.UUID(current.ID),
DocumentId: DocumentID(current.Documentid),
//nolint:gosec // Version is a small number, int32 is sufficient
Version: int32(current.Version), // Already 1-based in database
SingleFields: convertRepoSingleFieldsToAPI(current),
ArrayFields: convertRepoArrayFieldsToAPI(arrayFields),
CreatedAt: createdAt,
CreatedBy: openapi_types.Email(current.Createdby),
}
}
// convertAPISingleFieldsToRepo converts API SingleFields to repository AddFieldExtractionParams
func convertAPISingleFieldsToRepo(sf *SingleFields, createdBy string) *repository.AddFieldExtractionParams {
params := &repository.AddFieldExtractionParams{
Createdby: createdBy,
}
if sf == nil {
return params
}
params.Filename = sf.FileName
params.Contracttitle = sf.ContractTitle
params.Aaretederivedamendmentnum = sf.AareteDerivedAmendmentNum
params.Clientname = sf.ClientName
params.Payername = sf.PayerName
params.Payerstate = sf.PayerState
params.Providerstate = sf.ProviderState
params.Filenametin = sf.FilenameTin
params.Provgrouptin = sf.ProvGroupTin
params.Provgroupnpi = sf.ProvGroupNpi
params.Provgroupnamefull = sf.ProvGroupNameFull
params.Provothertin = sf.ProvOtherTin
params.Provothernpi = sf.ProvOtherNpi
params.Provothernamefull = sf.ProvOtherNameFull
params.Autorenewalind = sf.AutoRenewalInd
params.Autorenewalterm = sf.AutoRenewalTerm
// Convert dates
if sf.AareteDerivedEffectiveDt != nil {
params.Aaretederivedeffectivedt = dateToPgDate(*sf.AareteDerivedEffectiveDt)
}
if sf.AareteDerivedTerminationDt != nil {
params.Aaretederivedterminationdt = dateToPgDate(*sf.AareteDerivedTerminationDt)
}
return params
}
// convertRepoSingleFieldsToAPI converts repository Currentfieldextraction to API SingleFields
func convertRepoSingleFieldsToAPI(cf *repository.Currentfieldextraction) SingleFields {
sf := SingleFields{
FileName: cf.Filename,
ContractTitle: cf.Contracttitle,
AareteDerivedAmendmentNum: cf.Aaretederivedamendmentnum,
ClientName: cf.Clientname,
PayerName: cf.Payername,
PayerState: cf.Payerstate,
ProviderState: cf.Providerstate,
FilenameTin: cf.Filenametin,
ProvGroupTin: cf.Provgrouptin,
ProvGroupNpi: cf.Provgroupnpi,
ProvGroupNameFull: cf.Provgroupnamefull,
ProvOtherTin: cf.Provothertin,
ProvOtherNpi: cf.Provothernpi,
ProvOtherNameFull: cf.Provothernamefull,
AutoRenewalInd: cf.Autorenewalind,
AutoRenewalTerm: cf.Autorenewalterm,
}
// Convert dates
if cf.Aaretederivedeffectivedt.Valid {
d := pgDateToAPIDate(cf.Aaretederivedeffectivedt)
sf.AareteDerivedEffectiveDt = &d
}
if cf.Aaretederivedterminationdt.Valid {
d := pgDateToAPIDate(cf.Aaretederivedterminationdt)
sf.AareteDerivedTerminationDt = &d
}
return sf
}
// convertAPIArrayFieldsToRepo converts API ArrayFieldItem slice to repository AddFieldExtractionArrayFieldParams slice
func convertAPIArrayFieldsToRepo(items []ArrayFieldItem) []*repository.AddFieldExtractionArrayFieldParams {
result := make([]*repository.AddFieldExtractionArrayFieldParams, len(items))
for i, item := range items {
result[i] = convertAPIArrayFieldItemToRepo(&item)
}
return result
}
// convertAPIArrayFieldItemToRepo converts a single API ArrayFieldItem to repository params
func convertAPIArrayFieldItemToRepo(item *ArrayFieldItem) *repository.AddFieldExtractionArrayFieldParams {
params := &repository.AddFieldExtractionArrayFieldParams{}
if item == nil {
return params
}
// String fields
params.Exhibittitle = item.ExhibitTitle
params.Exhibitpage = item.ExhibitPage
params.Reimbprovtin = item.ReimbProvTin
params.Reimbprovnpi = item.ReimbProvNpi
params.Reimbprovname = item.ReimbProvName
params.Aaretederivedclaimtypecd = item.AareteDerivedClaimTypeCd
params.Aaretederivedproduct = item.AareteDerivedProduct
params.Aaretederivedlob = item.AareteDerivedLob
params.Aaretederivedprogram = item.AareteDerivedProgram
params.Aaretederivednetwork = item.AareteDerivedNetwork
params.Aaretederivedprovtype = item.AareteDerivedProvType
params.Provtaxonomycd = item.ProvTaxonomyCd
params.Provtaxonomycddesc = item.ProvTaxonomyCdDesc
params.Provspecialtycd = item.ProvSpecialtyCd
params.Provspecialtycddesc = item.ProvSpecialtyCdDesc
params.Placeofservicecd = item.PlaceOfServiceCd
params.Placeofservicecddesc = item.PlaceOfServiceCdDesc
params.Billtypecd = item.BillTypeCd
params.Billtypecddesc = item.BillTypeCdDesc
params.Patientagemin = item.PatientAgeMin
params.Patientagemax = item.PatientAgeMax
params.Reimbterm = item.ReimbTerm
params.Lobprogramrelationship = item.LobProgramRelationship
params.Lobproductrelationship = item.LobProductRelationship
params.Aaretederivedreimbmethod = item.AareteDerivedReimbMethod
params.Unitofmeasure = item.UnitOfMeasure
params.Additiondesc = item.AdditionDesc
params.Aaretederivedadditionratechangetimeline = item.AareteDerivedAdditionRateChangeTimeline
params.Aaretederivedfeeschedule = item.AareteDerivedFeeSchedule
params.Aaretederivedfeescheduleversion = item.AareteDerivedFeeScheduleVersion
params.Serviceterm = item.ServiceTerm
params.Cpt4proccd = item.Cpt4ProcCd
params.Cpt4proccddesc = item.Cpt4ProcCdDesc
params.Cpt4procmod = item.Cpt4ProcMod
params.Cpt4procmoddesc = item.Cpt4ProcModDesc
params.Revenuecd = item.RevenueCd
params.Revenuecddesc = item.RevenueCdDesc
params.Diagcd = item.DiagCd
params.Diagcddesc = item.DiagCdDesc
params.Ndccd = item.NdcCd
params.Ndccddesc = item.NdcCdDesc
params.Claimadmittypecd = item.ClaimAdmitTypeCd
params.Authadmittypedesc = item.AuthAdmitTypeDesc
params.Claimstatuscd = item.ClaimStatusCd
params.Claimstatuscddesc = item.ClaimStatusCdDesc
params.Groupertype = item.GrouperType
params.Groupercd = item.GrouperCd
params.Groupercddesc = item.GrouperCdDesc
params.Carveoutcd = item.CarveoutCd
// Boolean fields
params.Carveoutind = item.CarveoutInd
params.Lesserofind = item.LesserOfInd
params.Greaterofind = item.GreaterOfInd
params.Defaultind = item.DefaultInd
// Numeric fields
params.Reimbpctrate = float64ToPgNumeric(item.ReimbPctRate)
params.Reimbfeerate = float64ToPgNumeric(item.ReimbFeeRate)
params.Reimbconversionfactor = float64ToPgNumeric(item.ReimbConversionFactor)
params.Triggercapthresholdamt = float64ToPgNumeric(item.TriggerCapThresholdAmt)
params.Triggerbasethreshold = float64ToPgNumeric(item.TriggerBaseThreshold)
params.Additionmaxfeerateinc = float64ToPgNumeric(item.AdditionMaxFeeRateInc)
params.Additionmaxpctrateinc = float64ToPgNumeric(item.AdditionMaxPctRateInc)
params.Grouperpctrate = float64ToPgNumeric(item.GrouperPctRate)
params.Grouperbaserate = float64ToPgNumeric(item.GrouperBaseRate)
// Date fields
if item.ReimbEffectiveDt != nil {
params.Reimbeffectivedt = dateToPgDate(*item.ReimbEffectiveDt)
}
if item.ReimbTerminationDt != nil {
params.Reimbterminationdt = dateToPgDate(*item.ReimbTerminationDt)
}
return params
}
// convertRepoArrayFieldsToAPI converts repository array fields to API ArrayFieldItem slice
func convertRepoArrayFieldsToAPI(fields []*repository.Documentfieldextractionarrayfield) []ArrayFieldItem {
result := make([]ArrayFieldItem, len(fields))
for i, f := range fields {
result[i] = convertRepoArrayFieldToAPI(f)
}
return result
}
// convertRepoArrayFieldToAPI converts a single repository array field to API ArrayFieldItem
func convertRepoArrayFieldToAPI(f *repository.Documentfieldextractionarrayfield) ArrayFieldItem {
item := ArrayFieldItem{
// String fields
ExhibitTitle: f.Exhibittitle,
ExhibitPage: f.Exhibitpage,
ReimbProvTin: f.Reimbprovtin,
ReimbProvNpi: f.Reimbprovnpi,
ReimbProvName: f.Reimbprovname,
AareteDerivedClaimTypeCd: f.Aaretederivedclaimtypecd,
AareteDerivedProduct: f.Aaretederivedproduct,
AareteDerivedLob: f.Aaretederivedlob,
AareteDerivedProgram: f.Aaretederivedprogram,
AareteDerivedNetwork: f.Aaretederivednetwork,
AareteDerivedProvType: f.Aaretederivedprovtype,
ProvTaxonomyCd: f.Provtaxonomycd,
ProvTaxonomyCdDesc: f.Provtaxonomycddesc,
ProvSpecialtyCd: f.Provspecialtycd,
ProvSpecialtyCdDesc: f.Provspecialtycddesc,
PlaceOfServiceCd: f.Placeofservicecd,
PlaceOfServiceCdDesc: f.Placeofservicecddesc,
BillTypeCd: f.Billtypecd,
BillTypeCdDesc: f.Billtypecddesc,
PatientAgeMin: f.Patientagemin,
PatientAgeMax: f.Patientagemax,
ReimbTerm: f.Reimbterm,
LobProgramRelationship: f.Lobprogramrelationship,
LobProductRelationship: f.Lobproductrelationship,
AareteDerivedReimbMethod: f.Aaretederivedreimbmethod,
UnitOfMeasure: f.Unitofmeasure,
AdditionDesc: f.Additiondesc,
AareteDerivedAdditionRateChangeTimeline: f.Aaretederivedadditionratechangetimeline,
AareteDerivedFeeSchedule: f.Aaretederivedfeeschedule,
AareteDerivedFeeScheduleVersion: f.Aaretederivedfeescheduleversion,
ServiceTerm: f.Serviceterm,
Cpt4ProcCd: f.Cpt4proccd,
Cpt4ProcCdDesc: f.Cpt4proccddesc,
Cpt4ProcMod: f.Cpt4procmod,
Cpt4ProcModDesc: f.Cpt4procmoddesc,
RevenueCd: f.Revenuecd,
RevenueCdDesc: f.Revenuecddesc,
DiagCd: f.Diagcd,
DiagCdDesc: f.Diagcddesc,
NdcCd: f.Ndccd,
NdcCdDesc: f.Ndccddesc,
ClaimAdmitTypeCd: f.Claimadmittypecd,
AuthAdmitTypeDesc: f.Authadmittypedesc,
ClaimStatusCd: f.Claimstatuscd,
ClaimStatusCdDesc: f.Claimstatuscddesc,
GrouperType: f.Groupertype,
GrouperCd: f.Groupercd,
GrouperCdDesc: f.Groupercddesc,
CarveoutCd: f.Carveoutcd,
// Boolean fields
CarveoutInd: f.Carveoutind,
LesserOfInd: f.Lesserofind,
GreaterOfInd: f.Greaterofind,
DefaultInd: f.Defaultind,
}
// Numeric fields
item.ReimbPctRate = pgNumericToFloat64(f.Reimbpctrate)
item.ReimbFeeRate = pgNumericToFloat64(f.Reimbfeerate)
item.ReimbConversionFactor = pgNumericToFloat64(f.Reimbconversionfactor)
item.TriggerCapThresholdAmt = pgNumericToFloat64(f.Triggercapthresholdamt)
item.TriggerBaseThreshold = pgNumericToFloat64(f.Triggerbasethreshold)
item.AdditionMaxFeeRateInc = pgNumericToFloat64(f.Additionmaxfeerateinc)
item.AdditionMaxPctRateInc = pgNumericToFloat64(f.Additionmaxpctrateinc)
item.GrouperPctRate = pgNumericToFloat64(f.Grouperpctrate)
item.GrouperBaseRate = pgNumericToFloat64(f.Grouperbaserate)
// Date fields
if f.Reimbeffectivedt.Valid {
d := pgDateToAPIDate(f.Reimbeffectivedt)
item.ReimbEffectiveDt = &d
}
if f.Reimbterminationdt.Valid {
d := pgDateToAPIDate(f.Reimbterminationdt)
item.ReimbTerminationDt = &d
}
return item
}
// dateToPgDate converts openapi_types.Date to pgtype.Date
func dateToPgDate(d openapi_types.Date) pgtype.Date {
return pgtype.Date{
Time: d.Time,
Valid: true,
}
}
// pgDateToAPIDate converts pgtype.Date to openapi_types.Date
func pgDateToAPIDate(d pgtype.Date) openapi_types.Date {
return openapi_types.Date{
Time: d.Time,
}
}
// float64ToPgNumeric converts *float64 to pgtype.Numeric
func float64ToPgNumeric(f *float64) pgtype.Numeric {
if f == nil {
return pgtype.Numeric{}
}
var num pgtype.Numeric
// Scan doesn't accept float64 directly, need to convert to string
_ = num.Scan(fmt.Sprintf("%v", *f))
return num
}
// pgNumericToFloat64 converts pgtype.Numeric to *float64
func pgNumericToFloat64(n pgtype.Numeric) *float64 {
if !n.Valid {
return nil
}
f, err := n.Float64Value()
if err != nil || !f.Valid {
return nil
}
return &f.Float64
}