Merged in feature/textExtractionsPart2 (pull request #193)

Continue finishing the parts of the text extraction plan

* add missing fields
This commit is contained in:
Jay Brown
2025-12-02 19:13:08 +00:00
parent c45e1dd427
commit 3bdc27f4de
21 changed files with 2392 additions and 215 deletions
+20
View File
@@ -0,0 +1,20 @@
package client
import (
"context"
)
// List returns all clients ordered by name.
func (s *Service) List(ctx context.Context) ([]*Client, error) {
clients, err := s.cfg.GetDBQueries().ListClients(ctx)
if err != nil {
return nil, err
}
result := make([]*Client, len(clients))
for i, c := range clients {
result[i] = parseFullClient(c)
}
return result, nil
}
@@ -0,0 +1,10 @@
-- Remove the unique constraint and documentId column
DROP INDEX IF EXISTS idx_documentfieldextractionversions_documentid;
DROP INDEX IF EXISTS idx_unique_version_per_document;
ALTER TABLE documentFieldExtractionVersions
DROP CONSTRAINT IF EXISTS fk_documentfieldextractionversions_documentid;
ALTER TABLE documentFieldExtractionVersions
DROP COLUMN IF EXISTS documentId;
@@ -0,0 +1,30 @@
-- Add unique constraint to prevent duplicate versions per document
-- This ensures that concurrent requests cannot create the same version number
-- First, add documentId column to versions table for easier constraint enforcement
ALTER TABLE documentFieldExtractionVersions
ADD COLUMN documentId uuid;
-- Populate documentId from the joined extraction record
UPDATE documentFieldExtractionVersions dfev
SET documentId = dfe.documentId
FROM documentFieldExtractions dfe
WHERE dfev.fieldExtractionId = dfe.id;
-- Make documentId NOT NULL after populating
ALTER TABLE documentFieldExtractionVersions
ALTER COLUMN documentId SET NOT NULL;
-- Add foreign key constraint (documents table has 'id' as primary key)
ALTER TABLE documentFieldExtractionVersions
ADD CONSTRAINT fk_documentfieldextractionversions_documentid
FOREIGN KEY (documentId) REFERENCES documents(id);
-- Add unique constraint on documentId + version
-- This prevents duplicate version numbers for the same document
CREATE UNIQUE INDEX idx_unique_version_per_document
ON documentFieldExtractionVersions(documentId, version);
-- Add index for efficient lookups by documentId
CREATE INDEX idx_documentfieldextractionversions_documentid
ON documentFieldExtractionVersions(documentId);
+4
View File
@@ -4,6 +4,10 @@ INSERT INTO clients (clientId, name) VALUES ($1, $2);
-- name: GetClient :one
SELECT * FROM fullClients WHERE clientId = $1;
-- name: ListClients :many
-- Returns all clients ordered by name
SELECT * FROM fullClients ORDER BY name;
-- name: UpdateClient :exec
UPDATE clients SET name = $1 WHERE clientId = $2;
+52 -2
View File
@@ -158,13 +158,63 @@ INSERT INTO documentFieldExtractionArrayFields (
);
-- name: AddFieldExtractionEntry :exec
INSERT INTO documentFieldExtractionVersions (fieldExtractionId, version, createdBy)
VALUES ($1, $2, $3);
-- Insert a new version entry for a field extraction
-- documentId is required for the unique constraint enforcement
INSERT INTO documentFieldExtractionVersions (fieldExtractionId, documentId, version, createdBy)
VALUES ($1, $2, $3, $4);
-- name: LockFieldExtractionVersionsForDocument :many
-- Lock all existing version rows for a document to prevent concurrent inserts
-- Must be called within a transaction before GetMaxFieldExtractionVersion
-- Returns the locked rows (which may be empty for first version)
SELECT id, version FROM documentFieldExtractionVersions
WHERE documentId = $1
FOR UPDATE;
-- name: GetMaxFieldExtractionVersion :one
-- Get the current max version for a document (call after LockFieldExtractionVersionsForDocument)
-- Returns 0 if no versions exist, otherwise the max version number
-- COALESCE ensures we always get a non-NULL value that can be scanned into int64
SELECT COALESCE(MAX(version), 0) as max_version
FROM documentFieldExtractionVersions
WHERE documentId = $1;
-- name: GetCurrentFieldExtraction :one
SELECT * FROM currentFieldExtractions
WHERE documentId = $1;
-- name: GetFieldExtractionByVersion :one
-- Retrieves a specific version of field extraction for a document
-- Returns the field extraction record with version metadata
SELECT
dfe.id,
dfe.documentId,
dfe.fileName,
dfe.contractTitle,
dfe.aareteDerivedAmendmentNum,
dfe.clientName,
dfe.payerName,
dfe.payerState,
dfe.providerState,
dfe.filenameTin,
dfe.provGroupTin,
dfe.provGroupNpi,
dfe.provGroupNameFull,
dfe.provOtherTin,
dfe.provOtherNpi,
dfe.provOtherNameFull,
dfe.aareteDerivedEffectiveDt,
dfe.aareteDerivedTerminationDt,
dfe.autoRenewalInd,
dfe.autoRenewalTerm,
dfev.version,
dfev.createdBy,
dfev.createdAt
FROM documentFieldExtractions dfe
JOIN documentFieldExtractionVersions dfev
ON dfev.fieldExtractionId = dfe.id
WHERE dfev.documentId = $1 AND dfev.version = $2;
-- name: GetCurrentFieldExtractionWithArrayCount :one
SELECT * FROM currentFieldExtractionsWithArrayCount
WHERE documentId = $1;
@@ -255,6 +255,33 @@ func (q *Queries) IsClientSynced(ctx context.Context, dollar_1 *string) (bool, e
return is_synced, err
}
const listClients = `-- name: ListClients :many
SELECT clientid, name, cansync FROM fullClients ORDER BY name
`
// Returns all clients ordered by name
//
// SELECT clientid, name, cansync FROM fullClients ORDER BY name
func (q *Queries) ListClients(ctx context.Context) ([]*Fullclient, error) {
rows, err := q.db.Query(ctx, listClients)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*Fullclient{}
for rows.Next() {
var i Fullclient
if err := rows.Scan(&i.Clientid, &i.Name, &i.Cansync); err != nil {
return nil, err
}
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const updateClient = `-- name: UpdateClient :exec
UPDATE clients SET name = $1 WHERE clientId = $2
`
@@ -644,22 +644,29 @@ func (q *Queries) AddFieldExtractionArrayField(ctx context.Context, arg *AddFiel
}
const addFieldExtractionEntry = `-- name: AddFieldExtractionEntry :exec
INSERT INTO documentFieldExtractionVersions (fieldExtractionId, version, createdBy)
VALUES ($1, $2, $3)
INSERT INTO documentFieldExtractionVersions (fieldExtractionId, documentId, version, createdBy)
VALUES ($1, $2, $3, $4)
`
type AddFieldExtractionEntryParams struct {
Fieldextractionid uuid.UUID `db:"fieldextractionid"`
Documentid uuid.UUID `db:"documentid"`
Version int64 `db:"version"`
Createdby string `db:"createdby"`
}
// AddFieldExtractionEntry
// Insert a new version entry for a field extraction
// documentId is required for the unique constraint enforcement
//
// INSERT INTO documentFieldExtractionVersions (fieldExtractionId, version, createdBy)
// VALUES ($1, $2, $3)
// INSERT INTO documentFieldExtractionVersions (fieldExtractionId, documentId, version, createdBy)
// VALUES ($1, $2, $3, $4)
func (q *Queries) AddFieldExtractionEntry(ctx context.Context, arg *AddFieldExtractionEntryParams) error {
_, err := q.db.Exec(ctx, addFieldExtractionEntry, arg.Fieldextractionid, arg.Version, arg.Createdby)
_, err := q.db.Exec(ctx, addFieldExtractionEntry,
arg.Fieldextractionid,
arg.Documentid,
arg.Version,
arg.Createdby,
)
return err
}
@@ -930,6 +937,130 @@ func (q *Queries) GetFieldExtractionByID(ctx context.Context, id uuid.UUID) (*Do
return &i, err
}
const getFieldExtractionByVersion = `-- name: GetFieldExtractionByVersion :one
SELECT
dfe.id,
dfe.documentId,
dfe.fileName,
dfe.contractTitle,
dfe.aareteDerivedAmendmentNum,
dfe.clientName,
dfe.payerName,
dfe.payerState,
dfe.providerState,
dfe.filenameTin,
dfe.provGroupTin,
dfe.provGroupNpi,
dfe.provGroupNameFull,
dfe.provOtherTin,
dfe.provOtherNpi,
dfe.provOtherNameFull,
dfe.aareteDerivedEffectiveDt,
dfe.aareteDerivedTerminationDt,
dfe.autoRenewalInd,
dfe.autoRenewalTerm,
dfev.version,
dfev.createdBy,
dfev.createdAt
FROM documentFieldExtractions dfe
JOIN documentFieldExtractionVersions dfev
ON dfev.fieldExtractionId = dfe.id
WHERE dfev.documentId = $1 AND dfev.version = $2
`
type GetFieldExtractionByVersionParams struct {
Documentid uuid.UUID `db:"documentid"`
Version int64 `db:"version"`
}
type GetFieldExtractionByVersionRow struct {
ID uuid.UUID `db:"id"`
Documentid uuid.UUID `db:"documentid"`
Filename *string `db:"filename"`
Contracttitle *string `db:"contracttitle"`
Aaretederivedamendmentnum *int32 `db:"aaretederivedamendmentnum"`
Clientname *string `db:"clientname"`
Payername *string `db:"payername"`
Payerstate *string `db:"payerstate"`
Providerstate *string `db:"providerstate"`
Filenametin *string `db:"filenametin"`
Provgrouptin *string `db:"provgrouptin"`
Provgroupnpi *string `db:"provgroupnpi"`
Provgroupnamefull *string `db:"provgroupnamefull"`
Provothertin *string `db:"provothertin"`
Provothernpi *string `db:"provothernpi"`
Provothernamefull *string `db:"provothernamefull"`
Aaretederivedeffectivedt pgtype.Date `db:"aaretederivedeffectivedt"`
Aaretederivedterminationdt pgtype.Date `db:"aaretederivedterminationdt"`
Autorenewalind *bool `db:"autorenewalind"`
Autorenewalterm *string `db:"autorenewalterm"`
Version int64 `db:"version"`
Createdby string `db:"createdby"`
Createdat pgtype.Timestamp `db:"createdat"`
}
// Retrieves a specific version of field extraction for a document
// Returns the field extraction record with version metadata
//
// SELECT
// dfe.id,
// dfe.documentId,
// dfe.fileName,
// dfe.contractTitle,
// dfe.aareteDerivedAmendmentNum,
// dfe.clientName,
// dfe.payerName,
// dfe.payerState,
// dfe.providerState,
// dfe.filenameTin,
// dfe.provGroupTin,
// dfe.provGroupNpi,
// dfe.provGroupNameFull,
// dfe.provOtherTin,
// dfe.provOtherNpi,
// dfe.provOtherNameFull,
// dfe.aareteDerivedEffectiveDt,
// dfe.aareteDerivedTerminationDt,
// dfe.autoRenewalInd,
// dfe.autoRenewalTerm,
// dfev.version,
// dfev.createdBy,
// dfev.createdAt
// FROM documentFieldExtractions dfe
// JOIN documentFieldExtractionVersions dfev
// ON dfev.fieldExtractionId = dfe.id
// WHERE dfev.documentId = $1 AND dfev.version = $2
func (q *Queries) GetFieldExtractionByVersion(ctx context.Context, arg *GetFieldExtractionByVersionParams) (*GetFieldExtractionByVersionRow, error) {
row := q.db.QueryRow(ctx, getFieldExtractionByVersion, arg.Documentid, arg.Version)
var i GetFieldExtractionByVersionRow
err := row.Scan(
&i.ID,
&i.Documentid,
&i.Filename,
&i.Contracttitle,
&i.Aaretederivedamendmentnum,
&i.Clientname,
&i.Payername,
&i.Payerstate,
&i.Providerstate,
&i.Filenametin,
&i.Provgrouptin,
&i.Provgroupnpi,
&i.Provgroupnamefull,
&i.Provothertin,
&i.Provothernpi,
&i.Provothernamefull,
&i.Aaretederivedeffectivedt,
&i.Aaretederivedterminationdt,
&i.Autorenewalind,
&i.Autorenewalterm,
&i.Version,
&i.Createdby,
&i.Createdat,
)
return &i, err
}
const getFieldExtractionHistory = `-- name: GetFieldExtractionHistory :many
SELECT
dfe.id,
@@ -1045,6 +1176,64 @@ func (q *Queries) GetFieldExtractionsByDocumentID(ctx context.Context, documenti
return items, nil
}
const getMaxFieldExtractionVersion = `-- name: GetMaxFieldExtractionVersion :one
SELECT COALESCE(MAX(version), 0) as max_version
FROM documentFieldExtractionVersions
WHERE documentId = $1
`
// Get the current max version for a document (call after LockFieldExtractionVersionsForDocument)
// Returns 0 if no versions exist, otherwise the max version number
// COALESCE ensures we always get a non-NULL value that can be scanned into int64
//
// SELECT COALESCE(MAX(version), 0) as max_version
// FROM documentFieldExtractionVersions
// WHERE documentId = $1
func (q *Queries) GetMaxFieldExtractionVersion(ctx context.Context, documentid uuid.UUID) (*int64, error) {
row := q.db.QueryRow(ctx, getMaxFieldExtractionVersion, documentid)
var max_version *int64
err := row.Scan(&max_version)
return max_version, err
}
const lockFieldExtractionVersionsForDocument = `-- name: LockFieldExtractionVersionsForDocument :many
SELECT id, version FROM documentFieldExtractionVersions
WHERE documentId = $1
FOR UPDATE
`
type LockFieldExtractionVersionsForDocumentRow struct {
ID uuid.UUID `db:"id"`
Version int64 `db:"version"`
}
// Lock all existing version rows for a document to prevent concurrent inserts
// Must be called within a transaction before GetMaxFieldExtractionVersion
// Returns the locked rows (which may be empty for first version)
//
// SELECT id, version FROM documentFieldExtractionVersions
// WHERE documentId = $1
// FOR UPDATE
func (q *Queries) LockFieldExtractionVersionsForDocument(ctx context.Context, documentid uuid.UUID) ([]*LockFieldExtractionVersionsForDocumentRow, error) {
rows, err := q.db.Query(ctx, lockFieldExtractionVersionsForDocument, documentid)
if err != nil {
return nil, err
}
defer rows.Close()
items := []*LockFieldExtractionVersionsForDocumentRow{}
for rows.Next() {
var i LockFieldExtractionVersionsForDocumentRow
if err := rows.Scan(&i.ID, &i.Version); err != nil {
return nil, err
}
items = append(items, &i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const validateArrayFieldCount = `-- name: ValidateArrayFieldCount :one
SELECT COUNT(*) as arrayFieldCount
FROM documentFieldExtractionArrayFields
+1
View File
@@ -602,6 +602,7 @@ type Documentfieldextractionversion struct {
Version int64 `db:"version"`
Createdby string `db:"createdby"`
Createdat pgtype.Timestamp `db:"createdat"`
Documentid uuid.UUID `db:"documentid"`
}
type Documentlabel struct {
+34 -2
View File
@@ -1,6 +1,8 @@
package fieldextraction
import (
"fmt"
"queryorchestration/internal/database/repository"
"github.com/google/uuid"
@@ -13,8 +15,38 @@ type CreateFieldExtractionInput struct {
ArrayFields []*repository.AddFieldExtractionArrayFieldParams // All arrays must be same length N
}
// ValidateArrayConsistency checks that all array fields have consistent length
// Returns the array length if valid, error if inconsistent
// MaxArrayFieldCount is the maximum number of array field rows allowed per field extraction.
// This prevents excessively large payloads that could impact performance.
const MaxArrayFieldCount = 10000
// ValidateArrayConsistency validates the array fields for a field extraction.
// It checks:
// - No nil entries in the array
// - Array length does not exceed MaxArrayFieldCount
// - Empty arrays are allowed (returns 0, nil)
//
// Note: The service layer assigns arrayIndex values sequentially (0, 1, 2, ...)
// when inserting, so we don't validate incoming arrayIndex values here.
// The database enforces uniqueness and non-negative constraints.
//
// Returns the array length if valid, error if validation fails.
func ValidateArrayConsistency(arrayFields []*repository.AddFieldExtractionArrayFieldParams) (int, error) {
// Empty array is valid
if len(arrayFields) == 0 {
return 0, nil
}
// Check for maximum array size to prevent performance issues
if len(arrayFields) > MaxArrayFieldCount {
return 0, fmt.Errorf("array field count %d exceeds maximum allowed %d", len(arrayFields), MaxArrayFieldCount)
}
// Check for nil entries which would cause issues during insertion
for i, field := range arrayFields {
if field == nil {
return 0, fmt.Errorf("array field at index %d is nil", i)
}
}
return len(arrayFields), nil
}
+111
View File
@@ -0,0 +1,111 @@
package fieldextraction
import (
"testing"
"queryorchestration/internal/database/repository"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateArrayConsistency(t *testing.T) {
t.Run("empty array is valid", func(t *testing.T) {
count, err := ValidateArrayConsistency([]*repository.AddFieldExtractionArrayFieldParams{})
require.NoError(t, err)
assert.Equal(t, 0, count)
})
t.Run("nil slice is valid", func(t *testing.T) {
count, err := ValidateArrayConsistency(nil)
require.NoError(t, err)
assert.Equal(t, 0, count)
})
t.Run("single element array is valid", func(t *testing.T) {
fields := []*repository.AddFieldExtractionArrayFieldParams{
{},
}
count, err := ValidateArrayConsistency(fields)
require.NoError(t, err)
assert.Equal(t, 1, count)
})
t.Run("multiple elements array is valid", func(t *testing.T) {
exhibitTitle1 := "Exhibit A"
exhibitTitle2 := "Exhibit B"
exhibitTitle3 := "Exhibit C"
fields := []*repository.AddFieldExtractionArrayFieldParams{
{Exhibittitle: &exhibitTitle1},
{Exhibittitle: &exhibitTitle2},
{Exhibittitle: &exhibitTitle3},
}
count, err := ValidateArrayConsistency(fields)
require.NoError(t, err)
assert.Equal(t, 3, count)
})
t.Run("nil entry in array returns error", func(t *testing.T) {
exhibitTitle := "Exhibit A"
fields := []*repository.AddFieldExtractionArrayFieldParams{
{Exhibittitle: &exhibitTitle},
nil, // nil entry
{Exhibittitle: &exhibitTitle},
}
count, err := ValidateArrayConsistency(fields)
require.Error(t, err)
assert.Equal(t, 0, count)
assert.Contains(t, err.Error(), "array field at index 1 is nil")
})
t.Run("nil entry at first position returns error", func(t *testing.T) {
exhibitTitle := "Exhibit A"
fields := []*repository.AddFieldExtractionArrayFieldParams{
nil, // nil entry at first position
{Exhibittitle: &exhibitTitle},
}
count, err := ValidateArrayConsistency(fields)
require.Error(t, err)
assert.Equal(t, 0, count)
assert.Contains(t, err.Error(), "array field at index 0 is nil")
})
t.Run("nil entry at last position returns error", func(t *testing.T) {
exhibitTitle := "Exhibit A"
fields := []*repository.AddFieldExtractionArrayFieldParams{
{Exhibittitle: &exhibitTitle},
{Exhibittitle: &exhibitTitle},
nil, // nil entry at last position
}
count, err := ValidateArrayConsistency(fields)
require.Error(t, err)
assert.Equal(t, 0, count)
assert.Contains(t, err.Error(), "array field at index 2 is nil")
})
t.Run("array at max count is valid", func(t *testing.T) {
fields := make([]*repository.AddFieldExtractionArrayFieldParams, MaxArrayFieldCount)
for i := range fields {
fields[i] = &repository.AddFieldExtractionArrayFieldParams{}
}
count, err := ValidateArrayConsistency(fields)
require.NoError(t, err)
assert.Equal(t, MaxArrayFieldCount, count)
})
t.Run("array exceeding max count returns error", func(t *testing.T) {
fields := make([]*repository.AddFieldExtractionArrayFieldParams, MaxArrayFieldCount+1)
for i := range fields {
fields[i] = &repository.AddFieldExtractionArrayFieldParams{}
}
count, err := ValidateArrayConsistency(fields)
require.Error(t, err)
assert.Equal(t, 0, count)
assert.Contains(t, err.Error(), "exceeds maximum allowed")
})
}
func TestMaxArrayFieldCount(t *testing.T) {
// Verify the constant is set to a reasonable value
assert.Equal(t, 10000, MaxArrayFieldCount, "MaxArrayFieldCount should be 10000")
}
+48 -5
View File
@@ -67,16 +67,32 @@ func (s *Service) CreateFieldExtraction(ctx context.Context, input *CreateFieldE
return nil, fmt.Errorf("failed to create field extraction: %w", err)
}
// Get the next version number for this document
history, err := queries.GetFieldExtractionsByDocumentID(ctx, input.DocumentID)
// Lock existing version rows for this document to prevent concurrent version creation
// This ensures that two concurrent requests cannot create the same version number
_, err = queries.LockFieldExtractionVersionsForDocument(ctx, input.DocumentID)
if err != nil {
return nil, fmt.Errorf("failed to get field extraction history: %w", err)
return nil, fmt.Errorf("failed to lock version rows: %w", err)
}
version := int64(len(history))
// Create version entry
// Get the max version number for this document (after locking)
// COALESCE in the query ensures we get 0 if no versions exist
maxVersionPtr, err := queries.GetMaxFieldExtractionVersion(ctx, input.DocumentID)
if err != nil {
return nil, fmt.Errorf("failed to get max version: %w", err)
}
// SQLC returns *int64 even though COALESCE guarantees non-NULL
// Dereference safely - if somehow nil, treat as 0
var maxVersion int64
if maxVersionPtr != nil {
maxVersion = *maxVersionPtr
}
// Next version is always maxVersion + 1 (1 for first version since maxVersion is 0)
version := maxVersion + 1
// Create version entry with documentId for the unique constraint
err = queries.AddFieldExtractionEntry(ctx, &repository.AddFieldExtractionEntryParams{
Fieldextractionid: fieldExtraction.ID,
Documentid: input.DocumentID,
Version: version,
Createdby: input.SingleFields.Createdby,
})
@@ -152,3 +168,30 @@ func (s *Service) GetFieldExtractionArrayFields(ctx context.Context, fieldExtrac
return arrayFields, nil
}
// GetFieldExtractionByVersion retrieves a specific version of field extraction for a document
// Returns nil if no field extraction exists for the document at that version
// documentID: the UUID of the document
// version: the version number to retrieve (must be >= 1)
func (s *Service) GetFieldExtractionByVersion(ctx context.Context, documentID uuid.UUID, version int64) (*repository.GetFieldExtractionByVersionRow, error) {
if documentID == uuid.Nil {
return nil, fmt.Errorf("documentID cannot be nil")
}
if version < 1 {
return nil, fmt.Errorf("version must be at least 1")
}
fieldExtraction, err := s.cfg.GetDBQueries().GetFieldExtractionByVersion(ctx, &repository.GetFieldExtractionByVersionParams{
Documentid: documentID,
Version: version,
})
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
}
return nil, fmt.Errorf("failed to get field extraction by version: %w", err)
}
return fieldExtraction, nil
}