Merged in feature/textExtractionsPart2 (pull request #193)

Continue finishing the parts of the text extraction plan

* add missing fields

internal/fieldextraction/models.go

@@ -1,6 +1,8 @@
 package fieldextraction
 
 import (
+	"fmt"
+
 	"queryorchestration/internal/database/repository"
 
 	"github.com/google/uuid"
@@ -13,8 +15,38 @@ type CreateFieldExtractionInput struct {
 	ArrayFields  []*repository.AddFieldExtractionArrayFieldParams // All arrays must be same length N
 }
 
-// ValidateArrayConsistency checks that all array fields have consistent length
-// Returns the array length if valid, error if inconsistent
+// MaxArrayFieldCount is the maximum number of array field rows allowed per field extraction.
+// This prevents excessively large payloads that could impact performance.
+const MaxArrayFieldCount = 10000
+
+// ValidateArrayConsistency validates the array fields for a field extraction.
+// It checks:
+// - No nil entries in the array
+// - Array length does not exceed MaxArrayFieldCount
+// - Empty arrays are allowed (returns 0, nil)
+//
+// Note: The service layer assigns arrayIndex values sequentially (0, 1, 2, ...)
+// when inserting, so we don't validate incoming arrayIndex values here.
+// The database enforces uniqueness and non-negative constraints.
+//
+// Returns the array length if valid, error if validation fails.
 func ValidateArrayConsistency(arrayFields []*repository.AddFieldExtractionArrayFieldParams) (int, error) {
+	// Empty array is valid
+	if len(arrayFields) == 0 {
+		return 0, nil
+	}
+
+	// Check for maximum array size to prevent performance issues
+	if len(arrayFields) > MaxArrayFieldCount {
+		return 0, fmt.Errorf("array field count %d exceeds maximum allowed %d", len(arrayFields), MaxArrayFieldCount)
+	}
+
+	// Check for nil entries which would cause issues during insertion
+	for i, field := range arrayFields {
+		if field == nil {
+			return 0, fmt.Errorf("array field at index %d is nil", i)
+		}
+	}
+
 	return len(arrayFields), nil
 }

internal/fieldextraction/models_test.go

@@ -0,0 +1,111 @@
+package fieldextraction
+
+import (
+	"testing"
+
+	"queryorchestration/internal/database/repository"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestValidateArrayConsistency(t *testing.T) {
+	t.Run("empty array is valid", func(t *testing.T) {
+		count, err := ValidateArrayConsistency([]*repository.AddFieldExtractionArrayFieldParams{})
+		require.NoError(t, err)
+		assert.Equal(t, 0, count)
+	})
+
+	t.Run("nil slice is valid", func(t *testing.T) {
+		count, err := ValidateArrayConsistency(nil)
+		require.NoError(t, err)
+		assert.Equal(t, 0, count)
+	})
+
+	t.Run("single element array is valid", func(t *testing.T) {
+		fields := []*repository.AddFieldExtractionArrayFieldParams{
+			{},
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.NoError(t, err)
+		assert.Equal(t, 1, count)
+	})
+
+	t.Run("multiple elements array is valid", func(t *testing.T) {
+		exhibitTitle1 := "Exhibit A"
+		exhibitTitle2 := "Exhibit B"
+		exhibitTitle3 := "Exhibit C"
+		fields := []*repository.AddFieldExtractionArrayFieldParams{
+			{Exhibittitle: &exhibitTitle1},
+			{Exhibittitle: &exhibitTitle2},
+			{Exhibittitle: &exhibitTitle3},
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.NoError(t, err)
+		assert.Equal(t, 3, count)
+	})
+
+	t.Run("nil entry in array returns error", func(t *testing.T) {
+		exhibitTitle := "Exhibit A"
+		fields := []*repository.AddFieldExtractionArrayFieldParams{
+			{Exhibittitle: &exhibitTitle},
+			nil, // nil entry
+			{Exhibittitle: &exhibitTitle},
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.Error(t, err)
+		assert.Equal(t, 0, count)
+		assert.Contains(t, err.Error(), "array field at index 1 is nil")
+	})
+
+	t.Run("nil entry at first position returns error", func(t *testing.T) {
+		exhibitTitle := "Exhibit A"
+		fields := []*repository.AddFieldExtractionArrayFieldParams{
+			nil, // nil entry at first position
+			{Exhibittitle: &exhibitTitle},
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.Error(t, err)
+		assert.Equal(t, 0, count)
+		assert.Contains(t, err.Error(), "array field at index 0 is nil")
+	})
+
+	t.Run("nil entry at last position returns error", func(t *testing.T) {
+		exhibitTitle := "Exhibit A"
+		fields := []*repository.AddFieldExtractionArrayFieldParams{
+			{Exhibittitle: &exhibitTitle},
+			{Exhibittitle: &exhibitTitle},
+			nil, // nil entry at last position
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.Error(t, err)
+		assert.Equal(t, 0, count)
+		assert.Contains(t, err.Error(), "array field at index 2 is nil")
+	})
+
+	t.Run("array at max count is valid", func(t *testing.T) {
+		fields := make([]*repository.AddFieldExtractionArrayFieldParams, MaxArrayFieldCount)
+		for i := range fields {
+			fields[i] = &repository.AddFieldExtractionArrayFieldParams{}
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.NoError(t, err)
+		assert.Equal(t, MaxArrayFieldCount, count)
+	})
+
+	t.Run("array exceeding max count returns error", func(t *testing.T) {
+		fields := make([]*repository.AddFieldExtractionArrayFieldParams, MaxArrayFieldCount+1)
+		for i := range fields {
+			fields[i] = &repository.AddFieldExtractionArrayFieldParams{}
+		}
+		count, err := ValidateArrayConsistency(fields)
+		require.Error(t, err)
+		assert.Equal(t, 0, count)
+		assert.Contains(t, err.Error(), "exceeds maximum allowed")
+	})
+}
+
+func TestMaxArrayFieldCount(t *testing.T) {
+	// Verify the constant is set to a reasonable value
+	assert.Equal(t, 10000, MaxArrayFieldCount, "MaxArrayFieldCount should be 10000")
+}

internal/fieldextraction/service.go

@@ -67,16 +67,32 @@ func (s *Service) CreateFieldExtraction(ctx context.Context, input *CreateFieldE
 		return nil, fmt.Errorf("failed to create field extraction: %w", err)
 	}
 
-	// Get the next version number for this document
-	history, err := queries.GetFieldExtractionsByDocumentID(ctx, input.DocumentID)
+	// Lock existing version rows for this document to prevent concurrent version creation
+	// This ensures that two concurrent requests cannot create the same version number
+	_, err = queries.LockFieldExtractionVersionsForDocument(ctx, input.DocumentID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get field extraction history: %w", err)
+		return nil, fmt.Errorf("failed to lock version rows: %w", err)
 	}
-	version := int64(len(history))
 
-	// Create version entry
+	// Get the max version number for this document (after locking)
+	// COALESCE in the query ensures we get 0 if no versions exist
+	maxVersionPtr, err := queries.GetMaxFieldExtractionVersion(ctx, input.DocumentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get max version: %w", err)
+	}
+	// SQLC returns *int64 even though COALESCE guarantees non-NULL
+	// Dereference safely - if somehow nil, treat as 0
+	var maxVersion int64
+	if maxVersionPtr != nil {
+		maxVersion = *maxVersionPtr
+	}
+	// Next version is always maxVersion + 1 (1 for first version since maxVersion is 0)
+	version := maxVersion + 1
+
+	// Create version entry with documentId for the unique constraint
 	err = queries.AddFieldExtractionEntry(ctx, &repository.AddFieldExtractionEntryParams{
 		Fieldextractionid: fieldExtraction.ID,
+		Documentid:        input.DocumentID,
 		Version:           version,
 		Createdby:         input.SingleFields.Createdby,
 	})
@@ -152,3 +168,30 @@ func (s *Service) GetFieldExtractionArrayFields(ctx context.Context, fieldExtrac
 
 	return arrayFields, nil
 }
+
+// GetFieldExtractionByVersion retrieves a specific version of field extraction for a document
+// Returns nil if no field extraction exists for the document at that version
+// documentID: the UUID of the document
+// version: the version number to retrieve (must be >= 1)
+func (s *Service) GetFieldExtractionByVersion(ctx context.Context, documentID uuid.UUID, version int64) (*repository.GetFieldExtractionByVersionRow, error) {
+	if documentID == uuid.Nil {
+		return nil, fmt.Errorf("documentID cannot be nil")
+	}
+
+	if version < 1 {
+		return nil, fmt.Errorf("version must be at least 1")
+	}
+
+	fieldExtraction, err := s.cfg.GetDBQueries().GetFieldExtractionByVersion(ctx, &repository.GetFieldExtractionByVersionParams{
+		Documentid: documentID,
+		Version:    version,
+	})
+	if err != nil {
+		if err == pgx.ErrNoRows {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("failed to get field extraction by version: %w", err)
+	}
+
+	return fieldExtraction, nil
+}