query-orchestration/test/text_extraction_integration_test.go

package endtoend_test

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"testing"
	"time"

	"queryorchestration/internal/serviceconfig"
	"queryorchestration/internal/serviceconfig/aws"
	"queryorchestration/internal/serviceconfig/objectstore"
	"queryorchestration/internal/test"

	queryapi "queryorchestration/pkg/queryAPI"

	openapi_types "github.com/oapi-codegen/runtime/types"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// TextExtractionConfig embeds the configuration needed for the text extraction integration tests.
type TextExtractionConfig struct {
	serviceconfig.BaseConfig
	aws.AWSConfig
	objectstore.ObjectStoreConfig
}

// TestE2EDocWorkflow is a comprehensive integration test that exercises the
// document upload, folder management, label, and field extraction APIs via HTTP REST calls.
//
// The test performs the following steps:
//
// Document Upload Phase:
// 1. Create a new client via POST /client
// 2. Verify client exists via GET /client/{id}
// 2b. List all clients via GET /clients and verify new client is present
// 3. Update client to allow sync via PATCH /client/{id}
// 4. Confirm client status via GET /client/{id}/status
// 5. Create a ZIP with 3 PDFs in folder structures
// 6. Upload batch via POST /client/{id}/document/batch
// 7. Verify batch via GET /client/{id}/document/batch
// 8. Get documents via GET /client/{id}/document and verify 3 documents
// 9-10. Verify folder-based document retrieval (skipped - requires folder IDs from processing)
// 11. POST a single document to root
// 12. Verify 4 documents total
//
// Label Operations Phase:
// 13. Apply all labels one by one to a document via POST /documents/{documentId}/labels
// 14. After each label, verify via GET /documents/{documentId}/labels
// 15. List all folders for client via GET /clients/{clientId}/folders
// 16. Get metrics for each folder via GET /folders/{folderId}/metrics
// 17. Get documents by label via GET /labels/{labelName}/documents
//
// Field Extractions Phase:
// 18. Verify no field extraction exists via GET /field-extractions
// 19. Create field extraction via POST /field-extractions
// 20. Verify single extraction via GET /field-extractions/history
// 21. Modify and create new version via POST /field-extractions
// 22. Verify two versions via GET /field-extractions/history
// 23. Retrieve specific version via GET /field-extractions/version
func TestE2EDocWorkflow(t *testing.T) {
	t.Run("docs", func(t *testing.T) {
		cfg := &TextExtractionConfig{}

		// Create full network with database, S3, and HTTP server
		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
		defer cleanup()

		// Phase 1: Client and Document Upload
		clientID, clientName := runClientSetupPhase(t, net)
		documents := runDocumentUploadPhase(t, net, clientID)

		// Phase 2: Label Operations
		targetDocID := documents[0].Id
		runLabelOperationsPhase(t, net, clientID, targetDocID)

		// Phase 3: Field Extractions
		runFieldExtractionsPhase(t, net, targetDocID)

		fmt.Println("\n=== Test Complete: All steps passed ===")
		_ = clientName // Used in output
	})
}

// runClientSetupPhase handles steps 1-4: client creation, verification, and configuration.
// Returns the client ID and client name.
func runClientSetupPhase(t *testing.T, net test.Network) (string, string) {
	t.Helper()

	fmt.Println("=== Step 1: Create a new client ===")
	clientName := fmt.Sprintf("TextExtractionTestClient_%d", time.Now().UnixNano())
	clientExternalID := fmt.Sprintf("text-ext-%d", time.Now().UnixNano())

	createClientRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
		Name: clientName,
		Id:   clientExternalID,
	})
	require.NoError(t, err, "Failed to create client")
	if createClientRes.JSON400 != nil {
		t.Fatalf("Got 400 error: %s", createClientRes.JSON400.Message)
	}
	if createClientRes.JSON201 == nil {
		t.Fatalf("Expected 201 response, got status %d with body: %s", createClientRes.StatusCode(), string(createClientRes.Body))
	}
	clientID := createClientRes.JSON201.Id
	fmt.Printf("  Created client: ID=%s, Name=%s, ExternalID=%s\n", clientID, clientName, clientExternalID)

	fmt.Println("\n=== Step 2: Verify client exists via GET /client/{id} ===")
	getClientRes, err := net.Client.GetClientWithResponse(t.Context(), clientID)
	require.NoError(t, err, "Failed to get client")
	require.NotNil(t, getClientRes.JSON200, "Expected 200 response, got status %d", getClientRes.StatusCode())
	fmt.Printf("  Client verified: ID=%s, Name=%s\n", getClientRes.JSON200.Id, getClientRes.JSON200.Name)
	assert.Equal(t, clientName, getClientRes.JSON200.Name, "Client name mismatch")

	fmt.Println("\n=== Step 2b: List all clients via GET /clients ===")
	listClientsRes, err := net.Client.ListClientsWithResponse(t.Context())
	require.NoError(t, err, "Failed to list clients")
	require.NotNil(t, listClientsRes.JSON200, "Expected 200 response, got status %d", listClientsRes.StatusCode())
	fmt.Printf("  Found %d clients in system\n", listClientsRes.JSON200.TotalCount)
	// Verify our newly created client is in the list
	foundClient := false
	for _, c := range listClientsRes.JSON200.Clients {
		if c.Id == clientID {
			foundClient = true
			assert.Equal(t, clientName, c.Name, "Client name in list doesn't match")
			fmt.Printf("  Verified client %s is in the list\n", clientID)
			break
		}
	}
	require.True(t, foundClient, "Newly created client %s not found in list of clients", clientID)

	fmt.Println("\n=== Step 3: Update client to allow sync ===")
	canSync := true
	updateClientRes, err := net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
		CanSync: &canSync,
	})
	require.NoError(t, err, "Failed to update client")
	require.Equal(t, http.StatusOK, updateClientRes.StatusCode(), "Expected 200 response for update")
	fmt.Printf("  Client updated: canSync=%t\n", canSync)

	fmt.Println("\n=== Step 4: Confirm client status ===")
	statusRes, err := net.Client.GetStatusByClientIdWithResponse(t.Context(), clientID)
	require.NoError(t, err, "Failed to get client status")
	require.NotNil(t, statusRes.JSON200, "Expected 200 response, got status %d", statusRes.StatusCode())
	fmt.Printf("  Client status: %s\n", statusRes.JSON200.Status)

	return clientID, clientName
}

// runDocumentUploadPhase handles steps 5-12: ZIP creation, batch upload, and document verification.
// Returns the list of documents.
func runDocumentUploadPhase(t *testing.T, net test.Network, clientID string) []queryapi.DocumentSummary {
	t.Helper()

	fmt.Println("\n=== Step 5: Create ZIP with 3 PDFs ===")
	zipBuffer := createTestZIP(t)
	fmt.Printf("  ZIP created: %d bytes with 3 PDF files\n", zipBuffer.Len())
	fmt.Println("  Files in ZIP:")
	fmt.Println("    - folder1/subfolder2/file1.pdf")
	fmt.Println("    - folder1/subfolder2/file2.pdf")
	fmt.Println("    - folder2/subfolder1/file1.pdf")

	fmt.Println("\n=== Step 6: Upload batch via POST /client/{id}/document/batch ===")
	batchID := uploadBatchZIP(t, net.Client, clientID, zipBuffer.Bytes())
	fmt.Printf("  Batch uploaded: batchID=%s\n", batchID)

	fmt.Println("\n=== Step 7: Verify batch status ===")
	verifyBatchStatus(t, net, clientID, batchID)

	fmt.Println("\n=== Step 8: Get documents and verify count ===")
	var documents []queryapi.DocumentSummary
	waitForDocumentCount(t, net.Client, clientID, 3, &documents)
	fmt.Printf("  Documents found: %d\n", len(documents))
	for i, doc := range documents {
		fmt.Printf("    Document %d: ID=%s, Hash=%s\n", i+1, doc.Id, doc.Hash)
	}
	assert.Len(t, documents, 3, "Expected 3 documents from batch upload")

	fmt.Println("\n=== Step 9: Get documents for folder1 (should see 2) ===")
	fmt.Println("  Note: Folder-based document retrieval requires folder IDs from folder creation during processing")
	fmt.Println("  Skipping folder-specific document retrieval - documents verified at client level")

	fmt.Println("\n=== Step 10: Get documents for folder2 (should see 1) ===")
	fmt.Println("  Note: Folder-based document retrieval requires folder IDs from folder creation during processing")
	fmt.Println("  Skipping folder-specific document retrieval - documents verified at client level")

	fmt.Println("\n=== Step 11: Upload single document to root ===")
	uploadSingleDocument(t, net.Client, clientID, "single_root_document.pdf")
	fmt.Println("  Single document uploaded: single_root_document.pdf")

	fmt.Println("\n=== Step 12: Verify 4 documents total ===")
	waitForDocumentCount(t, net.Client, clientID, 4, &documents)
	fmt.Printf("  Final document count: %d\n", len(documents))
	for i, doc := range documents {
		fmt.Printf("    Document %d: ID=%s, Hash=%s\n", i+1, doc.Id, doc.Hash)
	}
	assert.Len(t, documents, 4, "Expected 4 documents total")

	return documents
}

// verifyBatchStatus polls for batch status and verifies completion.
func verifyBatchStatus(t *testing.T, net test.Network, clientID string, batchID queryapi.BatchID) {
	t.Helper()

	var batchStatus string
	for i := 0; i < 30; i++ {
		batchRes, err := net.Client.GetDocumentBatchWithResponse(t.Context(), clientID, batchID)
		require.NoError(t, err, "Failed to get batch status")
		require.NotNil(t, batchRes.JSON200, "Expected 200 response for batch, got status %d", batchRes.StatusCode())
		batchStatus = string(batchRes.JSON200.Status)
		fmt.Printf("  Batch status check %d: status=%s, processed=%d, total=%d\n",
			i+1, batchStatus, batchRes.JSON200.ProcessedDocuments, batchRes.JSON200.TotalDocuments)

		if batchStatus == "completed" || batchStatus == "failed" {
			break
		}
		time.Sleep(500 * time.Millisecond)
	}
	fmt.Printf("  Final batch status: %s\n", batchStatus)

	listBatchesRes, err := net.Client.ListDocumentBatchesWithResponse(t.Context(), clientID, nil)
	require.NoError(t, err, "Failed to list batches")
	require.NotNil(t, listBatchesRes.JSON200, "Expected 200 response for batch list")
	fmt.Printf("  Total batches for client: %d\n", listBatchesRes.JSON200.TotalCount)
}

// runLabelOperationsPhase handles steps 13-17: applying labels, verifying, and querying by label.
func runLabelOperationsPhase(t *testing.T, net test.Network, clientID string, targetDocID openapi_types.UUID) {
	t.Helper()

	fmt.Println("\n=== Step 13: Apply all labels to a document ===")
	fmt.Printf("  Target document for labels: ID=%s\n", targetDocID)

	labels := []string{"Ingested", "OCR_Processed", "GenAI_Processed", "Doczy_AI_Completed", "Dashboard_Ready"}
	applyAndVerifyLabels(t, net, targetDocID, labels)

	fmt.Println("\n=== Step 15: List folders and get folder metrics ===")
	verifyFolderMetrics(t, net, clientID)

	fmt.Println("\n=== Step 17: Get documents by label ===")
	verifyDocumentsByLabel(t, net, clientID, "Ingested")
}

// verifyFolderMetrics lists all folders for a client and verifies folder metrics.
func verifyFolderMetrics(t *testing.T, net test.Network, clientID string) {
	t.Helper()

	// List all folders for the client using the new endpoint
	foldersRes, err := net.Client.ListClientFoldersWithResponse(t.Context(), queryapi.ClientID(clientID), nil)
	require.NoError(t, err, "Failed to list client folders")
	if foldersRes.JSON200 == nil {
		t.Logf("Response status: %d, body: %s", foldersRes.StatusCode(), string(foldersRes.Body))
		if foldersRes.JSON404 != nil {
			t.Logf("404 error: %s", foldersRes.JSON404.Message)
		}
		if foldersRes.JSON500 != nil {
			t.Logf("500 error: %s", foldersRes.JSON500.Message)
			// Print container logs to diagnose the issue
			fmt.Println("\n=== QueryAPI Container Logs (for 500 error diagnosis) ===")
			if apiContainer, ok := net.APIs[test.QueryAPIName]; ok && apiContainer != nil {
				test.PrintContainerLogs(t, apiContainer.Container)
			}
			fmt.Println("=== End Container Logs ===")
			t.Fatalf("ListClientFolders returned 500 error: %s", foldersRes.JSON500.Message)
		}
	}
	require.NotNil(t, foldersRes.JSON200, "Expected 200 response for folder list")

	folders := foldersRes.JSON200.Folders
	fmt.Printf("  Folders found for client: %d\n", len(folders))

	if len(folders) == 0 {
		fmt.Println("  Note: No folders found - documents may be at root level")
		return
	}

	// Print folder hierarchy
	fmt.Println("  Folder hierarchy:")
	for _, folder := range folders {
		parentInfo := "root"
		if folder.ParentId.IsSpecified() {
			val, _ := folder.ParentId.Get()
			parentInfo = fmt.Sprintf("parent=%s", val)
		}
		fmt.Printf("    - %s (ID=%s, %s)\n", folder.Path, folder.Id, parentInfo)
	}

	// Test metrics for each folder
	fmt.Println("\n=== Step 16: Get metrics for each folder ===")
	for _, folder := range folders {
		metricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder.Id)
		require.NoError(t, err, "Failed to get metrics for folder %s", folder.Path)
		require.NotNil(t, metricsRes.JSON200, "Expected 200 response for folder metrics")

		fmt.Printf("  Folder '%s': totalDocuments=%d, labels=%v\n",
			folder.Path, metricsRes.JSON200.TotalDocuments, metricsRes.JSON200.ByLabel)
	}
}

// applyAndVerifyLabels applies each label and verifies after each application.
func applyAndVerifyLabels(t *testing.T, net test.Network, docID openapi_types.UUID, labels []string) {
	t.Helper()

	for i, label := range labels {
		fmt.Printf("\n=== Step 13.%d: Applying label '%s' ===\n", i+1, label)
		applyLabelRes, err := net.Client.ApplyLabelWithResponse(t.Context(), docID, queryapi.ApplyLabelJSONRequestBody{
			Label:     label,
			AppliedBy: openapi_types.Email("test@example.com"),
		})
		require.NoError(t, err, "Failed to apply label %s", label)
		if applyLabelRes.JSON400 != nil {
			t.Logf("WARNING: Got 400 error applying label '%s': %s", label, applyLabelRes.JSON400.Message)
			continue
		}
		if applyLabelRes.JSON201 == nil {
			t.Logf("WARNING: Expected 201 response for label '%s', got status %d with body: %s", label, applyLabelRes.StatusCode(), string(applyLabelRes.Body))
			continue
		}
		fmt.Printf("  Label '%s' applied: recordID=%s\n", label, applyLabelRes.JSON201.Id)

		fmt.Println("  Verifying labels via GET /documents/{documentId}/labels")
		getLabelsRes, err := net.Client.GetDocumentLabelsWithResponse(t.Context(), docID)
		require.NoError(t, err, "Failed to get document labels")
		if getLabelsRes.JSON200 == nil {
			t.Logf("WARNING: Expected 200 for get labels, got %d", getLabelsRes.StatusCode())
			continue
		}
		fmt.Printf("  Labels on document: count=%d\n", len(*getLabelsRes.JSON200.Labels))
		assert.Len(t, *getLabelsRes.JSON200.Labels, i+1, "Expected %d labels after applying '%s'", i+1, label)
	}
}

// verifyDocumentsByLabel queries documents by label and verifies the result.
func verifyDocumentsByLabel(t *testing.T, net test.Network, clientID string, label string) {
	t.Helper()

	fmt.Printf("  Getting documents with label '%s' for client '%s'\n", label, clientID)
	getDocsByLabelRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), label, &queryapi.GetDocumentsByLabelParams{
		ClientId: queryapi.ClientID(clientID),
	})
	require.NoError(t, err, "Failed to get documents by label")
	if getDocsByLabelRes.JSON200 == nil {
		t.Logf("WARNING: Expected 200 for get documents by label, got %d with body: %s", getDocsByLabelRes.StatusCode(), string(getDocsByLabelRes.Body))
	} else {
		docCount := 0
		if getDocsByLabelRes.JSON200.Documents != nil {
			docCount = len(*getDocsByLabelRes.JSON200.Documents)
		}
		fmt.Printf("  Documents with label '%s': count=%d\n", label, docCount)
		assert.GreaterOrEqual(t, docCount, 1, "Expected at least 1 document with label '%s'", label)
	}
}

// runFieldExtractionsPhase handles steps 18-23: field extraction creation, versioning, and specific version retrieval.
func runFieldExtractionsPhase(t *testing.T, net test.Network, targetDocID openapi_types.UUID) {
	t.Helper()

	fmt.Println("\n=== Step 18: Verify no field extraction exists ===")
	verifyNoFieldExtraction(t, net, targetDocID)

	fmt.Println("\n=== Step 19: Create field extraction ===")
	createFieldExtraction(t, net, targetDocID, "Test Contract Title", "Test Client Name")

	fmt.Println("\n=== Step 20: Verify single extraction via history ===")
	verifyFieldExtractionHistory(t, net, targetDocID, 1)

	fmt.Println("\n=== Step 21: Create second version of field extraction ===")
	createFieldExtraction(t, net, targetDocID, "Updated Contract Title v2", "Updated Client Name v2")

	fmt.Println("\n=== Step 22: Verify two versions in history ===")
	verifyFieldExtractionHistory(t, net, targetDocID, 2)

	fmt.Println("\n=== Step 23: Retrieve specific versions via GET /field-extractions/version ===")
	verifySpecificVersionRetrieval(t, net, targetDocID)
}

// verifyNoFieldExtraction checks that no field extraction exists for the document.
func verifyNoFieldExtraction(t *testing.T, net test.Network, docID openapi_types.UUID) {
	t.Helper()

	getFieldExtRes, err := net.Client.GetCurrentFieldExtractionWithResponse(t.Context(), &queryapi.GetCurrentFieldExtractionParams{
		DocumentId: docID,
	})
	require.NoError(t, err, "Failed to get current field extraction")
	if getFieldExtRes.JSON404 != nil {
		fmt.Println("  Confirmed: No field extraction exists for document (404)")
	} else if getFieldExtRes.JSON200 != nil {
		fmt.Printf("  WARNING: Field extraction already exists: version=%d\n", getFieldExtRes.JSON200.Version)
	} else {
		fmt.Printf("  Got status %d: %s\n", getFieldExtRes.StatusCode(), string(getFieldExtRes.Body))
	}
}

// createFieldExtraction creates a field extraction with the given values.
func createFieldExtraction(t *testing.T, net test.Network, docID openapi_types.UUID, contractTitle, clientName string) {
	t.Helper()

	createFieldExtRes, err := net.Client.CreateFieldExtractionWithResponse(t.Context(), queryapi.CreateFieldExtractionJSONRequestBody{
		DocumentId: docID,
		SingleFields: queryapi.SingleFields{
			FileName:      ptr("test_document.pdf"),
			ContractTitle: &contractTitle,
			ClientName:    &clientName,
		},
		ArrayFields: []queryapi.ArrayFieldItem{},
		CreatedBy:   openapi_types.Email("test@example.com"),
	})
	require.NoError(t, err, "Failed to create field extraction")
	if createFieldExtRes.JSON400 != nil {
		t.Logf("WARNING: Got 400 error creating field extraction: %s", createFieldExtRes.JSON400.Message)
	} else if createFieldExtRes.JSON201 == nil {
		t.Logf("WARNING: Expected 201 for field extraction, got %d with body: %s", createFieldExtRes.StatusCode(), string(createFieldExtRes.Body))
	} else {
		fmt.Printf("  Field extraction created: version=%d\n", createFieldExtRes.JSON201.Version)
	}
}

// verifyFieldExtractionHistory checks that the expected number of versions exist.
func verifyFieldExtractionHistory(t *testing.T, net test.Network, docID openapi_types.UUID, expectedCount int) {
	t.Helper()

	historyRes, err := net.Client.GetFieldExtractionHistoryWithResponse(t.Context(), &queryapi.GetFieldExtractionHistoryParams{
		DocumentId: docID,
	})
	require.NoError(t, err, "Failed to get field extraction history")
	if historyRes.JSON200 == nil {
		t.Logf("WARNING: Expected 200 for history, got %d with body: %s", historyRes.StatusCode(), string(historyRes.Body))
	} else {
		historyCount := 0
		if historyRes.JSON200.Versions != nil {
			historyCount = len(*historyRes.JSON200.Versions)
		}
		fmt.Printf("  Field extraction history: count=%d\n", historyCount)
		assert.Equal(t, expectedCount, historyCount, "Expected %d field extraction(s) in history", expectedCount)
	}
}

// verifySpecificVersionRetrieval tests the GET /field-extractions/version endpoint
// to retrieve specific versions of field extractions by document ID and version number.
func verifySpecificVersionRetrieval(t *testing.T, net test.Network, docID openapi_types.UUID) {
	t.Helper()

	// Test retrieving version 1
	fmt.Println("  Testing retrieval of version 1...")
	version1Res, err := net.Client.GetFieldExtractionByVersionWithResponse(t.Context(), &queryapi.GetFieldExtractionByVersionParams{
		DocumentId: docID,
		Version:    1,
	})
	require.NoError(t, err, "Failed to get field extraction version 1")
	require.NotNil(t, version1Res.JSON200, "Expected 200 for version 1, got %d with body: %s", version1Res.StatusCode(), string(version1Res.Body))
	assert.Equal(t, int32(1), version1Res.JSON200.Version, "Expected version number 1")
	assert.NotNil(t, version1Res.JSON200.SingleFields.ContractTitle, "Expected contractTitle in version 1")
	if version1Res.JSON200.SingleFields.ContractTitle != nil {
		assert.Equal(t, "Test Contract Title", *version1Res.JSON200.SingleFields.ContractTitle, "Expected original contract title in version 1")
	}
	fmt.Printf("  Version 1 retrieved: contractTitle=%s\n", *version1Res.JSON200.SingleFields.ContractTitle)

	// Test retrieving version 2
	fmt.Println("  Testing retrieval of version 2...")
	version2Res, err := net.Client.GetFieldExtractionByVersionWithResponse(t.Context(), &queryapi.GetFieldExtractionByVersionParams{
		DocumentId: docID,
		Version:    2,
	})
	require.NoError(t, err, "Failed to get field extraction version 2")
	require.NotNil(t, version2Res.JSON200, "Expected 200 for version 2, got %d with body: %s", version2Res.StatusCode(), string(version2Res.Body))
	assert.Equal(t, int32(2), version2Res.JSON200.Version, "Expected version number 2")
	assert.NotNil(t, version2Res.JSON200.SingleFields.ContractTitle, "Expected contractTitle in version 2")
	if version2Res.JSON200.SingleFields.ContractTitle != nil {
		assert.Equal(t, "Updated Contract Title v2", *version2Res.JSON200.SingleFields.ContractTitle, "Expected updated contract title in version 2")
	}
	fmt.Printf("  Version 2 retrieved: contractTitle=%s\n", *version2Res.JSON200.SingleFields.ContractTitle)

	// Test retrieving non-existent version (should return 404)
	fmt.Println("  Testing retrieval of non-existent version 999...")
	version999Res, err := net.Client.GetFieldExtractionByVersionWithResponse(t.Context(), &queryapi.GetFieldExtractionByVersionParams{
		DocumentId: docID,
		Version:    999,
	})
	require.NoError(t, err, "Failed to query for non-existent version")
	assert.NotNil(t, version999Res.JSON404, "Expected 404 for non-existent version 999, got status %d", version999Res.StatusCode())
	fmt.Println("  Non-existent version correctly returned 404")
}

// createTestZIP creates a ZIP file in memory containing 3 simple PDF files
// in the specified folder structure.
func createTestZIP(t *testing.T) *bytes.Buffer {
	t.Helper()

	buf := new(bytes.Buffer)
	zipWriter := zip.NewWriter(buf)

	// PDF files to create in the ZIP
	files := []struct {
		path    string
		content string
	}{
		{"folder1/subfolder2/file1.pdf", pdfContent("Document 1 in folder1/subfolder2")},
		{"folder1/subfolder2/file2.pdf", pdfContent("Document 2 in folder1/subfolder2")},
		{"folder2/subfolder1/file1.pdf", pdfContent("Document 1 in folder2/subfolder1")},
	}

	for _, file := range files {
		writer, err := zipWriter.Create(file.path)
		require.NoError(t, err, "Failed to create file in ZIP: %s", file.path)

		_, err = writer.Write([]byte(file.content))
		require.NoError(t, err, "Failed to write content to ZIP file: %s", file.path)
	}

	err := zipWriter.Close()
	require.NoError(t, err, "Failed to close ZIP writer")

	return buf
}

// pdfContent generates a minimal valid PDF with the given text content.
func pdfContent(text string) string {
	// Minimal valid PDF structure
	return fmt.Sprintf(`%%PDF-1.4
%%âãÏÓ
1 0 obj
<<
  /Type /Catalog
  /Pages 2 0 R
>>
endobj
2 0 obj
<<
  /Type /Pages
  /Kids [3 0 R]
  /Count 1
>>
endobj
3 0 obj
<<
  /Type /Page
  /Parent 2 0 R
  /MediaBox [0 0 612 792]
  /Resources <<
    /Font <<
      /F1 <<
        /Type /Font
        /Subtype /Type1
        /BaseFont /Helvetica
      >>
    >>
  >>
  /Contents 4 0 R
>>
endobj
4 0 obj
<<
  /Length 50
>>
stream
BT
/F1 12 Tf
100 700 Td
(%s) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000015 00000 n
0000000066 00000 n
0000000125 00000 n
0000000330 00000 n
trailer
<<
  /Size 5
  /Root 1 0 R
>>
startxref
430
%%%%EOF`, text)
}

// uploadBatchZIP uploads a ZIP file to the batch upload endpoint and returns the batch ID.
func uploadBatchZIP(t *testing.T, client *queryapi.ClientWithResponses, clientID string, zipContent []byte) queryapi.BatchID {
	t.Helper()

	// Create multipart form
	var buf bytes.Buffer
	writer := multipart.NewWriter(&buf)

	part, err := writer.CreateFormFile("archive", "test_batch.zip")
	require.NoError(t, err, "Failed to create form file")

	_, err = part.Write(zipContent)
	require.NoError(t, err, "Failed to write ZIP content")

	err = writer.Close()
	require.NoError(t, err, "Failed to close multipart writer")

	// Upload using the generated client
	res, err := client.UploadDocumentBatchWithBodyWithResponse(
		t.Context(),
		clientID,
		writer.FormDataContentType(),
		bytes.NewReader(buf.Bytes()),
	)
	require.NoError(t, err, "Failed to upload batch")
	require.NotNil(t, res.JSON202, "Expected 202 response, got status %d", res.StatusCode())

	return res.JSON202.BatchId
}

// uploadSingleDocument uploads a single PDF document to the client's root folder.
func uploadSingleDocument(t *testing.T, client *queryapi.ClientWithResponses, clientID string, filename string) {
	t.Helper()

	// Create multipart form
	var buf bytes.Buffer
	writer := multipart.NewWriter(&buf)

	part, err := writer.CreateFormFile("file", filename)
	require.NoError(t, err, "Failed to create form file")

	pdfBytes := []byte(pdfContent("Single root document"))
	_, err = part.Write(pdfBytes)
	require.NoError(t, err, "Failed to write PDF content")

	err = writer.Close()
	require.NoError(t, err, "Failed to close multipart writer")

	// Upload using the generated client
	res, err := client.UploadDocumentWithBodyWithResponse(
		t.Context(),
		clientID,
		writer.FormDataContentType(),
		bytes.NewReader(buf.Bytes()),
	)
	require.NoError(t, err, "Failed to upload single document")
	require.Equal(t, http.StatusOK, res.StatusCode(), "Expected 200 response for single document upload")
}

// waitForDocumentCount polls the documents endpoint until the expected count is reached or timeout.
func waitForDocumentCount(t *testing.T, client *queryapi.ClientWithResponses, clientID string, expectedCount int, documents *[]queryapi.DocumentSummary) {
	t.Helper()

	timeout := time.After(60 * time.Second)
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-timeout:
			t.Fatalf("Timeout waiting for %d documents, got %d", expectedCount, len(*documents))
		case <-ticker.C:
			res, err := client.ListDocumentsByClientIdWithResponse(t.Context(), clientID)
			if err != nil {
				continue
			}
			if res.JSON200 == nil {
				continue
			}
			*documents = *res.JSON200
			if len(*documents) >= expectedCount {
				return
			}
		}
	}
}

// TestFolderAPI is a more detailed test that also verifies folder creation
// and folder-based document retrieval.
func TestFolderAPI(t *testing.T) {
	t.Run("create_folders", func(t *testing.T) {
		cfg := &TextExtractionConfig{}

		// Create full network
		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
		defer cleanup()

		// Create test client
		clientName := fmt.Sprintf("FolderTestClient_%d", time.Now().UnixNano())
		createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
			Name: clientName,
			Id:   fmt.Sprintf("folder-test-%d", time.Now().UnixNano()),
		})
		require.NoError(t, err)
		if createRes.JSON400 != nil {
			t.Fatalf("Got 400 error creating client: %s", createRes.JSON400.Message)
		}
		if createRes.JSON201 == nil {
			t.Fatalf("Expected 201 response for client creation, got status %d with body: %s", createRes.StatusCode(), string(createRes.Body))
		}
		clientID := createRes.JSON201.Id
		fmt.Printf("Created client: %s\n", clientID)

		// Enable sync
		canSync := true
		_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
			CanSync: &canSync,
		})
		require.NoError(t, err)

		// Create folders manually to test folder API
		fmt.Println("\n=== Creating folders via API ===")

		// Create root folder1
		folder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/folder1",
			ClientId:  clientID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create folder1")
		if folder1Res.JSON400 != nil {
			t.Fatalf("Got 400 error creating folder1: %s", folder1Res.JSON400.Message)
		}
		if folder1Res.JSON201 == nil {
			t.Fatalf("Expected 201 for folder1, got %d with body: %s", folder1Res.StatusCode(), string(folder1Res.Body))
		}
		folder1ID := folder1Res.JSON201.Id
		fmt.Printf("  Created folder1: ID=%s, Path=%s\n", folder1ID, folder1Res.JSON201.Path)

		// Create subfolder2 under folder1
		subfolder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/folder1/subfolder2",
			ClientId:  clientID,
			ParentId:  &folder1ID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create subfolder2")
		require.NotNil(t, subfolder2Res.JSON201, "Expected 201 for subfolder2, got %d", subfolder2Res.StatusCode())
		subfolder2ID := subfolder2Res.JSON201.Id
		fmt.Printf("  Created subfolder2: ID=%s, Path=%s\n", subfolder2ID, subfolder2Res.JSON201.Path)

		// Create root folder2
		folder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/folder2",
			ClientId:  clientID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create folder2")
		require.NotNil(t, folder2Res.JSON201, "Expected 201 for folder2, got %d", folder2Res.StatusCode())
		folder2ID := folder2Res.JSON201.Id
		fmt.Printf("  Created folder2: ID=%s, Path=%s\n", folder2ID, folder2Res.JSON201.Path)

		// Create subfolder1 under folder2
		subfolder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/folder2/subfolder1",
			ClientId:  clientID,
			ParentId:  &folder2ID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create subfolder1 under folder2")
		require.NotNil(t, subfolder1Res.JSON201, "Expected 201 for subfolder1, got %d", subfolder1Res.StatusCode())
		fmt.Printf("  Created subfolder1: ID=%s, Path=%s\n", subfolder1Res.JSON201.Id, subfolder1Res.JSON201.Path)

		// Test folder metrics (should be empty initially)
		fmt.Println("\n=== Testing folder metrics (empty folders) ===")
		metricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder1ID)
		require.NoError(t, err, "Failed to get folder1 metrics")
		require.NotNil(t, metricsRes.JSON200, "Expected 200 for metrics")
		fmt.Printf("  folder1 metrics: totalDocuments=%d\n", metricsRes.JSON200.TotalDocuments)

		// Test folder documents endpoint (should be empty initially)
		fmt.Println("\n=== Testing folder documents (empty folders) ===")
		folderDocsRes, err := net.Client.GetFolderDocumentsWithResponse(t.Context(), subfolder2ID)
		require.NoError(t, err, "Failed to get subfolder2 documents")
		require.NotNil(t, folderDocsRes.JSON200, "Expected 200 for folder documents")
		docCount := 0
		if folderDocsRes.JSON200.Documents != nil {
			docCount = len(*folderDocsRes.JSON200.Documents)
		}
		fmt.Printf("  subfolder2 documents: count=%d\n", docCount)

		// Test folder rename
		fmt.Println("\n=== Testing folder rename ===")
		renameRes, err := net.Client.RenameFolderWithResponse(t.Context(), folder1ID, queryapi.RenameFolderJSONRequestBody{
			Path: "/folder1_renamed",
		})
		require.NoError(t, err, "Failed to rename folder1")
		// Note: Implementation returns 204 No Content for successful rename
		require.Equal(t, http.StatusNoContent, renameRes.StatusCode(), "Expected 204 for rename")
		fmt.Printf("  Renamed folder1 to /folder1_renamed\n")

		fmt.Println("\n=== Folder test complete ===")
	})

	// TestFolderMetricsSpecific tests that GetFolderMetrics returns metrics for only the
	// specified folder, not aggregated metrics across all folders.
	t.Run("folder_metrics_specific", func(t *testing.T) {
		cfg := &TextExtractionConfig{}

		// Create full network
		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
		defer cleanup()

		// Create test client
		// Note: Client external ID has maxLength of 36, so use Unix seconds instead of nanoseconds
		clientName := fmt.Sprintf("FolderMetricsTestClient_%d", time.Now().UnixNano())
		clientExternalID := fmt.Sprintf("fld-metrics-%d", time.Now().Unix())
		createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
			Name: clientName,
			Id:   clientExternalID,
		})
		require.NoError(t, err)
		if createRes.JSON400 != nil {
			t.Fatalf("Got 400 error creating client: %s", createRes.JSON400.Message)
		}
		if createRes.JSON201 == nil {
			t.Fatalf("Expected 201 response for client creation, got status %d with body: %s", createRes.StatusCode(), string(createRes.Body))
		}
		clientID := createRes.JSON201.Id
		fmt.Printf("Created client: %s\n", clientID)

		// Enable sync
		canSync := true
		_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
			CanSync: &canSync,
		})
		require.NoError(t, err)

		// Create two separate folders at root level
		fmt.Println("\n=== Creating folders for metrics test ===")

		folder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/metrics_folder1",
			ClientId:  clientID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create metrics_folder1")
		require.NotNil(t, folder1Res.JSON201, "Expected 201 for metrics_folder1, got %d", folder1Res.StatusCode())
		folder1ID := folder1Res.JSON201.Id
		fmt.Printf("  Created metrics_folder1: ID=%s\n", folder1ID)

		folder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
			Path:      "/metrics_folder2",
			ClientId:  clientID,
			CreatedBy: "test@example.com",
		})
		require.NoError(t, err, "Failed to create metrics_folder2")
		require.NotNil(t, folder2Res.JSON201, "Expected 201 for metrics_folder2, got %d", folder2Res.StatusCode())
		folder2ID := folder2Res.JSON201.Id
		fmt.Printf("  Created metrics_folder2: ID=%s\n", folder2ID)

		// Get metrics for folder1 - should return metrics specific to folder1 only
		fmt.Println("\n=== Testing specific folder metrics for folder1 ===")
		metrics1Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder1ID)
		require.NoError(t, err, "Failed to get metrics for folder1")
		require.NotNil(t, metrics1Res.JSON200, "Expected 200 for folder1 metrics, got %d", metrics1Res.StatusCode())

		// Verify the returned folderId matches the requested folder
		assert.Equal(t, folder1ID, metrics1Res.JSON200.FolderId,
			"Metrics folderId should match the requested folder1 ID")
		fmt.Printf("  folder1 metrics: folderId=%s, totalDocuments=%d\n",
			metrics1Res.JSON200.FolderId, metrics1Res.JSON200.TotalDocuments)

		// Get metrics for folder2 - should return metrics specific to folder2 only
		fmt.Println("\n=== Testing specific folder metrics for folder2 ===")
		metrics2Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder2ID)
		require.NoError(t, err, "Failed to get metrics for folder2")
		require.NotNil(t, metrics2Res.JSON200, "Expected 200 for folder2 metrics, got %d", metrics2Res.StatusCode())

		// Verify the returned folderId matches the requested folder
		assert.Equal(t, folder2ID, metrics2Res.JSON200.FolderId,
			"Metrics folderId should match the requested folder2 ID")
		fmt.Printf("  folder2 metrics: folderId=%s, totalDocuments=%d\n",
			metrics2Res.JSON200.FolderId, metrics2Res.JSON200.TotalDocuments)

		// Verify that folder1 and folder2 metrics are independent (different folder IDs returned)
		assert.NotEqual(t, metrics1Res.JSON200.FolderId, metrics2Res.JSON200.FolderId,
			"Each folder should have its own distinct metrics with its own folderId")

		// Get the root folder and verify its metrics separately
		fmt.Println("\n=== Testing root folder metrics ===")
		foldersRes, err := net.Client.ListClientFoldersWithResponse(t.Context(), queryapi.ClientID(clientID), nil)
		require.NoError(t, err, "Failed to list client folders")
		require.NotNil(t, foldersRes.JSON200, "Expected 200 for folder list")

		var rootFolderID openapi_types.UUID
		for _, folder := range foldersRes.JSON200.Folders {
			if folder.Path == "/" {
				rootFolderID = folder.Id
				break
			}
		}

		if rootFolderID != (openapi_types.UUID{}) {
			rootMetricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), rootFolderID)
			require.NoError(t, err, "Failed to get root folder metrics")
			require.NotNil(t, rootMetricsRes.JSON200, "Expected 200 for root folder metrics")

			assert.Equal(t, rootFolderID, rootMetricsRes.JSON200.FolderId,
				"Root folder metrics folderId should match root folder ID")
			fmt.Printf("  root folder metrics: folderId=%s, totalDocuments=%d\n",
				rootMetricsRes.JSON200.FolderId, rootMetricsRes.JSON200.TotalDocuments)

			// Verify root folder metrics are distinct from folder1 and folder2
			assert.NotEqual(t, rootFolderID, folder1ID, "Root folder should be different from folder1")
			assert.NotEqual(t, rootFolderID, folder2ID, "Root folder should be different from folder2")
		} else {
			fmt.Println("  Note: Root folder not found in folder list")
		}

		fmt.Println("\n=== Folder metrics specific test complete ===")
	})
}

// ptr returns a pointer to the given string value.
// Helper for creating pointers to string literals.
func ptr(s string) *string {
	return &s
}

// ptrFloat64 returns a pointer to a float64 value.
func ptrFloat64(f float64) *float64 {
	return &f
}

// ptrBool returns a pointer to a bool value.
func ptrBool(b bool) *bool {
	return &b
}

// TestDeepFolderHierarchy tests folder creation with 10+ levels of nesting
// (Phase 10 - Scenario 4: Folder hierarchy with recursive queries).
func TestDeepFolderHierarchy(t *testing.T) {
	cfg := &TextExtractionConfig{}

	// Create full network
	net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
	defer cleanup()

	// Create test client
	clientName := fmt.Sprintf("DeepFolderTestClient_%d", time.Now().UnixNano())
	clientExternalID := fmt.Sprintf("deep-fld-%d", time.Now().Unix())
	createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
		Name: clientName,
		Id:   clientExternalID,
	})
	require.NoError(t, err)
	require.NotNil(t, createRes.JSON201, "Expected 201 response for client creation, got status %d", createRes.StatusCode())
	clientID := createRes.JSON201.Id
	fmt.Printf("Created client: %s\n", clientID)

	// Enable sync
	canSync := true
	_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
		CanSync: &canSync,
	})
	require.NoError(t, err)

	// Create 12 levels of nested folders
	fmt.Println("\n=== Creating 12 levels of nested folders ===")
	const folderDepth = 12
	var parentID *openapi_types.UUID
	var folderIDs []openapi_types.UUID
	var currentPath string

	for i := 1; i <= folderDepth; i++ {
		currentPath = fmt.Sprintf("%s/level%d", currentPath, i)
		createFolderReq := queryapi.CreateFolderJSONRequestBody{
			Path:      currentPath,
			ClientId:  clientID,
			CreatedBy: "test@example.com",
		}
		if parentID != nil {
			createFolderReq.ParentId = parentID
		}

		folderRes, err := net.Client.CreateFolderWithResponse(t.Context(), createFolderReq)
		require.NoError(t, err, "Failed to create folder at level %d", i)
		require.NotNil(t, folderRes.JSON201, "Expected 201 for folder at level %d, got %d with body: %s",
			i, folderRes.StatusCode(), string(folderRes.Body))

		folderID := folderRes.JSON201.Id
		folderIDs = append(folderIDs, folderID)
		parentID = &folderID
		fmt.Printf("  Level %d: ID=%s, Path=%s\n", i, folderID, currentPath)
	}

	assert.Len(t, folderIDs, folderDepth, "Should have created %d folders", folderDepth)

	// List all folders and verify count
	fmt.Println("\n=== Listing all folders for client ===")
	foldersRes, err := net.Client.ListClientFoldersWithResponse(t.Context(), queryapi.ClientID(clientID), nil)
	require.NoError(t, err, "Failed to list client folders")
	require.NotNil(t, foldersRes.JSON200, "Expected 200 response for folder list")

	// Should have at least 12 folders (plus root if created)
	fmt.Printf("  Total folders found: %d\n", len(foldersRes.JSON200.Folders))
	assert.GreaterOrEqual(t, len(foldersRes.JSON200.Folders), folderDepth,
		"Expected at least %d folders", folderDepth)

	// Verify we can get metrics for the deepest folder
	fmt.Println("\n=== Testing metrics for deepest folder (level 12) ===")
	deepestFolderID := folderIDs[len(folderIDs)-1]
	metricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), deepestFolderID)
	require.NoError(t, err, "Failed to get metrics for deepest folder")
	require.NotNil(t, metricsRes.JSON200, "Expected 200 for deepest folder metrics")
	fmt.Printf("  Deepest folder metrics: folderId=%s, totalDocuments=%d\n",
		metricsRes.JSON200.FolderId, metricsRes.JSON200.TotalDocuments)
	assert.Equal(t, deepestFolderID, metricsRes.JSON200.FolderId,
		"Metrics should be for the requested folder")

	// Verify we can get documents for the deepest folder (should be empty)
	fmt.Println("\n=== Testing documents for deepest folder ===")
	docsRes, err := net.Client.GetFolderDocumentsWithResponse(t.Context(), deepestFolderID)
	require.NoError(t, err, "Failed to get documents for deepest folder")
	require.NotNil(t, docsRes.JSON200, "Expected 200 for folder documents")
	docCount := 0
	if docsRes.JSON200.Documents != nil {
		docCount = len(*docsRes.JSON200.Documents)
	}
	fmt.Printf("  Documents in deepest folder: %d\n", docCount)

	fmt.Println("\n=== Deep folder hierarchy test complete ===")
}

// TestLargeArrayFieldExtraction tests field extraction creation with large array sizes (N=100)
// (Phase 10 - Performance test: Large array sizes).
func TestLargeArrayFieldExtraction(t *testing.T) {
	cfg := &TextExtractionConfig{}

	// Create full network
	net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
	defer cleanup()

	// Create test client
	clientName := fmt.Sprintf("LargeArrayTestClient_%d", time.Now().UnixNano())
	clientExternalID := fmt.Sprintf("lg-array-%d", time.Now().Unix())
	createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
		Name: clientName,
		Id:   clientExternalID,
	})
	require.NoError(t, err)
	require.NotNil(t, createRes.JSON201, "Expected 201 response for client creation")
	clientID := createRes.JSON201.Id
	fmt.Printf("Created client: %s\n", clientID)

	// Enable sync
	canSync := true
	_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
		CanSync: &canSync,
	})
	require.NoError(t, err)

	// Upload a single document
	fmt.Println("\n=== Uploading document ===")
	uploadSingleDocument(t, net.Client, clientID, "large_array_test.pdf")

	// Wait for document to appear
	var documents []queryapi.DocumentSummary
	waitForDocumentCount(t, net.Client, clientID, 1, &documents)
	require.Len(t, documents, 1, "Expected 1 document")
	docID := documents[0].Id
	fmt.Printf("  Document uploaded: ID=%s\n", docID)

	// Create field extraction with 100 array field items
	fmt.Println("\n=== Creating field extraction with 100 array items ===")
	const arraySize = 100
	arrayFields := make([]queryapi.ArrayFieldItem, arraySize)
	for i := 0; i < arraySize; i++ {
		arrayFields[i] = queryapi.ArrayFieldItem{
			ExhibitTitle:             ptr(fmt.Sprintf("Exhibit %d", i+1)),
			ExhibitPage:              ptr(fmt.Sprintf("%d", i+1)),
			ReimbProvTin:             ptr(fmt.Sprintf("%09d", i+100000000)),
			ReimbProvNpi:             ptr(fmt.Sprintf("%010d", i+1000000000)),
			ReimbProvName:            ptr(fmt.Sprintf("Provider %d", i+1)),
			AareteDerivedClaimTypeCd: ptr("INPATIENT"),
			ReimbPctRate:             ptrFloat64(float64(80+i%20) + float64(i%100)/100.0),
			ReimbFeeRate:             ptrFloat64(float64(100 + i*10)),
			CarveoutInd:              ptrBool(i%2 == 0),
			DefaultInd:               ptrBool(i == 0),
		}
	}

	startTime := time.Now()
	createFieldExtRes, err := net.Client.CreateFieldExtractionWithResponse(t.Context(), queryapi.CreateFieldExtractionJSONRequestBody{
		DocumentId: docID,
		SingleFields: queryapi.SingleFields{
			FileName:      ptr("large_array_test.pdf"),
			ContractTitle: ptr("Large Array Performance Test Contract"),
			ClientName:    ptr("Performance Test Client"),
		},
		ArrayFields: arrayFields,
		CreatedBy:   openapi_types.Email("test@example.com"),
	})
	createDuration := time.Since(startTime)
	require.NoError(t, err, "Failed to create field extraction")

	if createFieldExtRes.JSON400 != nil {
		t.Fatalf("Got 400 error creating field extraction: %s", createFieldExtRes.JSON400.Message)
	}
	require.NotNil(t, createFieldExtRes.JSON201, "Expected 201 for field extraction, got %d with body: %s",
		createFieldExtRes.StatusCode(), string(createFieldExtRes.Body))

	fmt.Printf("  Field extraction created in %v\n", createDuration)
	fmt.Printf("  Version: %d\n", createFieldExtRes.JSON201.Version)

	// Verify the current field extraction
	fmt.Println("\n=== Verifying current field extraction ===")
	getFieldExtRes, err := net.Client.GetCurrentFieldExtractionWithResponse(t.Context(), &queryapi.GetCurrentFieldExtractionParams{
		DocumentId: docID,
	})
	require.NoError(t, err, "Failed to get current field extraction")
	require.NotNil(t, getFieldExtRes.JSON200, "Expected 200 for get field extraction")

	// Verify array field count
	actualArraySize := len(getFieldExtRes.JSON200.ArrayFields)
	fmt.Printf("  Array fields returned: %d\n", actualArraySize)
	assert.Equal(t, arraySize, actualArraySize, "Expected %d array fields", arraySize)

	// Verify single fields
	assert.NotNil(t, getFieldExtRes.JSON200.SingleFields.ContractTitle)
	if getFieldExtRes.JSON200.SingleFields.ContractTitle != nil {
		assert.Equal(t, "Large Array Performance Test Contract", *getFieldExtRes.JSON200.SingleFields.ContractTitle)
	}

	// Performance assertion: should complete reasonably quickly (under 30 seconds)
	assert.Less(t, createDuration, 30*time.Second,
		"Field extraction with 100 array items should complete in under 30 seconds")

	fmt.Println("\n=== Large array field extraction test complete ===")
}

// TestLabelReapplicationHistory tests applying the same label multiple times to track history
// (Phase 10 - Performance test: Many labels per document history).
func TestLabelReapplicationHistory(t *testing.T) {
	cfg := &TextExtractionConfig{}

	// Create full network
	net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
	defer cleanup()

	// Create test client
	clientName := fmt.Sprintf("LabelHistoryTestClient_%d", time.Now().UnixNano())
	clientExternalID := fmt.Sprintf("lbl-hist-%d", time.Now().Unix())
	createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
		Name: clientName,
		Id:   clientExternalID,
	})
	require.NoError(t, err)
	require.NotNil(t, createRes.JSON201, "Expected 201 response for client creation")
	clientID := createRes.JSON201.Id
	fmt.Printf("Created client: %s\n", clientID)

	// Enable sync
	canSync := true
	_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
		CanSync: &canSync,
	})
	require.NoError(t, err)

	// Upload a single document
	fmt.Println("\n=== Uploading document ===")
	uploadSingleDocument(t, net.Client, clientID, "label_history_test.pdf")

	// Wait for document to appear
	var documents []queryapi.DocumentSummary
	waitForDocumentCount(t, net.Client, clientID, 1, &documents)
	require.Len(t, documents, 1, "Expected 1 document")
	docID := documents[0].Id
	fmt.Printf("  Document uploaded: ID=%s\n", docID)

	// Apply each of the 5 available labels multiple times to build history
	// Note: The system has 5 predefined labels: Ingested, OCR_Processed, GenAI_Processed, Doczy_AI_Completed, Dashboard_Ready
	labels := []string{"Ingested", "OCR_Processed", "GenAI_Processed", "Doczy_AI_Completed", "Dashboard_Ready"}
	const applicationsPerLabel = 4

	fmt.Printf("\n=== Applying each label %d times (total %d label applications) ===\n",
		applicationsPerLabel, len(labels)*applicationsPerLabel)

	totalApplications := 0
	for round := 1; round <= applicationsPerLabel; round++ {
		for _, label := range labels {
			applyLabelRes, err := net.Client.ApplyLabelWithResponse(t.Context(), docID, queryapi.ApplyLabelJSONRequestBody{
				Label:     label,
				AppliedBy: openapi_types.Email(fmt.Sprintf("user%d@example.com", round)),
			})
			require.NoError(t, err, "Failed to apply label %s in round %d", label, round)

			if applyLabelRes.JSON201 != nil {
				totalApplications++
				if round == 1 || round == applicationsPerLabel {
					fmt.Printf("  Round %d: Applied '%s'\n", round, label)
				}
			} else if applyLabelRes.JSON400 != nil {
				// This might happen if re-application isn't allowed - that's informative
				t.Logf("  Round %d: Label '%s' returned 400: %s", round, label, applyLabelRes.JSON400.Message)
			}
		}
		if round == 1 {
			fmt.Println("  ...")
		}
	}

	fmt.Printf("  Total successful label applications: %d\n", totalApplications)

	// Get all labels for the document and verify history
	fmt.Println("\n=== Verifying label history ===")
	getLabelsRes, err := net.Client.GetDocumentLabelsWithResponse(t.Context(), docID)
	require.NoError(t, err, "Failed to get document labels")
	require.NotNil(t, getLabelsRes.JSON200, "Expected 200 for get labels")

	labelCount := 0
	if getLabelsRes.JSON200.Labels != nil {
		labelCount = len(*getLabelsRes.JSON200.Labels)
	}
	fmt.Printf("  Total label entries in history: %d\n", labelCount)

	// If re-application is allowed, we should have multiple entries
	// If not, we should have exactly 5 (one per unique label)
	if labelCount == len(labels) {
		fmt.Println("  Note: System appears to keep only most recent application per label")
	} else if labelCount == totalApplications {
		fmt.Println("  Note: System preserves full history of all label applications")
	}

	// The key test: verify we have at least one entry per label
	assert.GreaterOrEqual(t, labelCount, len(labels),
		"Should have at least one entry per label type")

	// Verify we can query documents by each label
	fmt.Println("\n=== Verifying query by label ===")
	for _, label := range labels {
		getDocsByLabelRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), label, &queryapi.GetDocumentsByLabelParams{
			ClientId: queryapi.ClientID(clientID),
		})
		require.NoError(t, err, "Failed to get documents by label %s", label)
		require.NotNil(t, getDocsByLabelRes.JSON200, "Expected 200 for get documents by label %s", label)

		docCount := 0
		if getDocsByLabelRes.JSON200.Documents != nil {
			docCount = len(*getDocsByLabelRes.JSON200.Documents)
		}
		assert.Equal(t, 1, docCount, "Expected 1 document with label '%s'", label)
	}
	fmt.Println("  All labels return the document correctly")

	fmt.Println("\n=== Label reapplication history test complete ===")
}

// TestFieldExtractionByFolderAndLabel tests Scenario 2: Query field extractions by folder and label
// (Phase 10 - Scenario 2: Cross-feature integration).
// This test uses a simpler approach - single document upload to test cross-feature integration.
func TestFieldExtractionByFolderAndLabel(t *testing.T) {
	cfg := &TextExtractionConfig{}

	// Create full network
	net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
	defer cleanup()

	// Create test client
	clientName := fmt.Sprintf("FolderLabelFETestClient_%d", time.Now().UnixNano())
	clientExternalID := fmt.Sprintf("fld-lbl-fe-%d", time.Now().Unix())
	createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
		Name: clientName,
		Id:   clientExternalID,
	})
	require.NoError(t, err)
	require.NotNil(t, createRes.JSON201, "Expected 201 response for client creation, got status: %d, body: %s",
		createRes.StatusCode(), string(createRes.Body))
	clientID := createRes.JSON201.Id
	fmt.Printf("Created client: %s\n", clientID)

	// Enable sync
	canSync := true
	_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
		CanSync: &canSync,
	})
	require.NoError(t, err)

	// Create two folders
	fmt.Println("\n=== Creating folders ===")
	folder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
		Path:      "/contracts",
		ClientId:  clientID,
		CreatedBy: "test@example.com",
	})
	require.NoError(t, err, "Failed to create folder1")
	require.NotNil(t, folder1Res.JSON201, "Expected 201 for folder1")
	folder1ID := folder1Res.JSON201.Id
	fmt.Printf("  Created /contracts folder: ID=%s\n", folder1ID)

	folder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
		Path:      "/invoices",
		ClientId:  clientID,
		CreatedBy: "test@example.com",
	})
	require.NoError(t, err, "Failed to create folder2")
	require.NotNil(t, folder2Res.JSON201, "Expected 201 for folder2")
	folder2ID := folder2Res.JSON201.Id
	fmt.Printf("  Created /invoices folder: ID=%s\n", folder2ID)

	// Upload single document and wait for it
	fmt.Println("\n=== Uploading document ===")
	uploadSingleDocument(t, net.Client, clientID, "contract_doc.pdf")

	var documents []queryapi.DocumentSummary
	waitForDocumentCount(t, net.Client, clientID, 1, &documents)
	require.Len(t, documents, 1, "Expected 1 document")
	docID := documents[0].Id
	fmt.Printf("  Uploaded document: ID=%s\n", docID)

	// Apply labels to document
	fmt.Println("\n=== Applying labels ===")
	for _, label := range []string{"Ingested", "OCR_Processed", "Dashboard_Ready"} {
		_, err := net.Client.ApplyLabelWithResponse(t.Context(), docID, queryapi.ApplyLabelJSONRequestBody{
			Label:     label,
			AppliedBy: openapi_types.Email("test@example.com"),
		})
		require.NoError(t, err, "Failed to apply label %s", label)
	}
	fmt.Printf("  Applied Ingested, OCR_Processed, Dashboard_Ready\n")

	// Create field extraction
	fmt.Println("\n=== Creating field extraction ===")
	feRes, err := net.Client.CreateFieldExtractionWithResponse(t.Context(), queryapi.CreateFieldExtractionJSONRequestBody{
		DocumentId: docID,
		SingleFields: queryapi.SingleFields{
			FileName:      ptr("contract_doc.pdf"),
			ContractTitle: ptr("Service Agreement"),
			ClientName:    ptr("Test Client"),
		},
		ArrayFields: []queryapi.ArrayFieldItem{
			{
				ExhibitTitle: ptr("Exhibit A"),
				ReimbPctRate: ptrFloat64(85.0),
				ReimbFeeRate: ptrFloat64(150.0),
			},
		},
		CreatedBy: openapi_types.Email("test@example.com"),
	})
	require.NoError(t, err, "Failed to create field extraction")
	require.NotNil(t, feRes.JSON201, "Expected 201 for field extraction")
	fmt.Printf("  Field extraction created: version=%d\n", feRes.JSON201.Version)

	// Query by label and verify field extraction exists
	fmt.Println("\n=== Querying documents by label and verifying field extractions ===")

	// Get documents with Dashboard_Ready label
	dashboardReadyDocsRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), "Dashboard_Ready", &queryapi.GetDocumentsByLabelParams{
		ClientId: queryapi.ClientID(clientID),
	})
	require.NoError(t, err, "Failed to get documents by Dashboard_Ready label")
	require.NotNil(t, dashboardReadyDocsRes.JSON200, "Expected 200 for get documents by label")

	dashboardReadyDocs := dashboardReadyDocsRes.JSON200.Documents
	require.NotNil(t, dashboardReadyDocs, "Expected documents list")
	assert.Len(t, *dashboardReadyDocs, 1, "Expected 1 document with Dashboard_Ready label")
	fmt.Printf("  Documents with Dashboard_Ready: %d\n", len(*dashboardReadyDocs))

	// Verify that document has a field extraction
	for _, doc := range *dashboardReadyDocs {
		feGetRes, err := net.Client.GetCurrentFieldExtractionWithResponse(t.Context(), &queryapi.GetCurrentFieldExtractionParams{
			DocumentId: doc.Id,
		})
		require.NoError(t, err, "Failed to get field extraction for document %s", doc.Id)
		require.NotNil(t, feGetRes.JSON200, "Expected field extraction for Dashboard_Ready document")
		assert.NotNil(t, feGetRes.JSON200.SingleFields.ContractTitle, "Expected contract title")
		fmt.Printf("  Document %s has field extraction with title: %s\n",
			doc.Id, *feGetRes.JSON200.SingleFields.ContractTitle)
	}

	// Get documents with OCR_Processed label
	ocrDocsRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), "OCR_Processed", &queryapi.GetDocumentsByLabelParams{
		ClientId: queryapi.ClientID(clientID),
	})
	require.NoError(t, err, "Failed to get documents by OCR_Processed label")
	require.NotNil(t, ocrDocsRes.JSON200, "Expected 200 for get documents by label")

	ocrDocs := ocrDocsRes.JSON200.Documents
	require.NotNil(t, ocrDocs, "Expected documents list")
	assert.Len(t, *ocrDocs, 1, "Expected 1 document with OCR_Processed label")
	fmt.Printf("  Documents with OCR_Processed: %d\n", len(*ocrDocs))

	// Verify folder metrics
	fmt.Println("\n=== Verifying folder metrics ===")
	metrics1Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder1ID)
	require.NoError(t, err, "Failed to get folder1 metrics")
	require.NotNil(t, metrics1Res.JSON200, "Expected 200 for folder1 metrics")
	fmt.Printf("  /contracts folder metrics: totalDocs=%d\n", metrics1Res.JSON200.TotalDocuments)

	metrics2Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder2ID)
	require.NoError(t, err, "Failed to get folder2 metrics")
	require.NotNil(t, metrics2Res.JSON200, "Expected 200 for folder2 metrics")
	fmt.Printf("  /invoices folder metrics: totalDocs=%d\n", metrics2Res.JSON200.TotalDocuments)

	// Test: Get all documents with Ingested label
	fmt.Println("\n=== Verifying Ingested documents ===")
	allIngestedRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), "Ingested", &queryapi.GetDocumentsByLabelParams{
		ClientId: queryapi.ClientID(clientID),
	})
	require.NoError(t, err, "Failed to get documents by Ingested label")
	require.NotNil(t, allIngestedRes.JSON200, "Expected 200 for get documents by label")
	ingestedDocs := allIngestedRes.JSON200.Documents
	require.NotNil(t, ingestedDocs, "Expected documents list")
	assert.Len(t, *ingestedDocs, 1, "Expected 1 document with Ingested label")
	fmt.Printf("  Documents with Ingested label: %d\n", len(*ingestedDocs))

	// Verify field extraction on the ingested document
	fmt.Println("\n=== Checking field extraction on Ingested document ===")
	for _, doc := range *ingestedDocs {
		feGetRes, err := net.Client.GetCurrentFieldExtractionWithResponse(t.Context(), &queryapi.GetCurrentFieldExtractionParams{
			DocumentId: doc.Id,
		})
		require.NoError(t, err, "Failed to query field extraction for document %s", doc.Id)
		require.NotNil(t, feGetRes.JSON200, "Expected field extraction for Ingested document")
		fmt.Printf("  Document %s has field extraction\n", doc.Id)
	}

	fmt.Println("\n=== Field extraction by folder and label test complete ===")
}

// Ensure io import is used
var _ = io.EOF