Merged in feature/textExtractionsPart1 (pull request #192)

all schema and rest apis for text extraction support * in progress * stage 8 complete * phase 9 completed * phase 9 complete * ongoing - s3 path fix * working * optimize ci build * e2e tests * missing test
2025-11-26 19:23:42 +00:00
parent 2b43799f56
commit c45e1dd427
67 changed files with 16120 additions and 817 deletions
@@ -0,0 +1,848 @@
+package endtoend_test
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"testing"
+	"time"
+
+	"queryorchestration/internal/serviceconfig"
+	"queryorchestration/internal/serviceconfig/aws"
+	"queryorchestration/internal/serviceconfig/objectstore"
+	"queryorchestration/internal/test"
+
+	queryapi "queryorchestration/pkg/queryAPI"
+
+	openapi_types "github.com/oapi-codegen/runtime/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TextExtractionConfig embeds the configuration needed for the text extraction integration tests.
+type TextExtractionConfig struct {
+	serviceconfig.BaseConfig
+	aws.AWSConfig
+	objectstore.ObjectStoreConfig
+}
+
+// TestE2EDocWorkflow is a comprehensive integration test that exercises the
+// document upload, folder management, label, and field extraction APIs via HTTP REST calls.
+//
+// The test performs the following steps:
+//
+// Document Upload Phase:
+// 1. Create a new client via POST /client
+// 2. Verify client exists via GET /client/{id}
+// 3. Update client to allow sync via PATCH /client/{id}
+// 4. Confirm client status via GET /client/{id}/status
+// 5. Create a ZIP with 3 PDFs in folder structures
+// 6. Upload batch via POST /client/{id}/document/batch
+// 7. Verify batch via GET /client/{id}/document/batch
+// 8. Get documents via GET /client/{id}/document and verify 3 documents
+// 9-10. Verify folder-based document retrieval (skipped - requires folder IDs from processing)
+// 11. POST a single document to root
+// 12. Verify 4 documents total
+//
+// Label Operations Phase:
+// 13. Apply all labels one by one to a document via POST /documents/{documentId}/labels
+// 14. After each label, verify via GET /documents/{documentId}/labels
+// 15. List all folders for client via GET /clients/{clientId}/folders
+// 16. Get metrics for each folder via GET /folders/{folderId}/metrics
+// 17. Get documents by label via GET /labels/{labelName}/documents
+//
+// Field Extractions Phase:
+// 18. Verify no field extraction exists via GET /field-extractions
+// 19. Create field extraction via POST /field-extractions
+// 20. Verify single extraction via GET /field-extractions/history
+// 21. Modify and create new version via POST /field-extractions
+// 22. Verify two versions via GET /field-extractions/history
+func TestE2EDocWorkflow(t *testing.T) {
+	t.Run("docs", func(t *testing.T) {
+		cfg := &TextExtractionConfig{}
+
+		// Create full network with database, S3, and HTTP server
+		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
+		defer cleanup()
+
+		// Phase 1: Client and Document Upload
+		clientID, clientName := runClientSetupPhase(t, net)
+		documents := runDocumentUploadPhase(t, net, clientID)
+
+		// Phase 2: Label Operations
+		targetDocID := documents[0].Id
+		runLabelOperationsPhase(t, net, clientID, targetDocID)
+
+		// Phase 3: Field Extractions
+		runFieldExtractionsPhase(t, net, targetDocID)
+
+		fmt.Println("\n=== Test Complete: All steps passed ===")
+		_ = clientName // Used in output
+	})
+}
+
+// runClientSetupPhase handles steps 1-4: client creation, verification, and configuration.
+// Returns the client ID and client name.
+func runClientSetupPhase(t *testing.T, net test.Network) (string, string) {
+	t.Helper()
+
+	fmt.Println("=== Step 1: Create a new client ===")
+	clientName := fmt.Sprintf("TextExtractionTestClient_%d", time.Now().UnixNano())
+	clientExternalID := fmt.Sprintf("text-ext-%d", time.Now().UnixNano())
+
+	createClientRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
+		Name: clientName,
+		Id:   clientExternalID,
+	})
+	require.NoError(t, err, "Failed to create client")
+	if createClientRes.JSON400 != nil {
+		t.Fatalf("Got 400 error: %s", createClientRes.JSON400.Message)
+	}
+	if createClientRes.JSON201 == nil {
+		t.Fatalf("Expected 201 response, got status %d with body: %s", createClientRes.StatusCode(), string(createClientRes.Body))
+	}
+	clientID := createClientRes.JSON201.Id
+	fmt.Printf("  Created client: ID=%s, Name=%s, ExternalID=%s\n", clientID, clientName, clientExternalID)
+
+	fmt.Println("\n=== Step 2: Verify client exists via GET /client/{id} ===")
+	getClientRes, err := net.Client.GetClientWithResponse(t.Context(), clientID)
+	require.NoError(t, err, "Failed to get client")
+	require.NotNil(t, getClientRes.JSON200, "Expected 200 response, got status %d", getClientRes.StatusCode())
+	fmt.Printf("  Client verified: ID=%s, Name=%s\n", getClientRes.JSON200.Id, getClientRes.JSON200.Name)
+	assert.Equal(t, clientName, getClientRes.JSON200.Name, "Client name mismatch")
+
+	fmt.Println("\n=== Step 3: Update client to allow sync ===")
+	canSync := true
+	updateClientRes, err := net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
+		CanSync: &canSync,
+	})
+	require.NoError(t, err, "Failed to update client")
+	require.Equal(t, http.StatusOK, updateClientRes.StatusCode(), "Expected 200 response for update")
+	fmt.Printf("  Client updated: canSync=%t\n", canSync)
+
+	fmt.Println("\n=== Step 4: Confirm client status ===")
+	statusRes, err := net.Client.GetStatusByClientIdWithResponse(t.Context(), clientID)
+	require.NoError(t, err, "Failed to get client status")
+	require.NotNil(t, statusRes.JSON200, "Expected 200 response, got status %d", statusRes.StatusCode())
+	fmt.Printf("  Client status: %s\n", statusRes.JSON200.Status)
+
+	return clientID, clientName
+}
+
+// runDocumentUploadPhase handles steps 5-12: ZIP creation, batch upload, and document verification.
+// Returns the list of documents.
+func runDocumentUploadPhase(t *testing.T, net test.Network, clientID string) []queryapi.DocumentSummary {
+	t.Helper()
+
+	fmt.Println("\n=== Step 5: Create ZIP with 3 PDFs ===")
+	zipBuffer := createTestZIP(t)
+	fmt.Printf("  ZIP created: %d bytes with 3 PDF files\n", zipBuffer.Len())
+	fmt.Println("  Files in ZIP:")
+	fmt.Println("    - folder1/subfolder2/file1.pdf")
+	fmt.Println("    - folder1/subfolder2/file2.pdf")
+	fmt.Println("    - folder2/subfolder1/file1.pdf")
+
+	fmt.Println("\n=== Step 6: Upload batch via POST /client/{id}/document/batch ===")
+	batchID := uploadBatchZIP(t, net.Client, clientID, zipBuffer.Bytes())
+	fmt.Printf("  Batch uploaded: batchID=%s\n", batchID)
+
+	fmt.Println("\n=== Step 7: Verify batch status ===")
+	verifyBatchStatus(t, net, clientID, batchID)
+
+	fmt.Println("\n=== Step 8: Get documents and verify count ===")
+	var documents []queryapi.DocumentSummary
+	waitForDocumentCount(t, net.Client, clientID, 3, &documents)
+	fmt.Printf("  Documents found: %d\n", len(documents))
+	for i, doc := range documents {
+		fmt.Printf("    Document %d: ID=%s, Hash=%s\n", i+1, doc.Id, doc.Hash)
+	}
+	assert.Len(t, documents, 3, "Expected 3 documents from batch upload")
+
+	fmt.Println("\n=== Step 9: Get documents for folder1 (should see 2) ===")
+	fmt.Println("  Note: Folder-based document retrieval requires folder IDs from folder creation during processing")
+	fmt.Println("  Skipping folder-specific document retrieval - documents verified at client level")
+
+	fmt.Println("\n=== Step 10: Get documents for folder2 (should see 1) ===")
+	fmt.Println("  Note: Folder-based document retrieval requires folder IDs from folder creation during processing")
+	fmt.Println("  Skipping folder-specific document retrieval - documents verified at client level")
+
+	fmt.Println("\n=== Step 11: Upload single document to root ===")
+	uploadSingleDocument(t, net.Client, clientID, "single_root_document.pdf")
+	fmt.Println("  Single document uploaded: single_root_document.pdf")
+
+	fmt.Println("\n=== Step 12: Verify 4 documents total ===")
+	waitForDocumentCount(t, net.Client, clientID, 4, &documents)
+	fmt.Printf("  Final document count: %d\n", len(documents))
+	for i, doc := range documents {
+		fmt.Printf("    Document %d: ID=%s, Hash=%s\n", i+1, doc.Id, doc.Hash)
+	}
+	assert.Len(t, documents, 4, "Expected 4 documents total")
+
+	return documents
+}
+
+// verifyBatchStatus polls for batch status and verifies completion.
+func verifyBatchStatus(t *testing.T, net test.Network, clientID string, batchID queryapi.BatchID) {
+	t.Helper()
+
+	var batchStatus string
+	for i := 0; i < 30; i++ {
+		batchRes, err := net.Client.GetDocumentBatchWithResponse(t.Context(), clientID, batchID)
+		require.NoError(t, err, "Failed to get batch status")
+		require.NotNil(t, batchRes.JSON200, "Expected 200 response for batch, got status %d", batchRes.StatusCode())
+		batchStatus = string(batchRes.JSON200.Status)
+		fmt.Printf("  Batch status check %d: status=%s, processed=%d, total=%d\n",
+			i+1, batchStatus, batchRes.JSON200.ProcessedDocuments, batchRes.JSON200.TotalDocuments)
+
+		if batchStatus == "completed" || batchStatus == "failed" {
+			break
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	fmt.Printf("  Final batch status: %s\n", batchStatus)
+
+	listBatchesRes, err := net.Client.ListDocumentBatchesWithResponse(t.Context(), clientID, nil)
+	require.NoError(t, err, "Failed to list batches")
+	require.NotNil(t, listBatchesRes.JSON200, "Expected 200 response for batch list")
+	fmt.Printf("  Total batches for client: %d\n", listBatchesRes.JSON200.TotalCount)
+}
+
+// runLabelOperationsPhase handles steps 13-17: applying labels, verifying, and querying by label.
+func runLabelOperationsPhase(t *testing.T, net test.Network, clientID string, targetDocID openapi_types.UUID) {
+	t.Helper()
+
+	fmt.Println("\n=== Step 13: Apply all labels to a document ===")
+	fmt.Printf("  Target document for labels: ID=%s\n", targetDocID)
+
+	labels := []string{"Ingested", "OCR_Processed", "GenAI_Processed", "Doczy_AI_Completed", "Dashboard_Ready"}
+	applyAndVerifyLabels(t, net, targetDocID, labels)
+
+	fmt.Println("\n=== Step 15: List folders and get folder metrics ===")
+	verifyFolderMetrics(t, net, clientID)
+
+	fmt.Println("\n=== Step 17: Get documents by label ===")
+	verifyDocumentsByLabel(t, net, clientID, "Ingested")
+}
+
+// verifyFolderMetrics lists all folders for a client and verifies folder metrics.
+func verifyFolderMetrics(t *testing.T, net test.Network, clientID string) {
+	t.Helper()
+
+	// List all folders for the client using the new endpoint
+	foldersRes, err := net.Client.ListClientFoldersWithResponse(t.Context(), queryapi.ClientID(clientID))
+	require.NoError(t, err, "Failed to list client folders")
+	if foldersRes.JSON200 == nil {
+		t.Logf("Response status: %d, body: %s", foldersRes.StatusCode(), string(foldersRes.Body))
+		if foldersRes.JSON404 != nil {
+			t.Logf("404 error: %s", foldersRes.JSON404.Message)
+		}
+		if foldersRes.JSON500 != nil {
+			t.Logf("500 error: %s", foldersRes.JSON500.Message)
+			// Print container logs to diagnose the issue
+			fmt.Println("\n=== QueryAPI Container Logs (for 500 error diagnosis) ===")
+			if apiContainer, ok := net.APIs[test.QueryAPIName]; ok && apiContainer != nil {
+				test.PrintContainerLogs(t, apiContainer.Container)
+			}
+			fmt.Println("=== End Container Logs ===")
+			t.Fatalf("ListClientFolders returned 500 error: %s", foldersRes.JSON500.Message)
+		}
+	}
+	require.NotNil(t, foldersRes.JSON200, "Expected 200 response for folder list")
+
+	folders := foldersRes.JSON200.Folders
+	fmt.Printf("  Folders found for client: %d\n", len(folders))
+
+	if len(folders) == 0 {
+		fmt.Println("  Note: No folders found - documents may be at root level")
+		return
+	}
+
+	// Print folder hierarchy
+	fmt.Println("  Folder hierarchy:")
+	for _, folder := range folders {
+		parentInfo := "root"
+		if folder.ParentId.IsSpecified() {
+			val, _ := folder.ParentId.Get()
+			parentInfo = fmt.Sprintf("parent=%s", val)
+		}
+		fmt.Printf("    - %s (ID=%s, %s)\n", folder.Path, folder.Id, parentInfo)
+	}
+
+	// Test metrics for each folder
+	fmt.Println("\n=== Step 16: Get metrics for each folder ===")
+	for _, folder := range folders {
+		metricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder.Id)
+		require.NoError(t, err, "Failed to get metrics for folder %s", folder.Path)
+		require.NotNil(t, metricsRes.JSON200, "Expected 200 response for folder metrics")
+
+		fmt.Printf("  Folder '%s': totalDocuments=%d, labels=%v\n",
+			folder.Path, metricsRes.JSON200.TotalDocuments, metricsRes.JSON200.ByLabel)
+	}
+}
+
+// applyAndVerifyLabels applies each label and verifies after each application.
+func applyAndVerifyLabels(t *testing.T, net test.Network, docID openapi_types.UUID, labels []string) {
+	t.Helper()
+
+	for i, label := range labels {
+		fmt.Printf("\n=== Step 13.%d: Applying label '%s' ===\n", i+1, label)
+		applyLabelRes, err := net.Client.ApplyLabelWithResponse(t.Context(), docID, queryapi.ApplyLabelJSONRequestBody{
+			Label:     label,
+			AppliedBy: openapi_types.Email("test@example.com"),
+		})
+		require.NoError(t, err, "Failed to apply label %s", label)
+		if applyLabelRes.JSON400 != nil {
+			t.Logf("WARNING: Got 400 error applying label '%s': %s", label, applyLabelRes.JSON400.Message)
+			continue
+		}
+		if applyLabelRes.JSON201 == nil {
+			t.Logf("WARNING: Expected 201 response for label '%s', got status %d with body: %s", label, applyLabelRes.StatusCode(), string(applyLabelRes.Body))
+			continue
+		}
+		fmt.Printf("  Label '%s' applied: recordID=%s\n", label, applyLabelRes.JSON201.Id)
+
+		fmt.Println("  Verifying labels via GET /documents/{documentId}/labels")
+		getLabelsRes, err := net.Client.GetDocumentLabelsWithResponse(t.Context(), docID)
+		require.NoError(t, err, "Failed to get document labels")
+		if getLabelsRes.JSON200 == nil {
+			t.Logf("WARNING: Expected 200 for get labels, got %d", getLabelsRes.StatusCode())
+			continue
+		}
+		fmt.Printf("  Labels on document: count=%d\n", len(*getLabelsRes.JSON200.Labels))
+		assert.Len(t, *getLabelsRes.JSON200.Labels, i+1, "Expected %d labels after applying '%s'", i+1, label)
+	}
+}
+
+// verifyDocumentsByLabel queries documents by label and verifies the result.
+func verifyDocumentsByLabel(t *testing.T, net test.Network, clientID string, label string) {
+	t.Helper()
+
+	fmt.Printf("  Getting documents with label '%s' for client '%s'\n", label, clientID)
+	getDocsByLabelRes, err := net.Client.GetDocumentsByLabelWithResponse(t.Context(), label, &queryapi.GetDocumentsByLabelParams{
+		ClientId: queryapi.ClientID(clientID),
+	})
+	require.NoError(t, err, "Failed to get documents by label")
+	if getDocsByLabelRes.JSON200 == nil {
+		t.Logf("WARNING: Expected 200 for get documents by label, got %d with body: %s", getDocsByLabelRes.StatusCode(), string(getDocsByLabelRes.Body))
+	} else {
+		docCount := 0
+		if getDocsByLabelRes.JSON200.Documents != nil {
+			docCount = len(*getDocsByLabelRes.JSON200.Documents)
+		}
+		fmt.Printf("  Documents with label '%s': count=%d\n", label, docCount)
+		assert.GreaterOrEqual(t, docCount, 1, "Expected at least 1 document with label '%s'", label)
+	}
+}
+
+// runFieldExtractionsPhase handles steps 18-22: field extraction creation and versioning.
+func runFieldExtractionsPhase(t *testing.T, net test.Network, targetDocID openapi_types.UUID) {
+	t.Helper()
+
+	fmt.Println("\n=== Step 18: Verify no field extraction exists ===")
+	verifyNoFieldExtraction(t, net, targetDocID)
+
+	fmt.Println("\n=== Step 19: Create field extraction ===")
+	createFieldExtraction(t, net, targetDocID, "Test Contract Title", "Test Client Name")
+
+	fmt.Println("\n=== Step 20: Verify single extraction via history ===")
+	verifyFieldExtractionHistory(t, net, targetDocID, 1)
+
+	fmt.Println("\n=== Step 21: Create second version of field extraction ===")
+	createFieldExtraction(t, net, targetDocID, "Updated Contract Title v2", "Updated Client Name v2")
+
+	fmt.Println("\n=== Step 22: Verify two versions in history ===")
+	verifyFieldExtractionHistory(t, net, targetDocID, 2)
+}
+
+// verifyNoFieldExtraction checks that no field extraction exists for the document.
+func verifyNoFieldExtraction(t *testing.T, net test.Network, docID openapi_types.UUID) {
+	t.Helper()
+
+	getFieldExtRes, err := net.Client.GetCurrentFieldExtractionWithResponse(t.Context(), &queryapi.GetCurrentFieldExtractionParams{
+		DocumentId: docID,
+	})
+	require.NoError(t, err, "Failed to get current field extraction")
+	if getFieldExtRes.JSON404 != nil {
+		fmt.Println("  Confirmed: No field extraction exists for document (404)")
+	} else if getFieldExtRes.JSON200 != nil {
+		fmt.Printf("  WARNING: Field extraction already exists: version=%d\n", getFieldExtRes.JSON200.Version)
+	} else {
+		fmt.Printf("  Got status %d: %s\n", getFieldExtRes.StatusCode(), string(getFieldExtRes.Body))
+	}
+}
+
+// createFieldExtraction creates a field extraction with the given values.
+func createFieldExtraction(t *testing.T, net test.Network, docID openapi_types.UUID, contractTitle, clientName string) {
+	t.Helper()
+
+	createFieldExtRes, err := net.Client.CreateFieldExtractionWithResponse(t.Context(), queryapi.CreateFieldExtractionJSONRequestBody{
+		DocumentId: docID,
+		SingleFields: queryapi.SingleFields{
+			FileName:      ptr("test_document.pdf"),
+			ContractTitle: &contractTitle,
+			ClientName:    &clientName,
+		},
+		ArrayFields: []queryapi.ArrayFieldItem{},
+		CreatedBy:   openapi_types.Email("test@example.com"),
+	})
+	require.NoError(t, err, "Failed to create field extraction")
+	if createFieldExtRes.JSON400 != nil {
+		t.Logf("WARNING: Got 400 error creating field extraction: %s", createFieldExtRes.JSON400.Message)
+	} else if createFieldExtRes.JSON201 == nil {
+		t.Logf("WARNING: Expected 201 for field extraction, got %d with body: %s", createFieldExtRes.StatusCode(), string(createFieldExtRes.Body))
+	} else {
+		fmt.Printf("  Field extraction created: version=%d\n", createFieldExtRes.JSON201.Version)
+	}
+}
+
+// verifyFieldExtractionHistory checks that the expected number of versions exist.
+func verifyFieldExtractionHistory(t *testing.T, net test.Network, docID openapi_types.UUID, expectedCount int) {
+	t.Helper()
+
+	historyRes, err := net.Client.GetFieldExtractionHistoryWithResponse(t.Context(), &queryapi.GetFieldExtractionHistoryParams{
+		DocumentId: docID,
+	})
+	require.NoError(t, err, "Failed to get field extraction history")
+	if historyRes.JSON200 == nil {
+		t.Logf("WARNING: Expected 200 for history, got %d with body: %s", historyRes.StatusCode(), string(historyRes.Body))
+	} else {
+		historyCount := 0
+		if historyRes.JSON200.Versions != nil {
+			historyCount = len(*historyRes.JSON200.Versions)
+		}
+		fmt.Printf("  Field extraction history: count=%d\n", historyCount)
+		assert.Equal(t, expectedCount, historyCount, "Expected %d field extraction(s) in history", expectedCount)
+	}
+}
+
+// createTestZIP creates a ZIP file in memory containing 3 simple PDF files
+// in the specified folder structure.
+func createTestZIP(t *testing.T) *bytes.Buffer {
+	t.Helper()
+
+	buf := new(bytes.Buffer)
+	zipWriter := zip.NewWriter(buf)
+
+	// PDF files to create in the ZIP
+	files := []struct {
+		path    string
+		content string
+	}{
+		{"folder1/subfolder2/file1.pdf", pdfContent("Document 1 in folder1/subfolder2")},
+		{"folder1/subfolder2/file2.pdf", pdfContent("Document 2 in folder1/subfolder2")},
+		{"folder2/subfolder1/file1.pdf", pdfContent("Document 1 in folder2/subfolder1")},
+	}
+
+	for _, file := range files {
+		writer, err := zipWriter.Create(file.path)
+		require.NoError(t, err, "Failed to create file in ZIP: %s", file.path)
+
+		_, err = writer.Write([]byte(file.content))
+		require.NoError(t, err, "Failed to write content to ZIP file: %s", file.path)
+	}
+
+	err := zipWriter.Close()
+	require.NoError(t, err, "Failed to close ZIP writer")
+
+	return buf
+}
+
+// pdfContent generates a minimal valid PDF with the given text content.
+func pdfContent(text string) string {
+	// Minimal valid PDF structure
+	return fmt.Sprintf(`%%PDF-1.4
+%%âãÏÓ
+1 0 obj
+<<
+  /Type /Catalog
+  /Pages 2 0 R
+>>
+endobj
+2 0 obj
+<<
+  /Type /Pages
+  /Kids [3 0 R]
+  /Count 1
+>>
+endobj
+3 0 obj
+<<
+  /Type /Page
+  /Parent 2 0 R
+  /MediaBox [0 0 612 792]
+  /Resources <<
+    /Font <<
+      /F1 <<
+        /Type /Font
+        /Subtype /Type1
+        /BaseFont /Helvetica
+      >>
+    >>
+  >>
+  /Contents 4 0 R
+>>
+endobj
+4 0 obj
+<<
+  /Length 50
+>>
+stream
+BT
+/F1 12 Tf
+100 700 Td
+(%s) Tj
+ET
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f
+0000000015 00000 n
+0000000066 00000 n
+0000000125 00000 n
+0000000330 00000 n
+trailer
+<<
+  /Size 5
+  /Root 1 0 R
+>>
+startxref
+430
+%%%%EOF`, text)
+}
+
+// uploadBatchZIP uploads a ZIP file to the batch upload endpoint and returns the batch ID.
+func uploadBatchZIP(t *testing.T, client *queryapi.ClientWithResponses, clientID string, zipContent []byte) queryapi.BatchID {
+	t.Helper()
+
+	// Create multipart form
+	var buf bytes.Buffer
+	writer := multipart.NewWriter(&buf)
+
+	part, err := writer.CreateFormFile("archive", "test_batch.zip")
+	require.NoError(t, err, "Failed to create form file")
+
+	_, err = part.Write(zipContent)
+	require.NoError(t, err, "Failed to write ZIP content")
+
+	err = writer.Close()
+	require.NoError(t, err, "Failed to close multipart writer")
+
+	// Upload using the generated client
+	res, err := client.UploadDocumentBatchWithBodyWithResponse(
+		t.Context(),
+		clientID,
+		writer.FormDataContentType(),
+		bytes.NewReader(buf.Bytes()),
+	)
+	require.NoError(t, err, "Failed to upload batch")
+	require.NotNil(t, res.JSON202, "Expected 202 response, got status %d", res.StatusCode())
+
+	return res.JSON202.BatchId
+}
+
+// uploadSingleDocument uploads a single PDF document to the client's root folder.
+func uploadSingleDocument(t *testing.T, client *queryapi.ClientWithResponses, clientID string, filename string) {
+	t.Helper()
+
+	// Create multipart form
+	var buf bytes.Buffer
+	writer := multipart.NewWriter(&buf)
+
+	part, err := writer.CreateFormFile("file", filename)
+	require.NoError(t, err, "Failed to create form file")
+
+	pdfBytes := []byte(pdfContent("Single root document"))
+	_, err = part.Write(pdfBytes)
+	require.NoError(t, err, "Failed to write PDF content")
+
+	err = writer.Close()
+	require.NoError(t, err, "Failed to close multipart writer")
+
+	// Upload using the generated client
+	res, err := client.UploadDocumentWithBodyWithResponse(
+		t.Context(),
+		clientID,
+		writer.FormDataContentType(),
+		bytes.NewReader(buf.Bytes()),
+	)
+	require.NoError(t, err, "Failed to upload single document")
+	require.Equal(t, http.StatusOK, res.StatusCode(), "Expected 200 response for single document upload")
+}
+
+// waitForDocumentCount polls the documents endpoint until the expected count is reached or timeout.
+func waitForDocumentCount(t *testing.T, client *queryapi.ClientWithResponses, clientID string, expectedCount int, documents *[]queryapi.DocumentSummary) {
+	t.Helper()
+
+	timeout := time.After(60 * time.Second)
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-timeout:
+			t.Fatalf("Timeout waiting for %d documents, got %d", expectedCount, len(*documents))
+		case <-ticker.C:
+			res, err := client.ListDocumentsByClientIdWithResponse(t.Context(), clientID)
+			if err != nil {
+				continue
+			}
+			if res.JSON200 == nil {
+				continue
+			}
+			*documents = *res.JSON200
+			if len(*documents) >= expectedCount {
+				return
+			}
+		}
+	}
+}
+
+// TestFolderAPI is a more detailed test that also verifies folder creation
+// and folder-based document retrieval.
+func TestFolderAPI(t *testing.T) {
+	t.Run("create_folders", func(t *testing.T) {
+		cfg := &TextExtractionConfig{}
+
+		// Create full network
+		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
+		defer cleanup()
+
+		// Create test client
+		clientName := fmt.Sprintf("FolderTestClient_%d", time.Now().UnixNano())
+		createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
+			Name: clientName,
+			Id:   fmt.Sprintf("folder-test-%d", time.Now().UnixNano()),
+		})
+		require.NoError(t, err)
+		if createRes.JSON400 != nil {
+			t.Fatalf("Got 400 error creating client: %s", createRes.JSON400.Message)
+		}
+		if createRes.JSON201 == nil {
+			t.Fatalf("Expected 201 response for client creation, got status %d with body: %s", createRes.StatusCode(), string(createRes.Body))
+		}
+		clientID := createRes.JSON201.Id
+		fmt.Printf("Created client: %s\n", clientID)
+
+		// Enable sync
+		canSync := true
+		_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
+			CanSync: &canSync,
+		})
+		require.NoError(t, err)
+
+		// Create folders manually to test folder API
+		fmt.Println("\n=== Creating folders via API ===")
+
+		// Create root folder1
+		folder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/folder1",
+			ClientId:  clientID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create folder1")
+		if folder1Res.JSON400 != nil {
+			t.Fatalf("Got 400 error creating folder1: %s", folder1Res.JSON400.Message)
+		}
+		if folder1Res.JSON201 == nil {
+			t.Fatalf("Expected 201 for folder1, got %d with body: %s", folder1Res.StatusCode(), string(folder1Res.Body))
+		}
+		folder1ID := folder1Res.JSON201.Id
+		fmt.Printf("  Created folder1: ID=%s, Path=%s\n", folder1ID, folder1Res.JSON201.Path)
+
+		// Create subfolder2 under folder1
+		subfolder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/folder1/subfolder2",
+			ClientId:  clientID,
+			ParentId:  &folder1ID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create subfolder2")
+		require.NotNil(t, subfolder2Res.JSON201, "Expected 201 for subfolder2, got %d", subfolder2Res.StatusCode())
+		subfolder2ID := subfolder2Res.JSON201.Id
+		fmt.Printf("  Created subfolder2: ID=%s, Path=%s\n", subfolder2ID, subfolder2Res.JSON201.Path)
+
+		// Create root folder2
+		folder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/folder2",
+			ClientId:  clientID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create folder2")
+		require.NotNil(t, folder2Res.JSON201, "Expected 201 for folder2, got %d", folder2Res.StatusCode())
+		folder2ID := folder2Res.JSON201.Id
+		fmt.Printf("  Created folder2: ID=%s, Path=%s\n", folder2ID, folder2Res.JSON201.Path)
+
+		// Create subfolder1 under folder2
+		subfolder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/folder2/subfolder1",
+			ClientId:  clientID,
+			ParentId:  &folder2ID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create subfolder1 under folder2")
+		require.NotNil(t, subfolder1Res.JSON201, "Expected 201 for subfolder1, got %d", subfolder1Res.StatusCode())
+		fmt.Printf("  Created subfolder1: ID=%s, Path=%s\n", subfolder1Res.JSON201.Id, subfolder1Res.JSON201.Path)
+
+		// Test folder metrics (should be empty initially)
+		fmt.Println("\n=== Testing folder metrics (empty folders) ===")
+		metricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder1ID)
+		require.NoError(t, err, "Failed to get folder1 metrics")
+		require.NotNil(t, metricsRes.JSON200, "Expected 200 for metrics")
+		fmt.Printf("  folder1 metrics: totalDocuments=%d\n", metricsRes.JSON200.TotalDocuments)
+
+		// Test folder documents endpoint (should be empty initially)
+		fmt.Println("\n=== Testing folder documents (empty folders) ===")
+		folderDocsRes, err := net.Client.GetFolderDocumentsWithResponse(t.Context(), subfolder2ID)
+		require.NoError(t, err, "Failed to get subfolder2 documents")
+		require.NotNil(t, folderDocsRes.JSON200, "Expected 200 for folder documents")
+		docCount := 0
+		if folderDocsRes.JSON200.Documents != nil {
+			docCount = len(*folderDocsRes.JSON200.Documents)
+		}
+		fmt.Printf("  subfolder2 documents: count=%d\n", docCount)
+
+		// Test folder rename
+		fmt.Println("\n=== Testing folder rename ===")
+		renameRes, err := net.Client.RenameFolderWithResponse(t.Context(), folder1ID, queryapi.RenameFolderJSONRequestBody{
+			Path: "/folder1_renamed",
+		})
+		require.NoError(t, err, "Failed to rename folder1")
+		// Note: Implementation returns 204 No Content for successful rename
+		require.Equal(t, http.StatusNoContent, renameRes.StatusCode(), "Expected 204 for rename")
+		fmt.Printf("  Renamed folder1 to /folder1_renamed\n")
+
+		fmt.Println("\n=== Folder test complete ===")
+	})
+
+	// TestFolderMetricsSpecific tests that GetFolderMetrics returns metrics for only the
+	// specified folder, not aggregated metrics across all folders.
+	t.Run("folder_metrics_specific", func(t *testing.T) {
+		cfg := &TextExtractionConfig{}
+
+		// Create full network
+		net, cleanup := test.CreateFullNetwork(t, t.Context(), cfg)
+		defer cleanup()
+
+		// Create test client
+		// Note: Client external ID has maxLength of 36, so use Unix seconds instead of nanoseconds
+		clientName := fmt.Sprintf("FolderMetricsTestClient_%d", time.Now().UnixNano())
+		clientExternalID := fmt.Sprintf("fld-metrics-%d", time.Now().Unix())
+		createRes, err := net.Client.CreateClientWithResponse(t.Context(), queryapi.ClientCreate{
+			Name: clientName,
+			Id:   clientExternalID,
+		})
+		require.NoError(t, err)
+		if createRes.JSON400 != nil {
+			t.Fatalf("Got 400 error creating client: %s", createRes.JSON400.Message)
+		}
+		if createRes.JSON201 == nil {
+			t.Fatalf("Expected 201 response for client creation, got status %d with body: %s", createRes.StatusCode(), string(createRes.Body))
+		}
+		clientID := createRes.JSON201.Id
+		fmt.Printf("Created client: %s\n", clientID)
+
+		// Enable sync
+		canSync := true
+		_, err = net.Client.UpdateClientWithResponse(t.Context(), clientID, queryapi.ClientUpdate{
+			CanSync: &canSync,
+		})
+		require.NoError(t, err)
+
+		// Create two separate folders at root level
+		fmt.Println("\n=== Creating folders for metrics test ===")
+
+		folder1Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/metrics_folder1",
+			ClientId:  clientID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create metrics_folder1")
+		require.NotNil(t, folder1Res.JSON201, "Expected 201 for metrics_folder1, got %d", folder1Res.StatusCode())
+		folder1ID := folder1Res.JSON201.Id
+		fmt.Printf("  Created metrics_folder1: ID=%s\n", folder1ID)
+
+		folder2Res, err := net.Client.CreateFolderWithResponse(t.Context(), queryapi.CreateFolderJSONRequestBody{
+			Path:      "/metrics_folder2",
+			ClientId:  clientID,
+			CreatedBy: "test@example.com",
+		})
+		require.NoError(t, err, "Failed to create metrics_folder2")
+		require.NotNil(t, folder2Res.JSON201, "Expected 201 for metrics_folder2, got %d", folder2Res.StatusCode())
+		folder2ID := folder2Res.JSON201.Id
+		fmt.Printf("  Created metrics_folder2: ID=%s\n", folder2ID)
+
+		// Get metrics for folder1 - should return metrics specific to folder1 only
+		fmt.Println("\n=== Testing specific folder metrics for folder1 ===")
+		metrics1Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder1ID)
+		require.NoError(t, err, "Failed to get metrics for folder1")
+		require.NotNil(t, metrics1Res.JSON200, "Expected 200 for folder1 metrics, got %d", metrics1Res.StatusCode())
+
+		// Verify the returned folderId matches the requested folder
+		assert.Equal(t, folder1ID, metrics1Res.JSON200.FolderId,
+			"Metrics folderId should match the requested folder1 ID")
+		fmt.Printf("  folder1 metrics: folderId=%s, totalDocuments=%d\n",
+			metrics1Res.JSON200.FolderId, metrics1Res.JSON200.TotalDocuments)
+
+		// Get metrics for folder2 - should return metrics specific to folder2 only
+		fmt.Println("\n=== Testing specific folder metrics for folder2 ===")
+		metrics2Res, err := net.Client.GetFolderMetricsWithResponse(t.Context(), folder2ID)
+		require.NoError(t, err, "Failed to get metrics for folder2")
+		require.NotNil(t, metrics2Res.JSON200, "Expected 200 for folder2 metrics, got %d", metrics2Res.StatusCode())
+
+		// Verify the returned folderId matches the requested folder
+		assert.Equal(t, folder2ID, metrics2Res.JSON200.FolderId,
+			"Metrics folderId should match the requested folder2 ID")
+		fmt.Printf("  folder2 metrics: folderId=%s, totalDocuments=%d\n",
+			metrics2Res.JSON200.FolderId, metrics2Res.JSON200.TotalDocuments)
+
+		// Verify that folder1 and folder2 metrics are independent (different folder IDs returned)
+		assert.NotEqual(t, metrics1Res.JSON200.FolderId, metrics2Res.JSON200.FolderId,
+			"Each folder should have its own distinct metrics with its own folderId")
+
+		// Get the root folder and verify its metrics separately
+		fmt.Println("\n=== Testing root folder metrics ===")
+		foldersRes, err := net.Client.ListClientFoldersWithResponse(t.Context(), queryapi.ClientID(clientID))
+		require.NoError(t, err, "Failed to list client folders")
+		require.NotNil(t, foldersRes.JSON200, "Expected 200 for folder list")
+
+		var rootFolderID openapi_types.UUID
+		for _, folder := range foldersRes.JSON200.Folders {
+			if folder.Path == "/" {
+				rootFolderID = folder.Id
+				break
+			}
+		}
+
+		if rootFolderID != (openapi_types.UUID{}) {
+			rootMetricsRes, err := net.Client.GetFolderMetricsWithResponse(t.Context(), rootFolderID)
+			require.NoError(t, err, "Failed to get root folder metrics")
+			require.NotNil(t, rootMetricsRes.JSON200, "Expected 200 for root folder metrics")
+
+			assert.Equal(t, rootFolderID, rootMetricsRes.JSON200.FolderId,
+				"Root folder metrics folderId should match root folder ID")
+			fmt.Printf("  root folder metrics: folderId=%s, totalDocuments=%d\n",
+				rootMetricsRes.JSON200.FolderId, rootMetricsRes.JSON200.TotalDocuments)
+
+			// Verify root folder metrics are distinct from folder1 and folder2
+			assert.NotEqual(t, rootFolderID, folder1ID, "Root folder should be different from folder1")
+			assert.NotEqual(t, rootFolderID, folder2ID, "Root folder should be different from folder2")
+		} else {
+			fmt.Println("  Note: Root folder not found in folder list")
+		}
+
+		fmt.Println("\n=== Folder metrics specific test complete ===")
+	})
+}
+
+// ptr returns a pointer to the given string value.
+// Helper for creating pointers to string literals.
+func ptr(s string) *string {
+	return &s
+}
+
+// Ensure io import is used
+var _ = io.EOF