#!/bin/bash
# =============================================================================
# text_extraction_integration_test_auth.sh
# =============================================================================
#
# DESCRIPTION:
#   This script performs end-to-end integration testing of the Query Orchestration
#   API with AUTHENTICATION ENABLED. It executes the same REST API calls as the
#   non-auth version but requires the user to login via Cognito first.
#
#   The script:
#   1. Opens a browser to the /login endpoint (redirects to Cognito)
#   2. Starts a local HTTP server to capture the auth callback
#   3. User completes Cognito login with MFA in the browser
#   4. Script captures the auth_token from the callback
#   5. All API calls are made with Authorization: Bearer <token>
#
# USAGE:
#   ./text_extraction_integration_test_auth.sh <base_url> [callback_port]
#
# EXAMPLE:
#   ./text_extraction_integration_test_auth.sh http://localhost:8080
#   ./text_extraction_integration_test_auth.sh http://localhost:8080 8888
#   ./text_extraction_integration_test_auth.sh http://queryo-query-q0pz6j2syaox-1001614225.us-east-2.elb.amazonaws.com
#
# REQUIREMENTS:
#   - bash 4.0+
#   - curl
#   - jq (for JSON parsing)
#   - zip
#   - python3 (for local HTTP server to capture callback)
#   - A web browser (for Cognito login)
#
# AUTHENTICATION FLOW:
#   ┌─────────────────────────────────────────────────────────────────────────┐
#   │ 1. Script opens browser to BASE_URL/login                               │
#   │ 2. Browser redirects to Cognito hosted UI                               │
#   │ 3. User enters credentials and completes MFA                            │
#   │ 4. Cognito redirects to BASE_URL/login-callback with auth code          │
#   │ 5. Server exchanges code for tokens, sets cookies, redirects to our     │
#   │    local capture server with the auth_token                             │
#   │ 6. Script extracts token and uses it for all subsequent API calls       │
#   └─────────────────────────────────────────────────────────────────────────┘
#
# API OPERATIONS PERFORMED (in order):
#   ┌─────────────────────────────────────────────────────────────────────────┐
#   │ PHASE 0: AUTHENTICATION                                                  │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │  Step 0.1: Start local callback server                                  │
#   │  Step 0.2: Open browser to /login                                       │
#   │  Step 0.3: Wait for user to complete login                              │
#   │  Step 0.4: Capture auth_token from callback                             │
#   │  Step 0.5: Verify token via GET /identity                               │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │ PHASE 1: CLIENT SETUP                                                   │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │  Step 1:   POST   /client                      Create new client        │
#   │  Step 2:   GET    /client/{id}                 Verify client exists     │
#   │  Step 2.5: GET    /clients                     List all clients         │
#   │  Step 3:   PATCH  /client/{id}                 Update client (can_sync) │
#   │  Step 4:   GET    /client/{id}/status          Get client sync status   │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │ PHASE 2: DOCUMENT UPLOAD                                                │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │  Step 5:   (local) Create ZIP with 3 PDFs in folder structure           │
#   │  Step 6:   POST   /client/{id}/document/batch  Upload ZIP batch         │
#   │  Step 7:   GET    /client/{id}/document/batch/{batchId}                 │
#   │                                                Poll batch status        │
#   │  Step 8:   GET    /client/{id}/document        List documents (expect 3)│
#   │  Step 9-10: (skipped) Folder-based document retrieval                   │
#   │  Step 11:  POST   /client/{id}/document        Upload single PDF        │
#   │  Step 12:  GET    /client/{id}/document        Verify total docs (4)    │
#   │  Step 12.5: GET   /document/{docId}            Verify file sizes        │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │ PHASE 3: LABEL OPERATIONS                                               │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │  Step 13-14: POST /documents/{docId}/labels    Apply labels (x5):       │
#   │              GET  /documents/{docId}/labels    - Ingested               │
#   │                                                - OCR_Processed          │
#   │                                                - GenAI_Processed        │
#   │                                                - Doczy_AI_Completed     │
#   │                                                - Dashboard_Ready        │
#   │  Step 15:  GET    /client/{id}/folders?metrics=true                     │
#   │                                                List folders w/ metrics  │
#   │  Step 16:  GET    /folders/{folderId}/metrics  Get metrics per folder   │
#   │  Step 16.5: GET   /folders/{folderId}/documents Verify file sizes       │
#   │  Step 17:  GET    /labels/{label}/documents?clientId={id}               │
#   │                                                Get docs by label        │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │ PHASE 4: FIELD EXTRACTIONS                                              │
#   ├─────────────────────────────────────────────────────────────────────────┤
#   │  Step 18:  GET    /field-extractions?documentId={docId}                 │
#   │                                                Verify none exist        │
#   │  Step 19:  POST   /field-extractions           Create extraction v1     │
#   │  Step 20:  GET    /field-extractions/history?documentId={docId}         │
#   │                                                Verify 1 version         │
#   │  Step 21:  POST   /field-extractions           Create extraction v2     │
#   │  Step 22:  GET    /field-extractions/history?documentId={docId}         │
#   │                                                Verify 2 versions        │
#   │  Step 23:  GET    /field-extractions/version?documentId={docId}&version=N│
#   │                                                Retrieve v1, v2, v999    │
#   └─────────────────────────────────────────────────────────────────────────┘
#
# EXIT CODES:
#   0 - All tests passed
#   1 - Missing dependencies or invalid arguments
#   2 - API call failed
#   3 - Authentication failed
#
# =============================================================================

set -e  # Exit on first error

# =============================================================================
# CONFIGURATION
# =============================================================================

# Colors for output (disable if not a terminal)
if [[ -t 1 ]]; then
    RED='\033[0;31m'
    GREEN='\033[0;32m'
    YELLOW='\033[1;33m'
    BLUE='\033[0;34m'
    CYAN='\033[0;36m'
    NC='\033[0m' # No Color
else
    RED=''
    GREEN=''
    YELLOW=''
    BLUE=''
    CYAN=''
    NC=''
fi

# Generate unique identifiers for this test run
TIMESTAMP=$(date +%s)
CLIENT_NAME="IntegrationTestClient_${TIMESTAMP}"
CLIENT_EXTERNAL_ID="int-test-${TIMESTAMP}"

# Variables to store IDs across test steps
CLIENT_ID=""
BATCH_ID=""
DOCUMENT_IDS=()
TARGET_DOC_ID=""
FOLDER_IDS=()

# Authentication variables
AUTH_TOKEN=""
CALLBACK_PORT="${2:-8888}"  # Default to port 8888 for callback
TOKEN_FILE=""
CALLBACK_PID=""

# =============================================================================
# HELPER FUNCTIONS
# =============================================================================

# print_header - Prints a formatted section header
# Parameters:
#   $1 - Step number
#   $2 - Description
print_header() {
    local step="$1"
    local desc="$2"
    echo ""
    echo -e "${BLUE}=== Step ${step}: ${desc} ===${NC}"
}

# print_success - Prints a success message
# Parameters:
#   $1 - Message
print_success() {
    echo -e "${GREEN}✓ $1${NC}"
}

# print_error - Prints an error message and exits
# Parameters:
#   $1 - Message
print_error() {
    echo -e "${RED}✗ ERROR: $1${NC}"
    cleanup_callback_server
    exit 2
}

# print_auth_error - Prints an authentication error message and exits
# Parameters:
#   $1 - Message
print_auth_error() {
    echo -e "${RED}✗ AUTH ERROR: $1${NC}"
    cleanup_callback_server
    exit 3
}

# print_warning - Prints a warning message
# Parameters:
#   $1 - Message
print_warning() {
    echo -e "${YELLOW}⚠ WARNING: $1${NC}"
}

# print_info - Prints an informational message
# Parameters:
#   $1 - Message
print_info() {
    echo -e "  $1"
}

# cleanup_callback_server - Stops the callback server if running
cleanup_callback_server() {
    if [[ -n "$CALLBACK_PID" ]]; then
        kill "$CALLBACK_PID" 2>/dev/null || true
        wait "$CALLBACK_PID" 2>/dev/null || true
    fi
    if [[ -n "$TOKEN_FILE" && -f "$TOKEN_FILE" ]]; then
        rm -f "$TOKEN_FILE"
    fi
}

# Trap to ensure cleanup on exit
trap cleanup_callback_server EXIT

# check_response - Checks if the HTTP response code indicates success
# Parameters:
#   $1 - Expected HTTP status code(s) (comma-separated, e.g., "200,201")
#   $2 - Actual HTTP status code
#   $3 - Response body (for error messages)
#   $4 - Endpoint description
check_response() {
    local expected="$1"
    local actual="$2"
    local body="$3"
    local desc="$4"

    # Check if actual code is in expected list
    IFS=',' read -ra CODES <<< "$expected"
    for code in "${CODES[@]}"; do
        if [[ "$actual" == "$code" ]]; then
            return 0
        fi
    done

    # Check for authentication errors
    if [[ "$actual" == "401" ]]; then
        echo -e "${RED}✗ AUTHENTICATION FAILED: $desc${NC}"
        echo "  Token may be expired or invalid"
        echo "  Response: $body"
        exit 3
    fi

    if [[ "$actual" == "403" ]]; then
        echo -e "${RED}✗ AUTHORIZATION FAILED: $desc${NC}"
        echo "  User does not have permission for this operation"
        echo "  Response: $body"
        exit 3
    fi

    echo -e "${RED}✗ FAILED: $desc${NC}"
    echo "  Expected: $expected, Got: $actual"
    echo "  Response: $body"
    exit 2
}

# api_call - Makes an authenticated API call and captures response
# Parameters:
#   $1 - HTTP method (GET, POST, PATCH, DELETE)
#   $2 - Endpoint path (e.g., "/client")
#   $3 - Request body (optional, empty string for none)
#   $4 - Content-Type header (optional, defaults to application/json)
# Returns:
#   Sets global variables: HTTP_CODE, RESPONSE_BODY
api_call() {
    local method="$1"
    local endpoint="$2"
    local body="$3"
    local content_type="${4:-application/json}"

    local url="${BASE_URL}${endpoint}"
    local curl_opts=(-s -w "\n%{http_code}")

    curl_opts+=(-X "$method")

    # Add Authorization header with Bearer token
    if [[ -n "$AUTH_TOKEN" ]]; then
        curl_opts+=(-H "Authorization: Bearer ${AUTH_TOKEN}")
    fi

    if [[ -n "$body" ]]; then
        curl_opts+=(-H "Content-Type: $content_type")
        curl_opts+=(-d "$body")
    fi

    local response
    local retries=3
    local delay=1

    # Retry loop for rate limiting (429 errors)
    for ((i=1; i<=retries; i++)); do
        response=$(curl "${curl_opts[@]}" "$url")

        # Split response into body and status code
        HTTP_CODE=$(echo "$response" | tail -n1)
        RESPONSE_BODY=$(echo "$response" | sed '$d')

        # If not rate limited, break out of retry loop
        if [[ "$HTTP_CODE" != "429" ]]; then
            break
        fi

        # Rate limited - wait and retry
        if [[ $i -lt $retries ]]; then
            print_info "Rate limited (429), waiting ${delay}s before retry $((i+1))/$retries..."
            sleep "$delay"
            delay=$((delay * 2))  # Exponential backoff
        fi
    done

    # Small delay between all API calls to avoid rate limiting
    sleep 0.1
}

# api_upload - Uploads a file using multipart/form-data with authentication
# Parameters:
#   $1 - Endpoint path
#   $2 - File path
#   $3 - Form field name (e.g., "archive" or "file")
# Returns:
#   Sets global variables: HTTP_CODE, RESPONSE_BODY
api_upload() {
    local endpoint="$1"
    local file_path="$2"
    local field_name="$3"

    local url="${BASE_URL}${endpoint}"

    local response
    response=$(curl -s -w "\n%{http_code}" -X POST \
        -H "Authorization: Bearer ${AUTH_TOKEN}" \
        -F "${field_name}=@${file_path}" \
        "$url")

    HTTP_CODE=$(echo "$response" | tail -n1)
    RESPONSE_BODY=$(echo "$response" | sed '$d')
}

# create_test_pdf - Creates a minimal valid PDF file with unique content
# Parameters:
#   $1 - Output file path
#   $2 - Text content to embed (used to make each PDF unique)
create_test_pdf() {
    local output="$1"
    local text="$2"

    # Calculate stream length based on the text content
    # The stream content will be: "BT\n/F1 12 Tf\n100 700 Td\n($text) Tj\nET"
    # We need to account for variable text length
    local stream_content="BT
/F1 12 Tf
100 700 Td
(${text}) Tj
ET"
    local stream_length=${#stream_content}

    cat > "$output" << PDFEOF
%PDF-1.4
%âãÏÓ
1 0 obj
<<
  /Type /Catalog
  /Pages 2 0 R
>>
endobj
2 0 obj
<<
  /Type /Pages
  /Kids [3 0 R]
  /Count 1
>>
endobj
3 0 obj
<<
  /Type /Page
  /Parent 2 0 R
  /MediaBox [0 0 612 792]
  /Resources <<
    /Font <<
      /F1 <<
        /Type /Font
        /Subtype /Type1
        /BaseFont /Helvetica
      >>
    >>
  >>
  /Contents 4 0 R
>>
endobj
4 0 obj
<<
  /Length ${stream_length}
>>
stream
${stream_content}
endstream
endobj
xref
0 5
0000000000 65535 f
0000000015 00000 n
0000000066 00000 n
0000000125 00000 n
0000000330 00000 n
trailer
<<
  /Size 5
  /Root 1 0 R
>>
startxref
430
%%EOF
PDFEOF
}

# create_test_zip - Creates a ZIP with test PDF files
# Parameters:
#   $1 - Output ZIP file path
create_test_zip() {
    local output="$1"
    local temp_dir
    temp_dir=$(mktemp -d)

    # Create folder structure
    mkdir -p "${temp_dir}/folder1/subfolder2"
    mkdir -p "${temp_dir}/folder2/subfolder1"

    # Create PDF files with unique content
    # IMPORTANT: Each PDF must have unique content to avoid deduplication by hash.
    # The system deduplicates documents based on their hash - identical files become
    # a single document. Using timestamps and UUIDs ensures uniqueness.
    local ts=$(date +%s%N)  # Nanosecond timestamp for uniqueness
    create_test_pdf "${temp_dir}/folder1/subfolder2/file1.pdf" "Doc1-${ts}-folder1-subfolder2-AAAA"
    create_test_pdf "${temp_dir}/folder1/subfolder2/file2.pdf" "Doc2-${ts}-folder1-subfolder2-BBBB"
    create_test_pdf "${temp_dir}/folder2/subfolder1/file3.pdf" "Doc3-${ts}-folder2-subfolder1-CCCC"

    # Create ZIP (using relative paths, output must be absolute path)
    local abs_output
    if [[ "$output" = /* ]]; then
        abs_output="$output"
    else
        abs_output="$(pwd)/$output"
    fi
    (cd "$temp_dir" && zip -r "$abs_output" folder1 folder2)

    # Cleanup
    rm -rf "$temp_dir"
}

# =============================================================================
# VALIDATION
# =============================================================================

# Check for required tools
check_dependencies() {
    local missing=()

    if ! command -v curl &> /dev/null; then
        missing+=("curl")
    fi

    if ! command -v jq &> /dev/null; then
        missing+=("jq")
    fi

    if ! command -v zip &> /dev/null; then
        missing+=("zip")
    fi

    if ! command -v python3 &> /dev/null; then
        missing+=("python3 (needed for callback server)")
    fi

    if [[ ${#missing[@]} -gt 0 ]]; then
        echo -e "${RED}ERROR: Missing required tools: ${missing[*]}${NC}"
        echo "Please install the missing tools and try again."
        exit 1
    fi
}

# Validate command line arguments
validate_args() {
    if [[ $# -lt 1 ]]; then
        echo "Usage: $0 <base_url> [callback_port]"
        echo ""
        echo "Arguments:"
        echo "  base_url      - The URL of the Query Orchestration API"
        echo "  callback_port - Port for local callback server (default: 8888)"
        echo ""
        echo "Examples:"
        echo "  $0 http://localhost:8080"
        echo "  $0 http://localhost:8080 9999"
        echo "  $0 http://queryo-query-q0pz6j2syaox-1001614225.us-east-2.elb.amazonaws.com"
        exit 1
    fi

    BASE_URL="${1%/}"  # Remove trailing slash if present

    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}Text Extraction Integration Test${NC}"
    echo -e "${BLUE}(WITH AUTHENTICATION)${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo ""
    echo "Base URL: ${BASE_URL}"
    echo "Callback Port: ${CALLBACK_PORT}"
    echo "Client Name: ${CLIENT_NAME}"
    echo "Client ID: ${CLIENT_EXTERNAL_ID}"
    echo ""
}

# =============================================================================
# PHASE 0: AUTHENTICATION
# =============================================================================

# Start a simple Python HTTP server to capture the callback token
# The server will capture the auth_token from the callback and write it to a file
start_callback_server() {
    TOKEN_FILE=$(mktemp)

    # Python script for the callback server
    local python_script
    python_script=$(cat << 'PYEOF'
import http.server
import socketserver
import urllib.parse
import sys
import os

PORT = int(sys.argv[1])
TOKEN_FILE = sys.argv[2]

class CallbackHandler(http.server.BaseHTTPRequestHandler):
    def log_message(self, format, *args):
        pass  # Suppress logging

    def do_GET(self):
        parsed = urllib.parse.urlparse(self.path)
        params = urllib.parse.parse_qs(parsed.query)

        # Check if this is a callback with token
        if 'token' in params:
            token = params['token'][0]
            with open(TOKEN_FILE, 'w') as f:
                f.write(token)

            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            response = '''
            <html>
            <head><title>Authentication Successful</title></head>
            <body style="font-family: Arial, sans-serif; text-align: center; padding: 50px;">
                <h1 style="color: green;">Authentication Successful!</h1>
                <p>You can close this browser window and return to the terminal.</p>
                <p>The integration test will continue automatically.</p>
            </body>
            </html>
            '''
            self.wfile.write(response.encode())

            # Signal to shutdown after handling request
            def shutdown():
                self.server.shutdown()
            import threading
            threading.Thread(target=shutdown).start()
        else:
            # Show a page asking user to complete login
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            response = '''
            <html>
            <head><title>Waiting for Authentication</title></head>
            <body style="font-family: Arial, sans-serif; text-align: center; padding: 50px;">
                <h1>Waiting for Authentication</h1>
                <p>Please complete the login process in the other browser window.</p>
            </body>
            </html>
            '''
            self.wfile.write(response.encode())

with socketserver.TCPServer(("", PORT), CallbackHandler) as httpd:
    httpd.handle_request()  # Handle one request then exit
PYEOF
)

    # Start the server in the background
    python3 -c "$python_script" "$CALLBACK_PORT" "$TOKEN_FILE" &
    CALLBACK_PID=$!

    # Give the server a moment to start
    sleep 0.5
}

# Open the browser to the login URL
# On macOS use 'open', on Linux use 'xdg-open'
open_browser() {
    local url="$1"

    if [[ "$(uname)" == "Darwin" ]]; then
        open "$url"
    elif command -v xdg-open &> /dev/null; then
        xdg-open "$url"
    elif command -v gnome-open &> /dev/null; then
        gnome-open "$url"
    else
        print_warning "Could not detect browser. Please manually open: $url"
        return 1
    fi

    return 0
}

# read_long_token - Reads a long JWT token from user input
# Handles very long tokens (1000+ chars) that standard read can't handle
# Sets global variable: AUTH_TOKEN
read_long_token() {
    echo "Options for entering your token:"

    # Check if pbpaste is available (macOS)
    if command -v pbpaste &> /dev/null; then
        echo "  1. Read from clipboard (copy token first, then press Enter)"
        echo "  2. Read from a file"
        echo ""
        read -p "Choose option [1]: " input_method
        input_method="${input_method:-1}"

        case "$input_method" in
            1)
                echo ""
                echo "Copy your auth_token to the clipboard, then press Enter..."
                read -p ""

                # Read from clipboard
                AUTH_TOKEN=$(pbpaste | tr -d '[:space:]')
                ;;
            2)
                read_token_from_file
                ;;
            *)
                print_auth_error "Invalid option"
                ;;
        esac
    else
        # Non-macOS: only offer file-based input
        echo "  1. Read from a file"
        echo ""
        echo "Note: Due to terminal buffer limits, please save your token to a file."
        echo ""
        read -p "Press Enter to continue: "
        read_token_from_file
    fi

    if [[ -z "$AUTH_TOKEN" ]]; then
        print_auth_error "No token provided"
    fi

    # Validate it looks like a JWT (3 parts separated by dots)
    if [[ ! "$AUTH_TOKEN" =~ ^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]]; then
        print_warning "Token doesn't appear to be a valid JWT format, but continuing anyway..."
    fi

    local token_length=${#AUTH_TOKEN}
    print_success "Token received (${token_length} characters)"
}

# read_token_from_file - Helper to read token from a file path
# Sets global variable: AUTH_TOKEN
read_token_from_file() {
    echo ""
    echo "Save your token to a file, then enter the path."
    echo "Example: echo 'your_token_here' > /tmp/token.txt"
    echo ""
    read -p "Enter path to file containing the token: " token_path

    if [[ ! -f "$token_path" ]]; then
        print_auth_error "File not found: $token_path"
    fi

    # Read token from file, removing all whitespace
    AUTH_TOKEN=$(tr -d '[:space:]' < "$token_path")
}

# Perform the authentication flow
# This is an interactive process that requires user action
step_00_authenticate() {
    print_header "0" "Authentication"

    echo ""
    echo -e "${CYAN}═══════════════════════════════════════════════════════════════════════${NC}"
    echo -e "${CYAN}  AUTHENTICATION REQUIRED${NC}"
    echo -e "${CYAN}═══════════════════════════════════════════════════════════════════════${NC}"
    echo ""
    echo "This test requires authentication via AWS Cognito."
    echo ""
    echo "The authentication process:"
    echo "  1. A browser window will open to the login page"
    echo "  2. You will be redirected to AWS Cognito"
    echo "  3. Enter your credentials and complete MFA"
    echo "  4. After successful login, you'll be redirected back"
    echo "  5. The script will automatically capture your auth token"
    echo ""

    # Method 1: Try to use the cookie-based flow with manual token extraction
    # Method 2: Use a callback URL to capture the token

    echo -e "${YELLOW}Choose authentication method:${NC}"
    echo "  1. Browser login with token capture (opens browser)"
    echo "  2. Manual token entry (if you already have a token)"
    echo ""
    read -p "Enter choice [1]: " auth_choice
    auth_choice="${auth_choice:-1}"

    case "$auth_choice" in
        1)
            authenticate_via_browser
            ;;
        2)
            authenticate_manual
            ;;
        *)
            print_auth_error "Invalid choice"
            ;;
    esac

    # Verify the token works
    verify_token
}

# Authenticate via browser with callback capture
authenticate_via_browser() {
    echo ""
    print_info "Starting authentication via browser..."
    echo ""

    # The login URL - this will redirect to Cognito
    local login_url="${BASE_URL}/login"

    echo -e "${CYAN}Login URL: ${login_url}${NC}"
    echo ""
    echo "Opening browser to login page..."
    echo ""

    # Try to open the browser
    if ! open_browser "$login_url"; then
        echo ""
        echo "Please open the following URL in your browser:"
        echo -e "${CYAN}${login_url}${NC}"
        echo ""
    fi

    echo ""
    echo "Complete the login process in your browser."
    echo "After successful login, you will need to provide the auth token."
    echo ""
    echo -e "${YELLOW}To get the auth token:${NC}"
    echo "  1. After logging in, open browser Developer Tools (F12)"
    echo "  2. Go to Application/Storage -> Cookies"
    echo "  3. Find the 'auth_token' cookie"
    echo "  4. Copy the entire cookie value (it's a long JWT string)"
    echo ""
    echo "Alternatively, if you're redirected to a page showing the token,"
    echo "copy the token value shown."
    echo ""

    read_long_token
}

# Manual token entry for users who already have a token
authenticate_manual() {
    echo ""
    echo "Manual token entry selected."
    echo ""
    echo "Enter your JWT access token (auth_token)."
    echo "This should be the token you received after logging in."
    echo ""

    read_long_token
}

# Verify the token by calling the /identity endpoint
verify_token() {
    print_info ""
    print_info "Verifying token via GET /identity..."

    api_call "GET" "/identity" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        print_success "Token verified successfully!"
        echo ""
        echo -e "${CYAN}=== /identity Response ===${NC}"
        echo "$RESPONSE_BODY" | jq .
        echo -e "${CYAN}==========================${NC}"
        echo ""

        # Summary of roles and permissions
        print_info "Summary:"
        local role_count
        role_count=$(echo "$RESPONSE_BODY" | jq '.roles | length' 2>/dev/null || echo "0")
        print_info "  Total roles: $role_count"

        echo "$RESPONSE_BODY" | jq -r '.roles[]? | "  - \(.name): \(.permissions | length) permissions"' 2>/dev/null

    elif [[ "$HTTP_CODE" == "401" ]]; then
        print_auth_error "Token is invalid or expired. Please try again with a fresh token."
    else
        print_warning "Could not verify token (HTTP $HTTP_CODE), but continuing..."
        print_info "Response: $RESPONSE_BODY"
    fi

    echo ""
    echo -e "${GREEN}Authentication complete! Starting integration tests...${NC}"
    echo ""
}

# =============================================================================
# PHASE 1: CLIENT SETUP (Steps 1-4)
# =============================================================================

# Step 1: Create a new client via POST /client
step_01_create_client() {
    print_header "1" "Create a new client via POST /client"

    local body
    body=$(cat << EOF
{
    "id": "${CLIENT_EXTERNAL_ID}",
    "name": "${CLIENT_NAME}"
}
EOF
)

    print_info "Request: POST /client"
    print_info "Body: $body"

    api_call "POST" "/client" "$body"
    check_response "201" "$HTTP_CODE" "$RESPONSE_BODY" "Create client"

    CLIENT_ID=$(echo "$RESPONSE_BODY" | jq -r '.id')
    print_success "Client created successfully"
    print_info "Client ID: $CLIENT_ID"
}

# Step 2: Verify client exists via GET /client/{id}
step_02_verify_client() {
    print_header "2" "Verify client exists via GET /client/{id}"

    print_info "Request: GET /client/${CLIENT_ID}"

    api_call "GET" "/client/${CLIENT_ID}" ""
    check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get client"

    local name
    name=$(echo "$RESPONSE_BODY" | jq -r '.name')
    local can_sync
    can_sync=$(echo "$RESPONSE_BODY" | jq -r '.can_sync')

    print_success "Client verified successfully"
    print_info "Name: $name"
    print_info "Can Sync: $can_sync"
}

# Step 2.5: List all clients via GET /clients
step_02_5_list_clients() {
    print_header "2.5" "List all clients via GET /clients"

    print_info "Request: GET /clients"

    api_call "GET" "/clients" ""
    check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "List clients"

    local total_count
    total_count=$(echo "$RESPONSE_BODY" | jq -r '.totalCount')

    print_success "Clients listed successfully"
    print_info "Total clients in system: $total_count"

    # Verify our client is in the list
    local found
    found=$(echo "$RESPONSE_BODY" | jq -r --arg id "$CLIENT_ID" '.clients[] | select(.id == $id) | .id')

    if [[ "$found" == "$CLIENT_ID" ]]; then
        print_success "Newly created client found in list"
    else
        print_warning "Newly created client not found in list"
    fi

    # Print all clients
    echo "$RESPONSE_BODY" | jq -r '.clients[] | "  - ID: \(.id), Name: \(.name), CanSync: \(.can_sync)"'
}

# Step 3: Update client to allow sync via PATCH /client/{id}
step_03_update_client() {
    print_header "3" "Update client to allow sync via PATCH /client/{id}"

    local body='{"can_sync": true}'

    print_info "Request: PATCH /client/${CLIENT_ID}"
    print_info "Body: $body"

    api_call "PATCH" "/client/${CLIENT_ID}" "$body"
    check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Update client"

    print_success "Client updated: can_sync = true"
}

# Step 4: Confirm client status via GET /client/{id}/status
step_04_confirm_status() {
    print_header "4" "Confirm client status via GET /client/{id}/status"

    print_info "Request: GET /client/${CLIENT_ID}/status"

    api_call "GET" "/client/${CLIENT_ID}/status" ""
    check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get client status"

    local status
    status=$(echo "$RESPONSE_BODY" | jq -r '.status')

    print_success "Client status retrieved"
    print_info "Status: $status"
}

# =============================================================================
# PHASE 2: DOCUMENT UPLOAD (Steps 5-12)
# =============================================================================

# Step 5: Create a ZIP with 3 PDFs in folder structures
step_05_create_zip() {
    print_header "5" "Create a ZIP with 3 PDFs in folder structures"

    ZIP_FILE=$(mktemp).zip
    create_test_zip "$ZIP_FILE"

    print_success "ZIP file created: $ZIP_FILE"
    print_info "Contents:"
    print_info "  - folder1/subfolder2/file1.pdf"
    print_info "  - folder1/subfolder2/file2.pdf"
    print_info "  - folder2/subfolder1/file1.pdf"

    # Show file size
    local size
    size=$(ls -lh "$ZIP_FILE" | awk '{print $5}')
    print_info "Size: $size"
}

# Step 6: Upload batch via POST /client/{id}/document/batch
step_06_upload_batch() {
    print_header "6" "Upload batch via POST /client/${CLIENT_ID}/document/batch"

    print_info "Request: POST /client/${CLIENT_ID}/document/batch"
    print_info "File: $ZIP_FILE"

    api_upload "/client/${CLIENT_ID}/document/batch" "$ZIP_FILE" "archive"
    check_response "202" "$HTTP_CODE" "$RESPONSE_BODY" "Upload batch"

    BATCH_ID=$(echo "$RESPONSE_BODY" | jq -r '.batch_id')
    local status
    status=$(echo "$RESPONSE_BODY" | jq -r '.status')
    local status_url
    status_url=$(echo "$RESPONSE_BODY" | jq -r '.status_url')

    print_success "Batch upload accepted"
    print_info "Batch ID: $BATCH_ID"
    print_info "Status: $status"
    print_info "Status URL: $status_url"

    # Cleanup temp file
    rm -f "$ZIP_FILE"
}

# Step 7: Verify batch via GET /client/{id}/document/batch/{batch_id}
step_07_verify_batch() {
    print_header "7" "Verify batch via GET /client/${CLIENT_ID}/document/batch/${BATCH_ID}"

    print_info "Request: GET /client/${CLIENT_ID}/document/batch/${BATCH_ID}"

    # Poll for batch completion (max 30 attempts, 1 second apart)
    local attempts=0
    local max_attempts=30
    local batch_status="processing"

    while [[ "$batch_status" == "processing" && $attempts -lt $max_attempts ]]; do
        api_call "GET" "/client/${CLIENT_ID}/document/batch/${BATCH_ID}" ""
        check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Get batch status"

        batch_status=$(echo "$RESPONSE_BODY" | jq -r '.status')
        local processed
        processed=$(echo "$RESPONSE_BODY" | jq -r '.processed_documents')
        local total
        total=$(echo "$RESPONSE_BODY" | jq -r '.total_documents')

        print_info "Attempt $((attempts+1)): status=$batch_status, processed=$processed/$total"

        if [[ "$batch_status" == "processing" ]]; then
            sleep 1
        fi

        ((attempts++))
    done

    if [[ "$batch_status" == "completed" ]]; then
        print_success "Batch processing completed"
    elif [[ "$batch_status" == "failed" ]]; then
        print_warning "Batch processing failed"
        local failed_files
        failed_files=$(echo "$RESPONSE_BODY" | jq -r '.failed_filenames[]?' 2>/dev/null || echo "none")
        print_info "Failed files: $failed_files"
    else
        print_warning "Batch still processing after $max_attempts attempts"
    fi

    # Also list all batches for this client
    print_info ""
    print_info "Listing all batches for client..."
    api_call "GET" "/client/${CLIENT_ID}/document/batch" ""
    local total_batches
    total_batches=$(echo "$RESPONSE_BODY" | jq -r '.total_count')
    print_info "Total batches for client: $total_batches"
}

# Step 8: Get documents via GET /client/{id}/document and verify 3 documents
step_08_get_documents() {
    print_header "8" "Get documents via GET /client/${CLIENT_ID}/document (expect 3 documents)"

    print_info "Request: GET /client/${CLIENT_ID}/document"

    # Poll until we have documents (they may still be processing)
    # Note: In local testing, batch processing may not complete since it requires
    # async queue processing. We'll continue even with 0 documents from batch.
    local attempts=0
    local max_attempts=10
    local doc_count=0

    while [[ $doc_count -lt 3 && $attempts -lt $max_attempts ]]; do
        api_call "GET" "/client/${CLIENT_ID}/document" ""
        check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "List documents"

        doc_count=$(echo "$RESPONSE_BODY" | jq 'length')
        print_info "Attempt $((attempts+1)): Found $doc_count document(s)"

        if [[ $doc_count -lt 3 ]]; then
            sleep 1
        fi

        ((attempts++))
    done

    if [[ $doc_count -ge 3 ]]; then
        print_success "Found expected number of documents"
    else
        print_warning "Expected 3 documents, found $doc_count (batch processing may be async)"
        print_info "Note: In local testing, batch documents process asynchronously via queues"
    fi

    # Store document IDs for later use
    DOCUMENT_IDS=($(echo "$RESPONSE_BODY" | jq -r '.[].id'))

    print_info "Documents:"
    for doc_id in "${DOCUMENT_IDS[@]}"; do
        local hash
        hash=$(echo "$RESPONSE_BODY" | jq -r --arg id "$doc_id" '.[] | select(.id == $id) | .hash')
        local file_size
        file_size=$(echo "$RESPONSE_BODY" | jq -r --arg id "$doc_id" '.[] | select(.id == $id) | .fileSizeBytes // "N/A"')
        print_info "  - ID: $doc_id, Hash: $hash, Size: ${file_size} bytes"
    done

    # Set target document for label operations
    if [[ ${#DOCUMENT_IDS[@]} -gt 0 ]]; then
        TARGET_DOC_ID="${DOCUMENT_IDS[0]}"
        print_info "Target document for label operations: $TARGET_DOC_ID"
    fi
}

# Steps 9-10: Folder-based document retrieval (skipped)
step_09_10_folder_documents() {
    print_header "9-10" "Folder-based document retrieval (SKIPPED)"

    print_info "Note: Folder-based document retrieval requires folder IDs from processing"
    print_info "Skipping folder-specific document retrieval - documents verified at client level"
    print_success "Steps 9-10 skipped as expected"
}

# Step 11: POST a single document to root
step_11_upload_single() {
    print_header "11" "Upload single document to root"

    # Create a single PDF file
    SINGLE_PDF=$(mktemp).pdf
    create_test_pdf "$SINGLE_PDF" "Single root document"

    print_info "Request: POST /client/${CLIENT_ID}/document"
    print_info "File: single_root_document.pdf"

    # Use multipart form upload - the file field must be named "file"
    # and the content type for the file should be application/octet-stream (let curl detect it)
    local url="${BASE_URL}/client/${CLIENT_ID}/document"

    local response
    response=$(curl -s -w "\n%{http_code}" -X POST \
        -H "Authorization: Bearer ${AUTH_TOKEN}" \
        -F "file=@${SINGLE_PDF};type=application/octet-stream;filename=single_root_document.pdf" \
        "$url")

    HTTP_CODE=$(echo "$response" | tail -n1)
    RESPONSE_BODY=$(echo "$response" | sed '$d')

    check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "Upload single document"

    print_success "Single document uploaded successfully"

    # Cleanup
    rm -f "$SINGLE_PDF"
}

# Step 12: Verify documents total (expecting at least 1 from single upload)
step_12_verify_total() {
    print_header "12" "Verify documents total (expecting at least 1)"

    print_info "Request: GET /client/${CLIENT_ID}/document"

    # Poll until we have at least 1 document
    local attempts=0
    local max_attempts=15
    local doc_count=0

    while [[ $doc_count -lt 1 && $attempts -lt $max_attempts ]]; do
        api_call "GET" "/client/${CLIENT_ID}/document" ""
        check_response "200" "$HTTP_CODE" "$RESPONSE_BODY" "List documents"

        doc_count=$(echo "$RESPONSE_BODY" | jq 'length')
        print_info "Attempt $((attempts+1)): Found $doc_count document(s)"

        if [[ $doc_count -lt 1 ]]; then
            sleep 1
        fi

        ((attempts++))
    done

    if [[ $doc_count -ge 4 ]]; then
        print_success "Verified 4+ documents total (batch processing complete)"
    elif [[ $doc_count -ge 1 ]]; then
        print_success "Verified $doc_count document(s) (single upload successful)"
        print_info "Note: Batch documents may still be processing asynchronously"
    else
        print_warning "Expected at least 1 document, found $doc_count"
    fi

    # Update document IDs
    DOCUMENT_IDS=($(echo "$RESPONSE_BODY" | jq -r '.[].id'))

    if [[ ${#DOCUMENT_IDS[@]} -gt 0 ]]; then
        TARGET_DOC_ID="${DOCUMENT_IDS[0]}"
    fi

    print_info "Final document list:"
    for doc_id in "${DOCUMENT_IDS[@]}"; do
        local hash
        hash=$(echo "$RESPONSE_BODY" | jq -r --arg id "$doc_id" '.[] | select(.id == $id) | .hash')
        local file_size
        file_size=$(echo "$RESPONSE_BODY" | jq -r --arg id "$doc_id" '.[] | select(.id == $id) | .fileSizeBytes // "N/A"')
        print_info "  - ID: $doc_id, Hash: $hash, Size: ${file_size} bytes"
    done
}

# Step 12.5: Verify file sizes are present in document details
step_12_5_verify_file_sizes() {
    print_header "12.5" "Verify file sizes are present in document details"

    if [[ ${#DOCUMENT_IDS[@]} -eq 0 ]]; then
        print_warning "No documents available - skipping file size verification"
        return
    fi

    local docs_with_size=0
    local docs_without_size=0

    for doc_id in "${DOCUMENT_IDS[@]}"; do
        print_info ""
        print_info "Request: GET /document/${doc_id}"

        api_call "GET" "/document/${doc_id}" ""

        if [[ "$HTTP_CODE" == "200" ]]; then
            local file_size
            file_size=$(echo "$RESPONSE_BODY" | jq -r '.fileSizeBytes // "null"')
            local filename
            filename=$(echo "$RESPONSE_BODY" | jq -r '.filename // "unknown"')

            if [[ "$file_size" != "null" && "$file_size" != "" ]]; then
                print_success "Document $doc_id has fileSizeBytes: $file_size bytes (filename: $filename)"
                ((docs_with_size++))
            else
                print_warning "Document $doc_id is missing fileSizeBytes (filename: $filename)"
                ((docs_without_size++))
            fi
        else
            print_warning "Failed to get document $doc_id: $HTTP_CODE"
        fi
    done

    print_info ""
    if [[ $docs_with_size -gt 0 ]]; then
        print_success "File size verification: $docs_with_size document(s) have file sizes"
    fi
    if [[ $docs_without_size -gt 0 ]]; then
        print_warning "File size verification: $docs_without_size document(s) missing file sizes"
        print_info "Note: Legacy documents or documents still processing may not have file sizes"
    fi
}

# =============================================================================
# PHASE 3: LABEL OPERATIONS (Steps 13-17)
# =============================================================================

# Step 13-14: Apply all labels one by one and verify after each
step_13_14_apply_labels() {
    print_header "13-14" "Apply and verify labels on document: ${TARGET_DOC_ID}"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping label operations"
        print_info "This may happen if document upload/processing hasn't completed"
        return
    fi

    local labels=("Ingested" "OCR_Processed" "GenAI_Processed" "Doczy_AI_Completed" "Dashboard_Ready")
    local applied_count=0

    for label in "${labels[@]}"; do
        print_info ""
        print_info "Applying label: $label"

        local body
        body=$(cat << EOF
{
    "label": "${label}",
    "appliedBy": "integration-test@example.com"
}
EOF
)

        print_info "Request: POST /documents/${TARGET_DOC_ID}/labels"

        api_call "POST" "/documents/${TARGET_DOC_ID}/labels" "$body"

        if [[ "$HTTP_CODE" == "201" ]]; then
            local record_id
            record_id=$(echo "$RESPONSE_BODY" | jq -r '.id')
            print_success "Label '$label' applied: recordID=$record_id"
            ((applied_count++))
        else
            print_warning "Failed to apply label '$label': $RESPONSE_BODY"
            continue
        fi

        # Verify labels via GET
        print_info "Verifying labels via GET /documents/${TARGET_DOC_ID}/labels"

        api_call "GET" "/documents/${TARGET_DOC_ID}/labels" ""

        if [[ "$HTTP_CODE" == "200" ]]; then
            local label_count
            label_count=$(echo "$RESPONSE_BODY" | jq '.labels | length')
            print_info "  Labels on document: $label_count (expected: $applied_count)"
        else
            print_warning "  Failed to get labels: $HTTP_CODE"
        fi
    done

    print_success "Applied $applied_count labels to document"
}

# Step 15: List all folders for client via GET /clients/{clientId}/folders?metrics=true
step_15_list_folders() {
    print_header "15" "List folders for client via GET /client/${CLIENT_ID}/folders?metrics=true"

    print_info "Request: GET /client/${CLIENT_ID}/folders?metrics=true"

    api_call "GET" "/client/${CLIENT_ID}/folders?metrics=true" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        local folder_count
        folder_count=$(echo "$RESPONSE_BODY" | jq '.folders | length')

        print_success "Folders listed successfully (with metrics)"
        print_info "Total folders: $folder_count"

        if [[ $folder_count -gt 0 ]]; then
            FOLDER_IDS=($(echo "$RESPONSE_BODY" | jq -r '.folders[].id'))

            print_info "Folder hierarchy with inline metrics:"
            echo "$RESPONSE_BODY" | jq -r '.folders[] | "  - \(.path) (ID: \(.id))"'
            echo "$RESPONSE_BODY" | jq -r '.folders[] | "      Metrics: totalDocuments=\(.metrics.totalDocuments // 0), byLabel=\(.metrics.byLabel // {})"'
        else
            print_info "Note: No folders found - documents may be at root level"
        fi
    else
        print_warning "Failed to list folders: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# Step 16: Get metrics for each folder via GET /folders/{folderId}/metrics
step_16_folder_metrics() {
    print_header "16" "Get metrics for each folder"

    if [[ ${#FOLDER_IDS[@]} -eq 0 ]]; then
        print_info "No folders to get metrics for (skipping)"
        return
    fi

    for folder_id in "${FOLDER_IDS[@]}"; do
        print_info ""
        print_info "Request: GET /folders/${folder_id}/metrics"

        api_call "GET" "/folders/${folder_id}/metrics" ""

        if [[ "$HTTP_CODE" == "200" ]]; then
            local total_docs
            total_docs=$(echo "$RESPONSE_BODY" | jq -r '.totalDocuments')
            local by_label
            by_label=$(echo "$RESPONSE_BODY" | jq -c '.byLabel')

            print_success "Folder $folder_id metrics:"
            print_info "  Total documents: $total_docs"
            print_info "  By label: $by_label"
        else
            print_warning "Failed to get metrics for folder $folder_id: $HTTP_CODE"
        fi
    done
}

# Step 16.5: Verify file sizes in folder documents
step_16_5_folder_document_sizes() {
    print_header "16.5" "Verify file sizes in folder document responses"

    if [[ ${#FOLDER_IDS[@]} -eq 0 ]]; then
        print_info "No folders available - skipping folder document file size verification"
        return
    fi

    local total_docs_with_size=0
    local total_docs_without_size=0

    for folder_id in "${FOLDER_IDS[@]}"; do
        print_info ""
        print_info "Request: GET /folders/${folder_id}/documents"

        api_call "GET" "/folders/${folder_id}/documents" ""

        if [[ "$HTTP_CODE" == "200" ]]; then
            local doc_count
            doc_count=$(echo "$RESPONSE_BODY" | jq '.documents | length')

            if [[ $doc_count -gt 0 ]]; then
                print_info "Folder $folder_id has $doc_count document(s)"

                # Check each document for file size
                for ((i=0; i<doc_count; i++)); do
                    local doc_id
                    doc_id=$(echo "$RESPONSE_BODY" | jq -r ".documents[$i].id")
                    local file_size
                    file_size=$(echo "$RESPONSE_BODY" | jq -r ".documents[$i].fileSizeBytes // \"null\"")
                    local filename
                    filename=$(echo "$RESPONSE_BODY" | jq -r ".documents[$i].filename // \"unknown\"")

                    if [[ "$file_size" != "null" && "$file_size" != "" ]]; then
                        print_success "  Document $doc_id: fileSizeBytes=$file_size (filename: $filename)"
                        ((total_docs_with_size++))
                    else
                        print_warning "  Document $doc_id: missing fileSizeBytes (filename: $filename)"
                        ((total_docs_without_size++))
                    fi
                done
            else
                print_info "Folder $folder_id has no documents"
            fi
        else
            print_warning "Failed to get documents for folder $folder_id: $HTTP_CODE"
        fi
    done

    print_info ""
    if [[ $total_docs_with_size -gt 0 ]]; then
        print_success "Folder documents file size check: $total_docs_with_size document(s) have file sizes"
    fi
    if [[ $total_docs_without_size -gt 0 ]]; then
        print_warning "Folder documents file size check: $total_docs_without_size document(s) missing file sizes"
    fi
}

# Step 17: Get documents by label via GET /labels/{labelName}/documents
step_17_documents_by_label() {
    print_header "17" "Get documents by label via GET /labels/{labelName}/documents"

    local label="Ingested"

    print_info "Request: GET /labels/${label}/documents?clientId=${CLIENT_ID}"

    api_call "GET" "/labels/${label}/documents?clientId=${CLIENT_ID}" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        local doc_count
        doc_count=$(echo "$RESPONSE_BODY" | jq '.documents | length')

        print_success "Documents with label '$label': $doc_count"

        if [[ $doc_count -gt 0 ]]; then
            echo "$RESPONSE_BODY" | jq -r '.documents[] | "  - ID: \(.id), Hash: \(.hash), Size: \(.fileSizeBytes // "N/A") bytes"'
        fi
    else
        print_warning "Failed to get documents by label: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# =============================================================================
# PHASE 4: FIELD EXTRACTIONS (Steps 18-23)
# =============================================================================

# Step 18: Verify no field extraction exists for document
step_18_verify_no_extraction() {
    print_header "18" "Verify no field extraction exists"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping field extraction verification"
        return
    fi

    print_info "Request: GET /field-extractions?documentId=${TARGET_DOC_ID}"

    api_call "GET" "/field-extractions?documentId=${TARGET_DOC_ID}" ""

    if [[ "$HTTP_CODE" == "404" ]]; then
        print_success "Confirmed: No field extraction exists for document (404)"
    elif [[ "$HTTP_CODE" == "200" ]]; then
        local version
        version=$(echo "$RESPONSE_BODY" | jq -r '.version')
        print_warning "Field extraction already exists: version=$version"
    else
        print_info "Got status $HTTP_CODE: $RESPONSE_BODY"
    fi
}

# Step 19: Create field extraction via POST /field-extractions
# Creates version 1 with all 18 singleFields and 92 arrayFields populated
step_19_create_extraction() {
    print_header "19" "Create field extraction via POST /field-extractions"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping field extraction creation"
        return
    fi

    local body
    body=$(cat << 'EOF'
{
    "documentId": "PLACEHOLDER_DOC_ID",
    "singleFields": {
        "fileName": "test_document_v1.pdf",
        "contractTitle": "Master Service Agreement v1",
        "aareteDerivedAmendmentNum": 1,
        "clientName": "Acme Healthcare Systems",
        "payerName": "BlueCross BlueShield of California",
        "payerState": "CA",
        "providerState": "NY",
        "filenameTin": "12-3456789",
        "provGroupTin": "98-7654321",
        "provGroupNpi": "1234567890",
        "provGroupNameFull": "Premier Medical Group LLC",
        "provOtherTin": "11-2233445",
        "provOtherNpi": "0987654321",
        "provOtherNameFull": "Secondary Care Associates",
        "aareteDerivedEffectiveDt": "2024-01-01",
        "aareteDerivedTerminationDt": "2025-12-31",
        "autoRenewalInd": true,
        "autoRenewalTerm": "12 months with 90-day notice"
    },
    "arrayFields": [
        {
            "exhibitTitle": "Exhibit A - Inpatient Services",
            "exhibitPage": "1",
            "reimbProvTin": "12-3456789",
            "reimbProvNpi": "1234567890",
            "reimbProvName": "Premier Medical Group LLC",
            "reimbEffectiveDt": "2024-01-01",
            "reimbTerminationDt": "2025-12-31",
            "aareteDerivedClaimTypeCd": "IP",
            "aareteDerivedProduct": "Commercial PPO",
            "aareteDerivedLob": "Commercial",
            "aareteDerivedProgram": "Standard",
            "aareteDerivedNetwork": "Preferred",
            "aareteDerivedProvType": "Acute Care Hospital",
            "provTaxonomyCd": "282N00000X",
            "provTaxonomyCdDesc": "General Acute Care Hospital",
            "provSpecialtyCd": "001",
            "provSpecialtyCdDesc": "General Practice",
            "placeOfServiceCd": "21",
            "placeOfServiceCdDesc": "Inpatient Hospital",
            "billTypeCd": "111",
            "billTypeCdDesc": "Hospital Inpatient Admit",
            "patientAgeMin": "0",
            "patientAgeMax": "120",
            "reimbTerm": "Per DRG with outlier provisions",
            "lobProgramRelationship": "Commercial-Standard",
            "lobProductRelationship": "Commercial-PPO",
            "carveoutInd": false,
            "carveoutCd": "",
            "lesserOfInd": true,
            "greaterOfInd": false,
            "aareteDerivedReimbMethod": "DRG",
            "unitOfMeasure": "Admission",
            "reimbPctRate": 95.5,
            "reimbFeeRate": 15000.00,
            "reimbConversionFactor": 1.25,
            "triggerCapThresholdAmt": 500000.00,
            "triggerBaseThreshold": 100000.00,
            "defaultInd": true,
            "additionDesc": "Annual rate increase based on CPI",
            "additionMaxFeeRateInc": 1500.00,
            "additionMaxPctRateInc": 3.5,
            "aareteDerivedAdditionRateChangeTimeline": "Annual on contract anniversary",
            "aareteDerivedFeeSchedule": "Medicare",
            "aareteDerivedFeeScheduleVersion": "2024",
            "serviceTerm": "All inpatient acute care services",
            "cpt4ProcCd": "99223",
            "cpt4ProcCdDesc": "Initial hospital care high severity",
            "cpt4ProcMod": "25",
            "cpt4ProcModDesc": "Significant separate E/M service",
            "revenueCd": "0120",
            "revenueCdDesc": "Room and Board Semi-Private",
            "diagCd": "A41.9",
            "diagCdDesc": "Sepsis unspecified organism",
            "ndcCd": "00069-0150-01",
            "ndcCdDesc": "Amoxicillin 500mg capsules",
            "claimAdmitTypeCd": "1",
            "authAdmitTypeDesc": "Emergency",
            "claimStatusCd": "A",
            "claimStatusCdDesc": "Approved",
            "grouperType": "MS-DRG",
            "grouperCd": "871",
            "grouperCdDesc": "Septicemia or severe sepsis w/o MV >96 hours w MCC",
            "grouperPctRate": 100.0,
            "grouperBaseRate": 12500.00,
            "aareteDerivedGrouperVersion": "v41.0",
            "grouperAlternativeLevelOfCare": "Standard Acute",
            "grouperSeverityInd": true,
            "grouperSeverity": "Major",
            "grouperRiskOfMortalitySubclass": "3",
            "grouperTransferInd": false,
            "grouperReadmissionsInd": false,
            "grouperHacInd": false,
            "outlierTerm": "Cost outlier at 2x fixed loss threshold",
            "outlierFirstDollarInd": false,
            "rangeNbrDays": "1-365",
            "outlierFixedLossNbrDaysThreshold": 30.0,
            "outlierFixedLossThreshold": 50000.00,
            "outlierMaximum": 1000000.00,
            "outlierMaximumFrequency": 2.0,
            "outlierPctRate": 80.0,
            "outlierExclusionCd": "TRANSPLANT",
            "outlierExclusionCdDesc": "Organ transplant cases excluded",
            "facilityAdjustmentTerm": "Standard facility adjustments apply",
            "dshInd": true,
            "dshPctRate": 15.5,
            "dshFeeRate": 2500.00,
            "imeInd": true,
            "imePctRate": 5.75,
            "imeFeeRate": 1000.00,
            "ntapInd": false,
            "ntapPctRate": 0.0,
            "ntapFeeRate": 0.00,
            "ucInd": true,
            "ucPctRate": 250.0,
            "ucFeeRate": 0.00,
            "gmeInd": true,
            "gmePctRate": 2.5,
            "gmeFeeRate": 500.00,
            "rateEscalatorInd": true,
            "rateEscalatorDesc": "Annual CPI-U adjustment capped at 3%",
            "rateEscalatorMaxRateIncPct": 3.0,
            "rateEscalatorRateChangeTimeline": 12.0,
            "stopLossTerm": "Individual stop loss at $500K",
            "stopLossFirstDollarInd": false,
            "stopLossRangeNbrDays": 365.0,
            "stopLossFixedLossThreshold": 500000.00,
            "stopLossMaximum": 2000000.00,
            "stopLossMaximumFrequency": 1.0,
            "stopLossDailyMaxRate": 10000.00,
            "stopLossPctRateOnExcessCharges": 60.0,
            "stopLossExclusionCd": "COSMETIC",
            "stopLossExclusionDesc": "Cosmetic procedures excluded from stop loss"
        },
        {
            "exhibitTitle": "Exhibit B - Outpatient Services",
            "exhibitPage": "15",
            "reimbProvTin": "12-3456789",
            "reimbProvNpi": "1234567890",
            "reimbProvName": "Premier Medical Group LLC",
            "reimbEffectiveDt": "2024-01-01",
            "reimbTerminationDt": "2025-12-31",
            "aareteDerivedClaimTypeCd": "OP",
            "aareteDerivedProduct": "Commercial PPO",
            "aareteDerivedLob": "Commercial",
            "aareteDerivedProgram": "Standard",
            "aareteDerivedNetwork": "Preferred",
            "aareteDerivedProvType": "Outpatient Clinic",
            "provTaxonomyCd": "261QM0801X",
            "provTaxonomyCdDesc": "Ambulatory Surgery Center",
            "provSpecialtyCd": "002",
            "provSpecialtyCdDesc": "Surgical Services",
            "placeOfServiceCd": "22",
            "placeOfServiceCdDesc": "Outpatient Hospital",
            "billTypeCd": "131",
            "billTypeCdDesc": "Hospital Outpatient",
            "patientAgeMin": "0",
            "patientAgeMax": "120",
            "reimbTerm": "Fee Schedule based on Medicare OPPS",
            "lobProgramRelationship": "Commercial-Standard",
            "lobProductRelationship": "Commercial-PPO",
            "carveoutInd": true,
            "carveoutCd": "IMPLANTS",
            "lesserOfInd": true,
            "greaterOfInd": false,
            "aareteDerivedReimbMethod": "Fee Schedule",
            "unitOfMeasure": "Procedure",
            "reimbPctRate": 150.0,
            "reimbFeeRate": 0.00,
            "reimbConversionFactor": 75.50,
            "triggerCapThresholdAmt": 100000.00,
            "triggerBaseThreshold": 25000.00,
            "defaultInd": false,
            "additionDesc": "Implant costs at invoice plus 10%",
            "additionMaxFeeRateInc": 500.00,
            "additionMaxPctRateInc": 2.0,
            "aareteDerivedAdditionRateChangeTimeline": "Quarterly review",
            "aareteDerivedFeeSchedule": "Medicare OPPS",
            "aareteDerivedFeeScheduleVersion": "2024-Q1",
            "serviceTerm": "Outpatient surgical and diagnostic services",
            "cpt4ProcCd": "29881",
            "cpt4ProcCdDesc": "Arthroscopy knee surgical",
            "cpt4ProcMod": "RT",
            "cpt4ProcModDesc": "Right side",
            "revenueCd": "0360",
            "revenueCdDesc": "Operating Room Services",
            "diagCd": "M17.11",
            "diagCdDesc": "Primary osteoarthritis right knee",
            "ndcCd": "00409-1966-01",
            "ndcCdDesc": "Bupivacaine injection",
            "claimAdmitTypeCd": "3",
            "authAdmitTypeDesc": "Elective",
            "claimStatusCd": "A",
            "claimStatusCdDesc": "Approved",
            "grouperType": "APC",
            "grouperCd": "5114",
            "grouperCdDesc": "Level 4 Musculoskeletal Procedures",
            "grouperPctRate": 150.0,
            "grouperBaseRate": 3500.00,
            "aareteDerivedGrouperVersion": "2024.1",
            "grouperAlternativeLevelOfCare": "Ambulatory",
            "grouperSeverityInd": false,
            "grouperSeverity": "Minor",
            "grouperRiskOfMortalitySubclass": "1",
            "grouperTransferInd": false,
            "grouperReadmissionsInd": false,
            "grouperHacInd": false,
            "outlierTerm": "No outlier provisions for outpatient",
            "outlierFirstDollarInd": false,
            "rangeNbrDays": "1",
            "outlierFixedLossNbrDaysThreshold": 0.0,
            "outlierFixedLossThreshold": 0.00,
            "outlierMaximum": 0.00,
            "outlierMaximumFrequency": 0.0,
            "outlierPctRate": 0.0,
            "outlierExclusionCd": "",
            "outlierExclusionCdDesc": "",
            "facilityAdjustmentTerm": "No facility adjustments for outpatient",
            "dshInd": false,
            "dshPctRate": 0.0,
            "dshFeeRate": 0.00,
            "imeInd": false,
            "imePctRate": 0.0,
            "imeFeeRate": 0.00,
            "ntapInd": false,
            "ntapPctRate": 0.0,
            "ntapFeeRate": 0.00,
            "ucInd": true,
            "ucPctRate": 200.0,
            "ucFeeRate": 0.00,
            "gmeInd": false,
            "gmePctRate": 0.0,
            "gmeFeeRate": 0.00,
            "rateEscalatorInd": true,
            "rateEscalatorDesc": "Annual Medicare OPPS update",
            "rateEscalatorMaxRateIncPct": 2.5,
            "rateEscalatorRateChangeTimeline": 12.0,
            "stopLossTerm": "N/A for outpatient",
            "stopLossFirstDollarInd": false,
            "stopLossRangeNbrDays": 0.0,
            "stopLossFixedLossThreshold": 0.00,
            "stopLossMaximum": 0.00,
            "stopLossMaximumFrequency": 0.0,
            "stopLossDailyMaxRate": 0.00,
            "stopLossPctRateOnExcessCharges": 0.0,
            "stopLossExclusionCd": "",
            "stopLossExclusionDesc": ""
        }
    ],
    "createdBy": "integration-test@example.com"
}
EOF
)

    # Replace placeholder with actual document ID
    body="${body//PLACEHOLDER_DOC_ID/$TARGET_DOC_ID}"

    print_info "Request: POST /field-extractions"
    print_info "Document ID: ${TARGET_DOC_ID}"
    print_info "Payload includes all 18 singleFields and 92 arrayFields per item"

    api_call "POST" "/field-extractions" "$body"

    if [[ "$HTTP_CODE" == "201" ]]; then
        local version
        version=$(echo "$RESPONSE_BODY" | jq -r '.version')
        print_success "Field extraction created: version=$version"
    else
        print_warning "Failed to create field extraction: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# Step 20: Verify single extraction via GET /field-extractions/history
step_20_verify_history_1() {
    print_header "20" "Verify single extraction via GET /field-extractions/history"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping field extraction history"
        return
    fi

    print_info "Request: GET /field-extractions/history?documentId=${TARGET_DOC_ID}"

    api_call "GET" "/field-extractions/history?documentId=${TARGET_DOC_ID}" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        local version_count
        version_count=$(echo "$RESPONSE_BODY" | jq '.versions | length')

        if [[ $version_count -eq 1 ]]; then
            print_success "Verified: 1 version in history"
        else
            print_warning "Expected 1 version, found $version_count"
        fi

        echo "$RESPONSE_BODY" | jq -r '.versions[] | "  - Version: \(.version), Created: \(.createdAt), By: \(.createdBy)"'
    else
        print_warning "Failed to get history: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# Step 21: Modify and create new version via POST /field-extractions
# Creates version 2 with updated values for all 18 singleFields and 92 arrayFields
# Includes 3 array items to demonstrate different scenarios
step_21_create_version_2() {
    print_header "21" "Create second version of field extraction"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping field extraction v2 creation"
        return
    fi

    local body
    body=$(cat << 'EOF'
{
    "documentId": "PLACEHOLDER_DOC_ID",
    "singleFields": {
        "fileName": "test_document_v2_amended.pdf",
        "contractTitle": "Master Service Agreement - First Amendment v2",
        "aareteDerivedAmendmentNum": 2,
        "clientName": "Acme Healthcare Systems (Updated)",
        "payerName": "BlueCross BlueShield of New York",
        "payerState": "NY",
        "providerState": "CA",
        "filenameTin": "12-3456789",
        "provGroupTin": "98-7654321",
        "provGroupNpi": "1234567890",
        "provGroupNameFull": "Premier Medical Group LLC - Western Region",
        "provOtherTin": "22-3344556",
        "provOtherNpi": "1122334455",
        "provOtherNameFull": "Tertiary Specialty Associates",
        "aareteDerivedEffectiveDt": "2024-07-01",
        "aareteDerivedTerminationDt": "2026-06-30",
        "autoRenewalInd": false,
        "autoRenewalTerm": "Manual renewal required 120 days prior"
    },
    "arrayFields": [
        {
            "exhibitTitle": "Exhibit A - Inpatient Services (Revised)",
            "exhibitPage": "1",
            "reimbProvTin": "12-3456789",
            "reimbProvNpi": "1234567890",
            "reimbProvName": "Premier Medical Group LLC",
            "reimbEffectiveDt": "2024-07-01",
            "reimbTerminationDt": "2026-06-30",
            "aareteDerivedClaimTypeCd": "IP",
            "aareteDerivedProduct": "Commercial HMO",
            "aareteDerivedLob": "Commercial",
            "aareteDerivedProgram": "Enhanced",
            "aareteDerivedNetwork": "Tier 1",
            "aareteDerivedProvType": "Acute Care Hospital",
            "provTaxonomyCd": "282N00000X",
            "provTaxonomyCdDesc": "General Acute Care Hospital",
            "provSpecialtyCd": "001",
            "provSpecialtyCdDesc": "General Practice",
            "placeOfServiceCd": "21",
            "placeOfServiceCdDesc": "Inpatient Hospital",
            "billTypeCd": "111",
            "billTypeCdDesc": "Hospital Inpatient Admit",
            "patientAgeMin": "0",
            "patientAgeMax": "120",
            "reimbTerm": "Per DRG with enhanced outlier provisions",
            "lobProgramRelationship": "Commercial-Enhanced",
            "lobProductRelationship": "Commercial-HMO",
            "carveoutInd": false,
            "carveoutCd": "",
            "lesserOfInd": true,
            "greaterOfInd": false,
            "aareteDerivedReimbMethod": "DRG",
            "unitOfMeasure": "Admission",
            "reimbPctRate": 98.0,
            "reimbFeeRate": 16500.00,
            "reimbConversionFactor": 1.35,
            "triggerCapThresholdAmt": 550000.00,
            "triggerBaseThreshold": 110000.00,
            "defaultInd": true,
            "additionDesc": "Annual rate increase based on CPI-U Medical Care",
            "additionMaxFeeRateInc": 1650.00,
            "additionMaxPctRateInc": 3.75,
            "aareteDerivedAdditionRateChangeTimeline": "Annual on July 1",
            "aareteDerivedFeeSchedule": "Medicare",
            "aareteDerivedFeeScheduleVersion": "2024-v2",
            "serviceTerm": "All inpatient acute care services including ICU",
            "cpt4ProcCd": "99223",
            "cpt4ProcCdDesc": "Initial hospital care high severity",
            "cpt4ProcMod": "25",
            "cpt4ProcModDesc": "Significant separate E/M service",
            "revenueCd": "0120",
            "revenueCdDesc": "Room and Board Semi-Private",
            "diagCd": "J18.9",
            "diagCdDesc": "Pneumonia unspecified organism",
            "ndcCd": "00069-0150-01",
            "ndcCdDesc": "Amoxicillin 500mg capsules",
            "claimAdmitTypeCd": "1",
            "authAdmitTypeDesc": "Emergency",
            "claimStatusCd": "A",
            "claimStatusCdDesc": "Approved",
            "grouperType": "MS-DRG",
            "grouperCd": "193",
            "grouperCdDesc": "Simple pneumonia and pleurisy w MCC",
            "grouperPctRate": 105.0,
            "grouperBaseRate": 13000.00,
            "aareteDerivedGrouperVersion": "v42.0",
            "grouperAlternativeLevelOfCare": "Standard Acute",
            "grouperSeverityInd": true,
            "grouperSeverity": "Major",
            "grouperRiskOfMortalitySubclass": "2",
            "grouperTransferInd": false,
            "grouperReadmissionsInd": false,
            "grouperHacInd": false,
            "outlierTerm": "Cost outlier at 1.75x fixed loss threshold",
            "outlierFirstDollarInd": false,
            "rangeNbrDays": "1-365",
            "outlierFixedLossNbrDaysThreshold": 25.0,
            "outlierFixedLossThreshold": 45000.00,
            "outlierMaximum": 1200000.00,
            "outlierMaximumFrequency": 3.0,
            "outlierPctRate": 85.0,
            "outlierExclusionCd": "TRANSPLANT",
            "outlierExclusionCdDesc": "Organ transplant cases excluded",
            "facilityAdjustmentTerm": "Enhanced facility adjustments apply",
            "dshInd": true,
            "dshPctRate": 16.0,
            "dshFeeRate": 2750.00,
            "imeInd": true,
            "imePctRate": 6.0,
            "imeFeeRate": 1100.00,
            "ntapInd": true,
            "ntapPctRate": 2.0,
            "ntapFeeRate": 350.00,
            "ucInd": true,
            "ucPctRate": 275.0,
            "ucFeeRate": 0.00,
            "gmeInd": true,
            "gmePctRate": 3.0,
            "gmeFeeRate": 600.00,
            "rateEscalatorInd": true,
            "rateEscalatorDesc": "Annual CPI-U Medical Care adjustment capped at 4%",
            "rateEscalatorMaxRateIncPct": 4.0,
            "rateEscalatorRateChangeTimeline": 12.0,
            "stopLossTerm": "Individual stop loss at $600K",
            "stopLossFirstDollarInd": false,
            "stopLossRangeNbrDays": 365.0,
            "stopLossFixedLossThreshold": 600000.00,
            "stopLossMaximum": 2500000.00,
            "stopLossMaximumFrequency": 1.0,
            "stopLossDailyMaxRate": 12000.00,
            "stopLossPctRateOnExcessCharges": 65.0,
            "stopLossExclusionCd": "COSMETIC",
            "stopLossExclusionDesc": "Cosmetic and experimental procedures excluded"
        },
        {
            "exhibitTitle": "Exhibit B - Outpatient Services (Revised)",
            "exhibitPage": "20",
            "reimbProvTin": "12-3456789",
            "reimbProvNpi": "1234567890",
            "reimbProvName": "Premier Medical Group LLC",
            "reimbEffectiveDt": "2024-07-01",
            "reimbTerminationDt": "2026-06-30",
            "aareteDerivedClaimTypeCd": "OP",
            "aareteDerivedProduct": "Commercial HMO",
            "aareteDerivedLob": "Commercial",
            "aareteDerivedProgram": "Enhanced",
            "aareteDerivedNetwork": "Tier 1",
            "aareteDerivedProvType": "Outpatient Clinic",
            "provTaxonomyCd": "261QM0801X",
            "provTaxonomyCdDesc": "Ambulatory Surgery Center",
            "provSpecialtyCd": "002",
            "provSpecialtyCdDesc": "Surgical Services",
            "placeOfServiceCd": "22",
            "placeOfServiceCdDesc": "Outpatient Hospital",
            "billTypeCd": "131",
            "billTypeCdDesc": "Hospital Outpatient",
            "patientAgeMin": "0",
            "patientAgeMax": "120",
            "reimbTerm": "Fee Schedule based on Medicare OPPS plus 55%",
            "lobProgramRelationship": "Commercial-Enhanced",
            "lobProductRelationship": "Commercial-HMO",
            "carveoutInd": true,
            "carveoutCd": "IMPLANTS-DEVICES",
            "lesserOfInd": true,
            "greaterOfInd": false,
            "aareteDerivedReimbMethod": "Fee Schedule",
            "unitOfMeasure": "Procedure",
            "reimbPctRate": 155.0,
            "reimbFeeRate": 0.00,
            "reimbConversionFactor": 78.25,
            "triggerCapThresholdAmt": 125000.00,
            "triggerBaseThreshold": 30000.00,
            "defaultInd": false,
            "additionDesc": "Implant and device costs at invoice plus 15%",
            "additionMaxFeeRateInc": 600.00,
            "additionMaxPctRateInc": 2.5,
            "aareteDerivedAdditionRateChangeTimeline": "Quarterly review",
            "aareteDerivedFeeSchedule": "Medicare OPPS",
            "aareteDerivedFeeScheduleVersion": "2024-Q3",
            "serviceTerm": "Outpatient surgical diagnostic and interventional",
            "cpt4ProcCd": "27447",
            "cpt4ProcCdDesc": "Total knee arthroplasty",
            "cpt4ProcMod": "LT",
            "cpt4ProcModDesc": "Left side",
            "revenueCd": "0360",
            "revenueCdDesc": "Operating Room Services",
            "diagCd": "M17.12",
            "diagCdDesc": "Primary osteoarthritis left knee",
            "ndcCd": "00409-1966-01",
            "ndcCdDesc": "Bupivacaine injection",
            "claimAdmitTypeCd": "3",
            "authAdmitTypeDesc": "Elective",
            "claimStatusCd": "A",
            "claimStatusCdDesc": "Approved",
            "grouperType": "APC",
            "grouperCd": "5115",
            "grouperCdDesc": "Level 5 Musculoskeletal Procedures",
            "grouperPctRate": 155.0,
            "grouperBaseRate": 4200.00,
            "aareteDerivedGrouperVersion": "2024.2",
            "grouperAlternativeLevelOfCare": "Ambulatory",
            "grouperSeverityInd": false,
            "grouperSeverity": "Moderate",
            "grouperRiskOfMortalitySubclass": "1",
            "grouperTransferInd": false,
            "grouperReadmissionsInd": false,
            "grouperHacInd": false,
            "outlierTerm": "No outlier provisions for outpatient",
            "outlierFirstDollarInd": false,
            "rangeNbrDays": "1",
            "outlierFixedLossNbrDaysThreshold": 0.0,
            "outlierFixedLossThreshold": 0.00,
            "outlierMaximum": 0.00,
            "outlierMaximumFrequency": 0.0,
            "outlierPctRate": 0.0,
            "outlierExclusionCd": "",
            "outlierExclusionCdDesc": "",
            "facilityAdjustmentTerm": "No facility adjustments for outpatient",
            "dshInd": false,
            "dshPctRate": 0.0,
            "dshFeeRate": 0.00,
            "imeInd": false,
            "imePctRate": 0.0,
            "imeFeeRate": 0.00,
            "ntapInd": false,
            "ntapPctRate": 0.0,
            "ntapFeeRate": 0.00,
            "ucInd": true,
            "ucPctRate": 225.0,
            "ucFeeRate": 0.00,
            "gmeInd": false,
            "gmePctRate": 0.0,
            "gmeFeeRate": 0.00,
            "rateEscalatorInd": true,
            "rateEscalatorDesc": "Annual Medicare OPPS update plus 2%",
            "rateEscalatorMaxRateIncPct": 3.0,
            "rateEscalatorRateChangeTimeline": 12.0,
            "stopLossTerm": "N/A for outpatient",
            "stopLossFirstDollarInd": false,
            "stopLossRangeNbrDays": 0.0,
            "stopLossFixedLossThreshold": 0.00,
            "stopLossMaximum": 0.00,
            "stopLossMaximumFrequency": 0.0,
            "stopLossDailyMaxRate": 0.00,
            "stopLossPctRateOnExcessCharges": 0.0,
            "stopLossExclusionCd": "",
            "stopLossExclusionDesc": ""
        },
        {
            "exhibitTitle": "Exhibit C - Professional Services (New)",
            "exhibitPage": "35",
            "reimbProvTin": "22-3344556",
            "reimbProvNpi": "1122334455",
            "reimbProvName": "Tertiary Specialty Associates",
            "reimbEffectiveDt": "2024-07-01",
            "reimbTerminationDt": "2026-06-30",
            "aareteDerivedClaimTypeCd": "PROF",
            "aareteDerivedProduct": "Commercial HMO",
            "aareteDerivedLob": "Commercial",
            "aareteDerivedProgram": "Enhanced",
            "aareteDerivedNetwork": "Tier 1",
            "aareteDerivedProvType": "Physician Group",
            "provTaxonomyCd": "207R00000X",
            "provTaxonomyCdDesc": "Internal Medicine",
            "provSpecialtyCd": "011",
            "provSpecialtyCdDesc": "Internal Medicine",
            "placeOfServiceCd": "11",
            "placeOfServiceCdDesc": "Office",
            "billTypeCd": "N/A",
            "billTypeCdDesc": "Professional claim",
            "patientAgeMin": "18",
            "patientAgeMax": "120",
            "reimbTerm": "RBRVS Medicare Fee Schedule plus 20%",
            "lobProgramRelationship": "Commercial-Enhanced",
            "lobProductRelationship": "Commercial-HMO",
            "carveoutInd": false,
            "carveoutCd": "",
            "lesserOfInd": true,
            "greaterOfInd": false,
            "aareteDerivedReimbMethod": "Fee Schedule",
            "unitOfMeasure": "Service",
            "reimbPctRate": 120.0,
            "reimbFeeRate": 0.00,
            "reimbConversionFactor": 45.75,
            "triggerCapThresholdAmt": 50000.00,
            "triggerBaseThreshold": 10000.00,
            "defaultInd": false,
            "additionDesc": "Annual RBRVS update applied",
            "additionMaxFeeRateInc": 100.00,
            "additionMaxPctRateInc": 2.0,
            "aareteDerivedAdditionRateChangeTimeline": "Annual on January 1",
            "aareteDerivedFeeSchedule": "Medicare RBRVS",
            "aareteDerivedFeeScheduleVersion": "2024",
            "serviceTerm": "Professional evaluation and management services",
            "cpt4ProcCd": "99214",
            "cpt4ProcCdDesc": "Office visit established moderate",
            "cpt4ProcMod": "",
            "cpt4ProcModDesc": "",
            "revenueCd": "N/A",
            "revenueCdDesc": "Professional claim",
            "diagCd": "E11.9",
            "diagCdDesc": "Type 2 diabetes mellitus without complications",
            "ndcCd": "",
            "ndcCdDesc": "",
            "claimAdmitTypeCd": "N/A",
            "authAdmitTypeDesc": "Not applicable",
            "claimStatusCd": "A",
            "claimStatusCdDesc": "Approved",
            "grouperType": "N/A",
            "grouperCd": "N/A",
            "grouperCdDesc": "Not applicable for professional",
            "grouperPctRate": 0.0,
            "grouperBaseRate": 0.00,
            "aareteDerivedGrouperVersion": "N/A",
            "grouperAlternativeLevelOfCare": "N/A",
            "grouperSeverityInd": false,
            "grouperSeverity": "N/A",
            "grouperRiskOfMortalitySubclass": "N/A",
            "grouperTransferInd": false,
            "grouperReadmissionsInd": false,
            "grouperHacInd": false,
            "outlierTerm": "N/A for professional services",
            "outlierFirstDollarInd": false,
            "rangeNbrDays": "N/A",
            "outlierFixedLossNbrDaysThreshold": 0.0,
            "outlierFixedLossThreshold": 0.00,
            "outlierMaximum": 0.00,
            "outlierMaximumFrequency": 0.0,
            "outlierPctRate": 0.0,
            "outlierExclusionCd": "",
            "outlierExclusionCdDesc": "",
            "facilityAdjustmentTerm": "N/A for professional services",
            "dshInd": false,
            "dshPctRate": 0.0,
            "dshFeeRate": 0.00,
            "imeInd": false,
            "imePctRate": 0.0,
            "imeFeeRate": 0.00,
            "ntapInd": false,
            "ntapPctRate": 0.0,
            "ntapFeeRate": 0.00,
            "ucInd": true,
            "ucPctRate": 150.0,
            "ucFeeRate": 0.00,
            "gmeInd": false,
            "gmePctRate": 0.0,
            "gmeFeeRate": 0.00,
            "rateEscalatorInd": true,
            "rateEscalatorDesc": "Annual Medicare RBRVS update",
            "rateEscalatorMaxRateIncPct": 2.0,
            "rateEscalatorRateChangeTimeline": 12.0,
            "stopLossTerm": "N/A for professional services",
            "stopLossFirstDollarInd": false,
            "stopLossRangeNbrDays": 0.0,
            "stopLossFixedLossThreshold": 0.00,
            "stopLossMaximum": 0.00,
            "stopLossMaximumFrequency": 0.0,
            "stopLossDailyMaxRate": 0.00,
            "stopLossPctRateOnExcessCharges": 0.0,
            "stopLossExclusionCd": "",
            "stopLossExclusionDesc": ""
        }
    ],
    "createdBy": "integration-test@example.com"
}
EOF
)

    # Replace placeholder with actual document ID
    body="${body//PLACEHOLDER_DOC_ID/$TARGET_DOC_ID}"

    print_info "Request: POST /field-extractions"
    print_info "Document ID: ${TARGET_DOC_ID}"
    print_info "Payload includes all 18 singleFields and 92 arrayFields per item (3 items)"

    api_call "POST" "/field-extractions" "$body"

    if [[ "$HTTP_CODE" == "201" ]]; then
        local version
        version=$(echo "$RESPONSE_BODY" | jq -r '.version')
        print_success "Field extraction v2 created: version=$version"
    else
        print_warning "Failed to create field extraction v2: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# Step 22: Verify two versions via GET /field-extractions/history
step_22_verify_history_2() {
    print_header "22" "Verify two versions in history"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping field extraction history"
        return
    fi

    print_info "Request: GET /field-extractions/history?documentId=${TARGET_DOC_ID}"

    api_call "GET" "/field-extractions/history?documentId=${TARGET_DOC_ID}" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        local version_count
        version_count=$(echo "$RESPONSE_BODY" | jq '.versions | length')

        if [[ $version_count -eq 2 ]]; then
            print_success "Verified: 2 versions in history"
        else
            print_warning "Expected 2 versions, found $version_count"
        fi

        echo "$RESPONSE_BODY" | jq -r '.versions[] | "  - Version: \(.version), Created: \(.createdAt), By: \(.createdBy)"'
    else
        print_warning "Failed to get history: $HTTP_CODE"
        print_info "Response: $RESPONSE_BODY"
    fi
}

# Step 23: Retrieve specific versions via GET /field-extractions/version
# Prints all field values to verify round-trip data integrity
step_23_get_specific_versions() {
    print_header "23" "Retrieve specific versions via GET /field-extractions/version"

    if [[ -z "$TARGET_DOC_ID" ]]; then
        print_warning "No target document available - skipping specific version retrieval"
        return
    fi

    # Get version 1
    print_info ""
    print_info "Testing retrieval of version 1..."
    print_info "Request: GET /field-extractions/version?documentId=${TARGET_DOC_ID}&version=1"

    api_call "GET" "/field-extractions/version?documentId=${TARGET_DOC_ID}&version=1" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        print_success "Version 1 retrieved successfully"
        print_info ""
        print_info "=== VERSION 1 FULL ROUND-TRIP DATA ==="
        print_info ""
        print_info "--- Single Fields (18 fields) ---"
        echo "$RESPONSE_BODY" | jq -r '.singleFields | to_entries[] | "  \(.key): \(.value)"'
        print_info ""
        print_info "--- Array Fields (2 items, 92 fields each) ---"
        local array_count
        array_count=$(echo "$RESPONSE_BODY" | jq '.arrayFields | length')
        print_info "  Total array items: $array_count"

        for ((i=0; i<array_count; i++)); do
            print_info ""
            local exhibit_title
            exhibit_title=$(echo "$RESPONSE_BODY" | jq -r ".arrayFields[$i].exhibitTitle")
            print_info "  --- Array Item $((i+1)): $exhibit_title ---"
            echo "$RESPONSE_BODY" | jq -r ".arrayFields[$i] | to_entries[] | \"    \(.key): \(.value)\""
        done
    else
        print_warning "Failed to get version 1: $HTTP_CODE"
    fi

    # Get version 2
    print_info ""
    print_info "Testing retrieval of version 2..."
    print_info "Request: GET /field-extractions/version?documentId=${TARGET_DOC_ID}&version=2"

    api_call "GET" "/field-extractions/version?documentId=${TARGET_DOC_ID}&version=2" ""

    if [[ "$HTTP_CODE" == "200" ]]; then
        print_success "Version 2 retrieved successfully"
        print_info ""
        print_info "=== VERSION 2 FULL ROUND-TRIP DATA ==="
        print_info ""
        print_info "--- Single Fields (18 fields) ---"
        echo "$RESPONSE_BODY" | jq -r '.singleFields | to_entries[] | "  \(.key): \(.value)"'
        print_info ""
        print_info "--- Array Fields (3 items, 92 fields each) ---"
        local array_count
        array_count=$(echo "$RESPONSE_BODY" | jq '.arrayFields | length')
        print_info "  Total array items: $array_count"

        for ((i=0; i<array_count; i++)); do
            print_info ""
            local exhibit_title
            exhibit_title=$(echo "$RESPONSE_BODY" | jq -r ".arrayFields[$i].exhibitTitle")
            print_info "  --- Array Item $((i+1)): $exhibit_title ---"
            echo "$RESPONSE_BODY" | jq -r ".arrayFields[$i] | to_entries[] | \"    \(.key): \(.value)\""
        done
    else
        print_warning "Failed to get version 2: $HTTP_CODE"
    fi

    # Test non-existent version (should return 404)
    print_info ""
    print_info "Testing retrieval of non-existent version 999..."
    print_info "Request: GET /field-extractions/version?documentId=${TARGET_DOC_ID}&version=999"

    api_call "GET" "/field-extractions/version?documentId=${TARGET_DOC_ID}&version=999" ""

    if [[ "$HTTP_CODE" == "404" ]]; then
        print_success "Non-existent version correctly returned 404"
    else
        print_warning "Expected 404 for non-existent version, got $HTTP_CODE"
    fi
}

# =============================================================================
# MAIN EXECUTION
# =============================================================================

main() {
    check_dependencies
    validate_args "$@"

    # Phase 0: Authentication
    echo ""
    echo -e "${BLUE}=== PHASE 0: AUTHENTICATION ===${NC}"
    step_00_authenticate

    # Phase 1: Client Setup
    echo ""
    echo -e "${BLUE}=== PHASE 1: CLIENT SETUP ===${NC}"
    step_01_create_client
    step_02_verify_client
    step_02_5_list_clients
    step_03_update_client
    step_04_confirm_status

    # Phase 2: Document Upload
    echo ""
    echo -e "${BLUE}=== PHASE 2: DOCUMENT UPLOAD ===${NC}"
    step_05_create_zip
    step_06_upload_batch
    step_07_verify_batch
    step_08_get_documents
    step_09_10_folder_documents
    step_11_upload_single
    step_12_verify_total
    step_12_5_verify_file_sizes

    # Phase 3: Label Operations
    echo ""
    echo -e "${BLUE}=== PHASE 3: LABEL OPERATIONS ===${NC}"
    step_13_14_apply_labels
    step_15_list_folders
    step_16_folder_metrics
    step_16_5_folder_document_sizes
    step_17_documents_by_label

    # Phase 4: Field Extractions
    echo ""
    echo -e "${BLUE}=== PHASE 4: FIELD EXTRACTIONS ===${NC}"
    step_18_verify_no_extraction
    step_19_create_extraction
    step_20_verify_history_1
    step_21_create_version_2
    step_22_verify_history_2
    step_23_get_specific_versions

    # Summary
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${GREEN}All integration tests completed!${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo ""
    echo "Summary:"
    echo "  Client ID: $CLIENT_ID"
    echo "  Client Name: $CLIENT_NAME"
    echo "  Documents Created: ${#DOCUMENT_IDS[@]}"
    echo "  Target Document: $TARGET_DOC_ID"
    echo ""
    echo "The system is now populated with test data for UI development."
}

# Run main function with all arguments
main "$@"