Merged in feature/mutable-metadata1 (pull request #221)

M1, M2 and M3 complete

* M1, M2 and M3 complete

* review changes

* docs

* docs

scripts/manual_tests/test_zip_all_types_upload.sh

@@ -27,6 +27,12 @@ TEMP_DIR="temp_all_types_structure"
 TOTAL_VALID=10
 TOTAL_INVALID=10
 TOTAL_FILES=$((TOTAL_VALID + TOTAL_INVALID))
+VERBOSE=0
+
+# Path to the valid PDF inside the generated test set. The same file is hashed
+# before upload and again after download to prove the bytes are unchanged.
+VALID_PDF_REL_PATH="valid/test_document.pdf"
+ORIGINAL_PDF_SHA256=""
 
 # --- Terminal outcome constants ---
 TERMINAL_OUTCOMES="duplicate init_duplicate failed_open failed_read failed_s3_upload failed_upload invalid_type clean_passed clean_failed sync_skipped"
@@ -55,6 +61,42 @@ log_step() {
     echo -e "${BLUE}[STEP]${NC} $1"
 }
 
+usage() {
+    cat <<EOF
+Usage: $0 [-v]
+
+Options:
+  -v    Print the formatted final batch status JSON on success
+EOF
+}
+
+parse_args() {
+    while getopts ":v" opt; do
+        case "$opt" in
+            v)
+                VERBOSE=1
+                ;;
+            \?)
+                log_error "Unknown option: -$OPTARG"
+                usage
+                exit 1
+                ;;
+        esac
+    done
+
+    shift $((OPTIND - 1))
+
+    if [ $# -gt 0 ]; then
+        log_error "Unexpected argument(s): $*"
+        usage
+        exit 1
+    fi
+}
+
+print_formatted_json() {
+    printf '%s\n' "$1" | python3 -m json.tool
+}
+
 # Check if service is running
 check_service() {
     log_step "Checking if queryAPI service is running..."
@@ -499,6 +541,92 @@ create_test_zip() {
     log_info "Created $ZIP_FILE ($file_size bytes)"
 }
 
+# Compute SHA256 of the original valid PDF before it is packed into the ZIP.
+# Saved into ORIGINAL_PDF_SHA256 so verify_pdf_integrity can compare after download.
+compute_original_pdf_hash() {
+    log_step "Computing SHA256 of original PDF before upload..."
+    local pdf_path="$TEMP_DIR/$VALID_PDF_REL_PATH"
+
+    if [ ! -f "$pdf_path" ]; then
+        log_error "Original PDF not found at $pdf_path"
+        return 1
+    fi
+
+    ORIGINAL_PDF_SHA256=$(PDF_PATH="$pdf_path" python3 -c "
+import hashlib, os, sys
+with open(os.environ['PDF_PATH'], 'rb') as f:
+    print(hashlib.sha256(f.read()).hexdigest())
+")
+
+    if [ -z "$ORIGINAL_PDF_SHA256" ]; then
+        log_error "Failed to compute SHA256 of original PDF"
+        return 1
+    fi
+
+    log_info "Original PDF SHA256: $ORIGINAL_PDF_SHA256"
+}
+
+# Download the stored PDF by document_id and verify the bytes match the
+# pre-upload hash exactly. Proves the pipeline did not mutate the file.
+verify_pdf_integrity() {
+    log_step "Verifying downloaded PDF is byte-identical to the original..."
+
+    if [ -z "$ORIGINAL_PDF_SHA256" ]; then
+        log_error "Original PDF hash was not captured; cannot verify integrity"
+        return 1
+    fi
+
+    # Look up the PDF's document_id from the final batch status.
+    local batch_response
+    batch_response=$(curl -s "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID")
+
+    local pdf_doc_id
+    pdf_doc_id=$(PDF_REL="$VALID_PDF_REL_PATH" python3 -c "
+import json, os, sys
+data = json.loads(sys.stdin.read())
+target = os.environ['PDF_REL']
+for o in data.get('document_outcomes', []):
+    if o.get('filename') == target:
+        print(o.get('document_id') or '')
+        break
+" <<< "$batch_response")
+
+    if [ -z "$pdf_doc_id" ]; then
+        log_error "Could not resolve document_id for $VALID_PDF_REL_PATH in batch status"
+        return 1
+    fi
+    log_info "PDF document_id: $pdf_doc_id"
+
+    local downloaded_path="$TEMP_DIR/downloaded_test_document.pdf"
+    local http_code
+    http_code=$(curl -s -o "$downloaded_path" -w "%{http_code}" \
+        "$BASE_URL/document/$pdf_doc_id/download")
+
+    if [ "$http_code" != "200" ]; then
+        log_error "Download failed (HTTP $http_code) for document $pdf_doc_id"
+        return 1
+    fi
+
+    local downloaded_sha256
+    downloaded_sha256=$(PDF_PATH="$downloaded_path" python3 -c "
+import hashlib, os
+with open(os.environ['PDF_PATH'], 'rb') as f:
+    print(hashlib.sha256(f.read()).hexdigest())
+")
+
+    log_info "Downloaded PDF SHA256: $downloaded_sha256"
+
+    if [ "$downloaded_sha256" = "$ORIGINAL_PDF_SHA256" ]; then
+        log_info "PDF integrity verified: downloaded bytes match original exactly"
+        return 0
+    fi
+
+    log_error "PDF integrity check FAILED: downloaded bytes differ from original"
+    log_error "  expected (original):   $ORIGINAL_PDF_SHA256"
+    log_error "  actual   (downloaded): $downloaded_sha256"
+    return 1
+}
+
 # Upload ZIP batch
 upload_batch() {
     log_step "Uploading ZIP batch..."
@@ -701,6 +829,12 @@ PYEOF
         log_error "Validation FAILED"
         return 1
     fi
+
+    if [ "$VERBOSE" -eq 1 ]; then
+        log_info "Final batch status JSON:"
+        print_formatted_json "$response"
+    fi
+
     log_info "Validation PASSED"
     return 0
 }
@@ -736,6 +870,7 @@ cleanup() {
 
 # Main test flow
 main() {
+    parse_args "$@"
     log_step "Starting multi-type ZIP batch upload test..."
     log_info "Testing file types: PDF, TIFF, JPEG, PNG, BMP, DOCX, XLSX, PPTX, TXT, EML"
     log_info "Using client ID: $CLIENT_ID"
@@ -744,6 +879,7 @@ main() {
 
     check_service
     generate_test_files
+    compute_original_pdf_hash
     create_test_zip
     create_client
     enable_client_sync
@@ -752,6 +888,7 @@ main() {
         poll_batch_extraction
         poll_pipeline_complete
         validate_results
+        verify_pdf_integrity
 
         test_invalid_zip_upload