Jay Brown
451da3d26d
Support for service accounts with static credentials * feature complete * tests and docs * lint fix
325 lines
10 KiB
Bash
Executable File
325 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Test script for ZIP batch upload functionality
|
|
#!/bin/bash
|
|
|
|
# Test script for ZIP batch upload functionality
|
|
# This script exercises all batch endpoints with a cloud deployment
|
|
# This version uses only API calls and does not require database access
|
|
# It's designed to work with cloud-deployed instances of the query orchestration service
|
|
#
|
|
# REST API operations tested:
|
|
# - GET /metrics - Check service health
|
|
# - POST /client - Create new test client
|
|
# - PATCH /client/{CLIENT_ID} - Enable CanSync flag via API
|
|
# - POST /client/{CLIENT_ID}/document/batch - Upload ZIP file with multiple PDFs
|
|
# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Get batch status and details
|
|
# - GET /client/{CLIENT_ID}/document/batch - List all batches for client with pagination
|
|
# - POST /client/{CLIENT_ID}/document/batch (invalid) - Test error handling with non-ZIP file
|
|
# - DELETE /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Cancel/delete batch processing
|
|
|
|
set -e # Exit on any error
|
|
|
|
# local host version of the test
|
|
#BASE_URL="http://localhost:8080"
|
|
# Configuration for cloud
|
|
#BASE_URL="http://queryo-query-rtxkppcxpjsb-742555588.us-east-2.elb.amazonaws.com"
|
|
BASE_URL="http://queryo-query-q0pz6j2syaox-1001614225.us-east-2.elb.amazonaws.com"
|
|
# CLIENT_ID will be generated in main() to ensure it's fresh
|
|
ZIP_FILE="test_batch.zip"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Logging functions
|
|
log_info() {
|
|
echo -e "${GREEN}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_warn() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
# Check if service is running
|
|
check_service() {
|
|
log_info "Checking if queryAPI service is running..."
|
|
if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
|
|
log_error "queryAPI service is not running at $BASE_URL"
|
|
log_info "Please check the cloud deployment status"
|
|
exit 1
|
|
fi
|
|
log_info "Service is running"
|
|
}
|
|
|
|
# Create test ZIP file with real PDFs from test.pdf.batch directory, each in its own subfolder
|
|
create_test_zip() {
|
|
log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch, each in its own subfolder..."
|
|
|
|
# Remove existing file if it exists
|
|
rm -f "$ZIP_FILE"
|
|
|
|
# Check if test.pdf.batch directory exists
|
|
pdf_dir="./test.pdf.batch"
|
|
if [ ! -d "$pdf_dir" ]; then
|
|
log_error "Directory $pdf_dir does not exist"
|
|
exit 1
|
|
fi
|
|
|
|
# Check if there are PDF files in the directory
|
|
pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
|
|
if [ "$pdf_count" -eq 0 ]; then
|
|
log_error "No PDF files found in $pdf_dir"
|
|
exit 1
|
|
fi
|
|
|
|
# Create temporary directory structure
|
|
temp_dir="temp_zip_structure"
|
|
rm -rf "$temp_dir"
|
|
mkdir -p "$temp_dir"
|
|
|
|
# Copy each PDF to its own subfolder
|
|
folder_num=1
|
|
for pdf_file in "$pdf_dir"/*.pdf; do
|
|
if [ -f "$pdf_file" ]; then
|
|
folder_name="folder$folder_num"
|
|
mkdir -p "$temp_dir/$folder_name"
|
|
cp "$pdf_file" "$temp_dir/$folder_name/"
|
|
log_info "Placed $(basename "$pdf_file") in $folder_name/"
|
|
folder_num=$((folder_num + 1))
|
|
fi
|
|
done
|
|
|
|
# Create ZIP file with the folder structure
|
|
current_dir=$(pwd)
|
|
cd "$temp_dir"
|
|
zip -q -r "$current_dir/$ZIP_FILE" .
|
|
cd "$current_dir"
|
|
|
|
# Cleanup temporary directory
|
|
rm -rf "$temp_dir"
|
|
|
|
file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
|
|
log_info "Created $ZIP_FILE with $pdf_count real PDFs in separate subfolders ($file_size bytes)"
|
|
}
|
|
|
|
# Create client (idempotent)
|
|
create_client() {
|
|
log_info "Creating/checking client: $CLIENT_ID"
|
|
log_info "API call: POST $BASE_URL/client with ID=$CLIENT_ID"
|
|
|
|
# Try to create client, ignore if already exists
|
|
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
|
|
if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
|
|
log_info "Client created/exists: $CLIENT_ID"
|
|
elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
|
|
log_info "Client already exists or duplicate name: $CLIENT_ID"
|
|
else
|
|
log_warn "Unexpected response when creating client (HTTP $http_code), continuing anyway..."
|
|
fi
|
|
}
|
|
|
|
# Enable CanSync flag for client using API PATCH call
|
|
enable_client_sync() {
|
|
log_info "Enabling CanSync flag for client: $CLIENT_ID"
|
|
log_info "API call: PATCH $BASE_URL/client/$CLIENT_ID"
|
|
|
|
# Use PATCH API to update client with can_sync: true
|
|
response=$(curl -s -w "\n%{http_code}" -X PATCH "$BASE_URL/client/$CLIENT_ID" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"can_sync":true}' 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
response_body=$(echo "$response" | sed '$d')
|
|
|
|
if [ "$http_code" = "200" ] || [ "$http_code" = "204" ]; then
|
|
log_info "CanSync flag enabled via API - documents can now be processed"
|
|
return 0
|
|
else
|
|
log_warn "Failed to enable CanSync flag via API (HTTP $http_code), but continuing with test"
|
|
log_warn "Documents may not process without CanSync enabled"
|
|
echo "$response_body"
|
|
return 0 # Continue anyway since this is a test
|
|
fi
|
|
}
|
|
|
|
# Upload ZIP batch
|
|
upload_batch() {
|
|
log_info "Uploading ZIP batch..."
|
|
log_info "API call: POST $BASE_URL/client/$CLIENT_ID/document/batch"
|
|
|
|
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
|
-F "archive=@$ZIP_FILE" 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
response_body=$(echo "$response" | sed '$d')
|
|
|
|
if [ "$http_code" = "202" ]; then
|
|
BATCH_ID=$(echo "$response_body" | grep -o '"batch_id":"[^"]*"' | cut -d'"' -f4)
|
|
STATUS_URL=$(echo "$response_body" | grep -o '"status_url":"[^"]*"' | cut -d'"' -f4)
|
|
log_info "Batch uploaded successfully"
|
|
log_info "Batch ID: $BATCH_ID"
|
|
log_info "Status URL: $STATUS_URL"
|
|
|
|
# Print batch number prominently for easy capture
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "BATCH NUMBER CREATED: $BATCH_ID"
|
|
echo "=========================================="
|
|
echo ""
|
|
|
|
return 0
|
|
else
|
|
log_error "Failed to upload batch (HTTP $http_code)"
|
|
echo "$response_body"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Get batch status
|
|
get_batch_status() {
|
|
log_info "Getting batch status..."
|
|
|
|
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
response_body=$(echo "$response" | sed '$d')
|
|
|
|
if [ "$http_code" = "200" ]; then
|
|
status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
|
filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
|
|
log_info "Batch status: $status"
|
|
log_info "Original filename: $filename"
|
|
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
|
return 0
|
|
else
|
|
log_error "Failed to get batch status (HTTP $http_code)"
|
|
echo "$response_body"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# List batches for client
|
|
list_batches() {
|
|
log_info "Listing batches for client..."
|
|
|
|
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
response_body=$(echo "$response" | sed '$d')
|
|
|
|
if [ "$http_code" = "200" ]; then
|
|
batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
|
|
log_info "Found $batch_count batches"
|
|
echo "$response_body" | python3 -m json.tool 2>/dev/null || echo "$response_body"
|
|
return 0
|
|
else
|
|
log_error "Failed to list batches (HTTP $http_code)"
|
|
echo "$response_body"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Test invalid file upload
|
|
test_invalid_file() {
|
|
log_info "Testing invalid file upload (should fail)..."
|
|
|
|
# Create invalid file
|
|
echo "This is not a ZIP file" > invalid.txt
|
|
|
|
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
|
-F "archive=@invalid.txt" 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
response_body=$(echo "$response" | sed '$d')
|
|
|
|
# Cleanup
|
|
rm -f invalid.txt
|
|
|
|
if [ "$http_code" = "400" ]; then
|
|
log_info "Invalid file correctly rejected (HTTP $http_code)"
|
|
return 0
|
|
else
|
|
log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
|
|
echo "$response_body"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Cancel batch
|
|
cancel_batch() {
|
|
log_info "Canceling batch..."
|
|
|
|
response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
|
|
|
http_code=$(echo "$response" | tail -n1)
|
|
|
|
if [ "$http_code" = "204" ]; then
|
|
log_info "Batch canceled successfully"
|
|
return 0
|
|
elif [ "$http_code" = "404" ]; then
|
|
log_warn "Batch not found or already completed (HTTP $http_code)"
|
|
return 0
|
|
else
|
|
log_error "Failed to cancel batch (HTTP $http_code)"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Cleanup function
|
|
cleanup() {
|
|
log_info "Cleaning up..."
|
|
rm -f "$ZIP_FILE" invalid.txt
|
|
rm -rf temp_zip_structure
|
|
log_info "Cleanup complete"
|
|
}
|
|
|
|
# Main test flow
|
|
main() {
|
|
# Generate fresh CLIENT_ID inside main to ensure it's current
|
|
CLIENT_ID="test_client_$(date +%s)"
|
|
|
|
log_info "Starting ZIP batch upload test..."
|
|
log_info "Generated client ID: $CLIENT_ID"
|
|
|
|
# Setup trap for cleanup
|
|
trap cleanup EXIT
|
|
|
|
# Run tests
|
|
check_service
|
|
create_test_zip
|
|
create_client
|
|
enable_client_sync
|
|
|
|
if upload_batch; then
|
|
sleep 1 # Brief pause to let the system process
|
|
get_batch_status
|
|
list_batches
|
|
test_invalid_file
|
|
cancel_batch
|
|
|
|
# Verify cancellation
|
|
sleep 1
|
|
log_info "Verifying batch was canceled..."
|
|
get_batch_status
|
|
|
|
log_info "✅ All tests completed successfully!"
|
|
else
|
|
log_error "❌ Batch upload failed, skipping remaining tests"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Run main function
|
|
main "$@" |