Merged in feature/doc_import (pull request #177)
integration of background processor * integration part 1 * feature working * fix mimetype issue
This commit is contained in:
Jay Brown
Executable
+1
@@ -0,0 +1 @@
|
||||
aws --endpoint-url=http://localhost:4566 s3 ls s3://documentin/ --recursive
|
||||
@@ -0,0 +1,109 @@
|
||||
# Document Processing Monitor
|
||||
|
||||
This directory contains scripts to monitor document processing status after running batch uploads.
|
||||
|
||||
## Quick Start
|
||||
|
||||
1. Navigate to the manual tests directory:
|
||||
```bash
|
||||
cd scripts/manual_tests
|
||||
```
|
||||
|
||||
2. Run your batch upload:
|
||||
```bash
|
||||
./test_zip_upload2.sh
|
||||
```
|
||||
|
||||
3. Check processing status:
|
||||
```bash
|
||||
./monitoring/check_processing.sh
|
||||
```
|
||||
|
||||
## What the Monitor Shows
|
||||
|
||||
The `check_processing.sh` script provides a post-upload analysis showing:
|
||||
|
||||
- **Batch Status**: Overall batch processing state and progress
|
||||
- **Document Breakdown**: Document counts through each processing stage (uploaded, cleaned, extracted)
|
||||
- **Queue Depths**: Current number of messages in each SQS queue
|
||||
- **Processing Analysis**: Where processing stopped and why
|
||||
- **Next Steps**: Specific solutions for common issues
|
||||
|
||||
## Sample Output
|
||||
|
||||
```
|
||||
========================================
|
||||
Document Processing Status Check
|
||||
========================================
|
||||
|
||||
[INFO] Finding most recent batch...
|
||||
[SUCCESS] Found batch: 550e8400-e29b-41d4-a716-446655440000
|
||||
|
||||
[INFO] Batch Status:
|
||||
Status: processing
|
||||
Progress: 2/3 documents processed
|
||||
|
||||
[INFO] Document Breakdown:
|
||||
Total documents in DB: 3
|
||||
Uploaded to storage: 3
|
||||
Passed cleaning: 0
|
||||
Text extracted: 0
|
||||
|
||||
[INFO] Current Queue Depths:
|
||||
store_event: 0 messages
|
||||
document_init: 0 messages
|
||||
document_sync: 1 messages
|
||||
document_clean: 0 messages
|
||||
document_text: 0 messages
|
||||
query_sync: 0 messages
|
||||
query_runner: 0 messages
|
||||
|
||||
[INFO] Processing Analysis:
|
||||
[WARNING] Documents blocked at sync stage: Client CanSync flag is false
|
||||
Solution: Run 'INSERT INTO clientcansync (clientid, cansync) VALUES ('[client_id]', true);'
|
||||
|
||||
⚠️ Processing stopped at document sync (client CanSync issue)
|
||||
========================================
|
||||
```
|
||||
|
||||
## Common Issues and Solutions
|
||||
|
||||
### 1. Processing Stopped at Document Sync
|
||||
**Symptom**: Documents stuck in `document_sync` queue
|
||||
**Cause**: Client's CanSync flag is false or unset
|
||||
|
||||
**About the CanSync Flag**:
|
||||
The CanSync flag is a client-level permission control that determines whether documents for a specific client are allowed to proceed through the document synchronization stage. This is a business rule enforcement mechanism that can be used for:
|
||||
- Client onboarding workflows (only allow sync after certain prerequisites)
|
||||
- Compliance requirements (only sync documents for authorized clients)
|
||||
- Feature gating (premium vs basic client tiers)
|
||||
|
||||
**Default Behavior**: New clients do NOT have a CanSync entry by default (NULL state), which blocks processing.
|
||||
|
||||
**Solution**:
|
||||
```sql
|
||||
INSERT INTO clientcansync (clientid, cansync) VALUES ('your_client_id', true);
|
||||
```
|
||||
|
||||
**For Demo Purposes**: Always set `cansync = true` to allow the full document processing pipeline to run and demonstrate all stages working.
|
||||
|
||||
### 2. Processing Stopped at Document Cleaning
|
||||
**Symptom**: Documents stuck in `document_clean` queue with MIME type errors
|
||||
**Cause**: S3 objects missing proper Content-Type headers
|
||||
**Solution**: Ensure MIME type detection is working in upload code
|
||||
|
||||
### 3. No Processing Activity
|
||||
**Symptom**: Documents remain in `pending` status
|
||||
**Cause**: Queue runners may not be running
|
||||
**Solution**: Check `docker ps` and restart services if needed
|
||||
|
||||
## Requirements
|
||||
|
||||
- AWS CLI (for SQS queue monitoring)
|
||||
- Docker (for database queries and service log access)
|
||||
- Running query-orchestration stack
|
||||
|
||||
## Files
|
||||
|
||||
- `check_processing.sh` - Main monitoring script
|
||||
- `README.md` - This documentation
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
#!/bin/bash
|
||||
|
||||
# check_processing.sh - Post-upload processing status checker
|
||||
# Run this after test_zip_upload2.sh to see how far document processing got
|
||||
|
||||
set -e
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configuration
|
||||
LOCALSTACK_ENDPOINT="http://localhost:4566"
|
||||
DB_CONTAINER="deployments-db-1"
|
||||
|
||||
# Detect if we're running from scripts/manual_tests or project root
|
||||
if [[ "$(basename $(pwd))" == "manual_tests" ]]; then
|
||||
SCRIPT_DIR="."
|
||||
else
|
||||
SCRIPT_DIR="./scripts/manual_tests"
|
||||
fi
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
check_queue_depth() {
|
||||
local queue_name=$1
|
||||
local depth=$(aws --endpoint-url=$LOCALSTACK_ENDPOINT sqs get-queue-attributes \
|
||||
--queue-url "http://localhost:4566/000000000000/$queue_name" \
|
||||
--attribute-names ApproximateNumberOfMessages \
|
||||
--output text --query 'Attributes.ApproximateNumberOfMessages' 2>/dev/null || echo "0")
|
||||
echo "$depth"
|
||||
}
|
||||
|
||||
check_database_status() {
|
||||
local query=$1
|
||||
docker exec $DB_CONTAINER psql -U postgres -d query_orchestration -t -c "$query" 2>/dev/null | tr -d ' ' || echo ""
|
||||
}
|
||||
|
||||
get_recent_batch_id() {
|
||||
local batch_id=$(check_database_status "SELECT id FROM batch_uploads ORDER BY created_at DESC LIMIT 1;")
|
||||
echo "$batch_id"
|
||||
}
|
||||
|
||||
main() {
|
||||
echo "========================================"
|
||||
echo "Document Processing Status Check"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# Get most recent batch
|
||||
log_info "Finding most recent batch..."
|
||||
BATCH_ID=$(get_recent_batch_id)
|
||||
|
||||
if [[ -z "$BATCH_ID" ]]; then
|
||||
log_error "No batches found in database"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Found batch: $BATCH_ID"
|
||||
echo ""
|
||||
|
||||
# Check batch status
|
||||
log_info "Batch Status:"
|
||||
BATCH_STATUS=$(check_database_status "SELECT status FROM batch_uploads WHERE id='$BATCH_ID';")
|
||||
TOTAL_DOCS=$(check_database_status "SELECT total_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||||
PROCESSED_DOCS=$(check_database_status "SELECT processed_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||||
FAILED_DOCS=$(check_database_status "SELECT failed_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||||
|
||||
echo " Status: $BATCH_STATUS"
|
||||
echo " Progress: $PROCESSED_DOCS/$TOTAL_DOCS documents processed"
|
||||
if [[ "$FAILED_DOCS" != "0" ]]; then
|
||||
echo " Failed: $FAILED_DOCS documents"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Check individual document counts
|
||||
log_info "Document Breakdown:"
|
||||
TOTAL_IN_DB=$(check_database_status "SELECT COUNT(*) FROM documents WHERE batch_id='$BATCH_ID';")
|
||||
UPLOADED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documententries WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');")
|
||||
CLEANED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documentcleans WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');")
|
||||
EXTRACTED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documenttextextractions WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');" 2>/dev/null || echo "0")
|
||||
|
||||
echo " Total documents in DB: $TOTAL_IN_DB"
|
||||
echo " Uploaded to storage: $UPLOADED_COUNT"
|
||||
echo " Passed cleaning: $CLEANED_COUNT"
|
||||
echo " Text extracted: $EXTRACTED_COUNT"
|
||||
echo ""
|
||||
|
||||
# Check current queue depths
|
||||
log_info "Current Queue Depths:"
|
||||
STORE_EVENT=$(check_queue_depth "store_event")
|
||||
DOC_INIT=$(check_queue_depth "document_init")
|
||||
DOC_SYNC=$(check_queue_depth "document_sync")
|
||||
DOC_CLEAN=$(check_queue_depth "document_clean")
|
||||
DOC_TEXT=$(check_queue_depth "document_text")
|
||||
QUERY_SYNC=$(check_queue_depth "query_sync")
|
||||
QUERY_RUNNER=$(check_queue_depth "query_runner")
|
||||
|
||||
echo " store_event: $STORE_EVENT messages"
|
||||
echo " document_init: $DOC_INIT messages"
|
||||
echo " document_sync: $DOC_SYNC messages"
|
||||
echo " document_clean: $DOC_CLEAN messages"
|
||||
echo " document_text: $DOC_TEXT messages"
|
||||
echo " query_sync: $QUERY_SYNC messages"
|
||||
echo " query_runner: $QUERY_RUNNER messages"
|
||||
echo ""
|
||||
|
||||
# Determine where processing stopped and why
|
||||
log_info "Processing Analysis:"
|
||||
|
||||
# Check if client has sync permission
|
||||
CLIENT_ID=$(check_database_status "SELECT client_id FROM batch_uploads WHERE id='$BATCH_ID';")
|
||||
SYNC_BLOCKED=$(check_database_status "SELECT COUNT(*) FROM clients c LEFT JOIN clientcansync cs ON c.clientid = cs.clientid WHERE c.clientid='$CLIENT_ID' AND (cs.cansync IS NULL OR cs.cansync = false);")
|
||||
|
||||
if [[ "$SYNC_BLOCKED" != "0" ]]; then
|
||||
log_warning "Documents blocked at sync stage: Client CanSync flag is false"
|
||||
echo " Solution: Run 'INSERT INTO clientcansync (clientid, cansync) VALUES ('[client_id]', true);'"
|
||||
fi
|
||||
|
||||
# Check for MIME type issues by looking at recent logs
|
||||
MIME_ERRORS=$(docker logs deployments-doc_clean_runner-1 2>&1 | grep "invalid_mimetype" | wc -l 2>/dev/null || echo "0")
|
||||
if [[ "$MIME_ERRORS" -gt "0" ]]; then
|
||||
log_warning "Found $MIME_ERRORS MIME type errors in doc clean runner logs"
|
||||
fi
|
||||
|
||||
# Determine overall status
|
||||
echo ""
|
||||
if [[ "$BATCH_STATUS" == "completed" ]]; then
|
||||
log_success "✅ Processing completed successfully!"
|
||||
elif [[ "$BATCH_STATUS" == "processing" ]]; then
|
||||
if [[ "$SYNC_BLOCKED" != "0" ]]; then
|
||||
log_warning "⚠️ Processing stopped at document sync (client CanSync issue)"
|
||||
elif [[ "$DOC_CLEAN" != "0" ]] || [[ "$MIME_ERRORS" != "0" ]]; then
|
||||
log_warning "⚠️ Processing stopped at document cleaning (MIME type issue)"
|
||||
elif [[ "$DOC_TEXT" != "0" ]]; then
|
||||
log_info "🔄 Processing at text extraction stage"
|
||||
elif [[ "$QUERY_RUNNER" != "0" ]]; then
|
||||
log_info "🔄 Processing at query execution stage"
|
||||
else
|
||||
log_info "🔄 Processing in progress..."
|
||||
fi
|
||||
elif [[ "$BATCH_STATUS" == "failed" ]]; then
|
||||
log_error "❌ Batch processing failed"
|
||||
else
|
||||
log_warning "⚠️ Unknown batch status: $BATCH_STATUS"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
}
|
||||
|
||||
# Check if required tools are available
|
||||
if ! command -v aws &> /dev/null; then
|
||||
log_error "AWS CLI not found. Please install it to use this script."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker ps | grep -q $DB_CONTAINER; then
|
||||
log_error "Database container not running: $DB_CONTAINER"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
main "$@"
|
||||
@@ -1,4 +1,8 @@
|
||||
# Manual tests
|
||||
The assets in this directory are not part of the automated tests.
|
||||
They are manual tests for debugging and demo purposes so that
|
||||
the tests can be paused and interacted with.
|
||||
the tests can be paused and interacted with.
|
||||
|
||||
list all of the files recursively in the target s3 bucket
|
||||
aws --endpoint-url=http://localhost:4566 s3 ls s3://documentin/ --recursive
|
||||
|
||||
|
||||
Binary file not shown.
@@ -1,5 +1,8 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for ZIP batch upload functionality
|
||||
# This script exercises all batch endpoints with a running deployment
|
||||
# use: task compose:up (or compose:refresh if you have made service changes)
|
||||
@@ -31,7 +34,6 @@ set -e # Exit on any error
|
||||
BASE_URL="http://localhost:8080"
|
||||
CLIENT_ID="test_client_$(date +%s)"
|
||||
ZIP_FILE="test_batch.zip"
|
||||
TEST_PDF_COUNT=3
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
@@ -63,83 +65,48 @@ check_service() {
|
||||
log_info "Service is running"
|
||||
}
|
||||
|
||||
# Create test ZIP file with PDFs
|
||||
# Create test ZIP file with real PDFs from test.pdf.batch directory
|
||||
create_test_zip() {
|
||||
log_info "Creating test ZIP file with $TEST_PDF_COUNT PDFs..."
|
||||
|
||||
log_info "Creating test ZIP file with real PDFs from ./test.pdf.batch..."
|
||||
|
||||
# Remove existing file if it exists
|
||||
rm -f "$ZIP_FILE"
|
||||
|
||||
# Create temporary directory for PDF files
|
||||
temp_dir=$(mktemp -d)
|
||||
|
||||
# Create test PDF files
|
||||
for i in $(seq 1 $TEST_PDF_COUNT); do
|
||||
pdf_file="$temp_dir/test${i}.pdf"
|
||||
cat > "$pdf_file" << EOF
|
||||
%PDF-1.4
|
||||
%Test content for test${i}.pdf
|
||||
1 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 2 0 R
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/Type /Pages
|
||||
/Kids [3 0 R]
|
||||
/Count 1
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/Type /Page
|
||||
/Parent 2 0 R
|
||||
/MediaBox [0 0 612 792]
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 4
|
||||
0000000000 65535 f
|
||||
0000000010 00000 n
|
||||
0000000053 00000 n
|
||||
0000000125 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 4
|
||||
/Root 1 0 R
|
||||
>>
|
||||
startxref
|
||||
284
|
||||
%%EOF
|
||||
EOF
|
||||
done
|
||||
|
||||
# Create ZIP file using absolute path
|
||||
|
||||
# Check if test.pdf.batch directory exists
|
||||
pdf_dir="./test.pdf.batch"
|
||||
if [ ! -d "$pdf_dir" ]; then
|
||||
log_error "Directory $pdf_dir does not exist"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if there are PDF files in the directory
|
||||
pdf_count=$(find "$pdf_dir" -name "*.pdf" -type f | wc -l)
|
||||
if [ "$pdf_count" -eq 0 ]; then
|
||||
log_error "No PDF files found in $pdf_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create ZIP file with all PDFs from the batch directory
|
||||
current_dir=$(pwd)
|
||||
cd "$temp_dir"
|
||||
cd "$pdf_dir"
|
||||
zip -q "$current_dir/$ZIP_FILE" *.pdf
|
||||
cd "$current_dir"
|
||||
|
||||
# Cleanup temp directory
|
||||
rm -rf "$temp_dir"
|
||||
|
||||
|
||||
file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null || echo "unknown")
|
||||
log_info "Created $ZIP_FILE with $TEST_PDF_COUNT PDFs ($file_size bytes)"
|
||||
log_info "Created $ZIP_FILE with $pdf_count real PDFs ($file_size bytes)"
|
||||
}
|
||||
|
||||
# Create client (idempotent)
|
||||
create_client() {
|
||||
log_info "Creating/checking client: $CLIENT_ID"
|
||||
|
||||
|
||||
# Try to create client, ignore if already exists
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test Client $CLIENT_ID\"}" 2>/dev/null || echo "000")
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
|
||||
if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
|
||||
log_info "Client created/exists: $CLIENT_ID"
|
||||
elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
|
||||
@@ -149,16 +116,43 @@ create_client() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Enable CanSync flag for client to allow document processing
|
||||
enable_client_sync() {
|
||||
log_info "Enabling CanSync flag for client: $CLIENT_ID"
|
||||
|
||||
# Check if entry already exists
|
||||
existing=$(docker exec deployments-db-1 psql -U postgres -d query_orchestration -t -c \
|
||||
"SELECT COUNT(*) FROM clientcansync WHERE clientid='$CLIENT_ID';" 2>/dev/null | tr -d ' ')
|
||||
|
||||
if [ "$existing" = "0" ]; then
|
||||
# Insert new record
|
||||
docker exec deployments-db-1 psql -U postgres -d query_orchestration -c \
|
||||
"INSERT INTO clientcansync (clientid, cansync) VALUES ('$CLIENT_ID', true);" >/dev/null 2>&1
|
||||
else
|
||||
# Update existing record
|
||||
docker exec deployments-db-1 psql -U postgres -d query_orchestration -c \
|
||||
"UPDATE clientcansync SET cansync = true WHERE clientid='$CLIENT_ID';" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_info "CanSync flag enabled - documents can now be processed"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to enable CanSync flag"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Upload ZIP batch
|
||||
upload_batch() {
|
||||
log_info "Uploading ZIP batch..."
|
||||
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@$ZIP_FILE" 2>/dev/null)
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
|
||||
if [ "$http_code" = "202" ]; then
|
||||
BATCH_ID=$(echo "$response_body" | grep -o '"batch_id":"[^"]*"' | cut -d'"' -f4)
|
||||
STATUS_URL=$(echo "$response_body" | grep -o '"status_url":"[^"]*"' | cut -d'"' -f4)
|
||||
@@ -176,12 +170,12 @@ upload_batch() {
|
||||
# Get batch status
|
||||
get_batch_status() {
|
||||
log_info "Getting batch status..."
|
||||
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
status=$(echo "$response_body" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
filename=$(echo "$response_body" | grep -o '"originalFilename":"[^"]*"' | cut -d'"' -f4)
|
||||
@@ -199,12 +193,12 @@ get_batch_status() {
|
||||
# List batches for client
|
||||
list_batches() {
|
||||
log_info "Listing batches for client..."
|
||||
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" "$BASE_URL/client/$CLIENT_ID/document/batch?limit=10&offset=0" 2>/dev/null)
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
|
||||
if [ "$http_code" = "200" ]; then
|
||||
batch_count=$(echo "$response_body" | grep -o '"totalCount":[0-9]*' | cut -d':' -f2)
|
||||
log_info "Found $batch_count batches"
|
||||
@@ -220,19 +214,19 @@ list_batches() {
|
||||
# Test invalid file upload
|
||||
test_invalid_file() {
|
||||
log_info "Testing invalid file upload (should fail)..."
|
||||
|
||||
|
||||
# Create invalid file
|
||||
echo "This is not a ZIP file" > invalid.txt
|
||||
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@invalid.txt" 2>/dev/null)
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
|
||||
# Cleanup
|
||||
rm -f invalid.txt
|
||||
|
||||
|
||||
if [ "$http_code" = "400" ]; then
|
||||
log_info "Invalid file correctly rejected (HTTP $http_code)"
|
||||
return 0
|
||||
@@ -246,11 +240,11 @@ test_invalid_file() {
|
||||
# Cancel batch
|
||||
cancel_batch() {
|
||||
log_info "Canceling batch..."
|
||||
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X DELETE "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" 2>/dev/null)
|
||||
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
|
||||
if [ "$http_code" = "204" ]; then
|
||||
log_info "Batch canceled successfully"
|
||||
return 0
|
||||
@@ -274,14 +268,15 @@ cleanup() {
|
||||
main() {
|
||||
log_info "Starting ZIP batch upload test..."
|
||||
log_info "Using client ID: $CLIENT_ID"
|
||||
|
||||
|
||||
# Setup trap for cleanup
|
||||
trap cleanup EXIT
|
||||
|
||||
|
||||
# Run tests
|
||||
check_service
|
||||
create_test_zip
|
||||
create_client
|
||||
enable_client_sync
|
||||
|
||||
if upload_batch; then
|
||||
sleep 1 # Brief pause to let the system process
|
||||
|
||||
+1
-1
@@ -33,7 +33,7 @@ vars:
|
||||
TEST_PARALLEL:
|
||||
sh: echo "2" # Minimal test parallelism
|
||||
# yamllint disable-line rule:line-length
|
||||
EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go|api/queryAPI/controllers.go|api/queryAPI/test_helpers.go"
|
||||
EXCLUDED_FILES: ".gen.go|internal/serviceconfig/observability/prometheus/generator/main.go|internal/cognitoauth/middleware.go|internal/cognitoauth/token.go|internal/cognitoauth/auth.go|internal/cognitoauth/handler.go|internal/cognitoauth/models.go|api/queryAPI/authHandlers.go|api/queryAPI/homehandler.go|api/queryAPI/controllers.go|api/queryAPI/test_helpers.go|internal/serviceconfig/common.go"
|
||||
# yamllint disable-line rule:line-length
|
||||
TESTS: "./internal/database/repository ./test ./test/queryAPI ./test/... ./internal/serviceconfig/queue ./internal/server/... ./..."
|
||||
|
||||
|
||||
Reference in New Issue
Block a user