180 lines
6.6 KiB
Bash
180 lines
6.6 KiB
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# check_processing.sh - Post-upload processing status checker
|
||
|
|
# Run this after test_zip_upload2.sh to see how far document processing got
|
||
|
|
|
||
|
|
set -e
|
||
|
|
|
||
|
|
# Colors for output
|
||
|
|
RED='\033[0;31m'
|
||
|
|
GREEN='\033[0;32m'
|
||
|
|
YELLOW='\033[0;33m'
|
||
|
|
BLUE='\033[0;34m'
|
||
|
|
NC='\033[0m' # No Color
|
||
|
|
|
||
|
|
# Configuration
|
||
|
|
LOCALSTACK_ENDPOINT="http://localhost:4566"
|
||
|
|
DB_CONTAINER="deployments-db-1"
|
||
|
|
|
||
|
|
# Detect if we're running from scripts/manual_tests or project root
|
||
|
|
if [[ "$(basename $(pwd))" == "manual_tests" ]]; then
|
||
|
|
SCRIPT_DIR="."
|
||
|
|
else
|
||
|
|
SCRIPT_DIR="./scripts/manual_tests"
|
||
|
|
fi
|
||
|
|
|
||
|
|
log_info() {
|
||
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
||
|
|
}
|
||
|
|
|
||
|
|
log_success() {
|
||
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
||
|
|
}
|
||
|
|
|
||
|
|
log_warning() {
|
||
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
||
|
|
}
|
||
|
|
|
||
|
|
log_error() {
|
||
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
||
|
|
}
|
||
|
|
|
||
|
|
check_queue_depth() {
|
||
|
|
local queue_name=$1
|
||
|
|
local depth=$(aws --endpoint-url=$LOCALSTACK_ENDPOINT sqs get-queue-attributes \
|
||
|
|
--queue-url "http://localhost:4566/000000000000/$queue_name" \
|
||
|
|
--attribute-names ApproximateNumberOfMessages \
|
||
|
|
--output text --query 'Attributes.ApproximateNumberOfMessages' 2>/dev/null || echo "0")
|
||
|
|
echo "$depth"
|
||
|
|
}
|
||
|
|
|
||
|
|
check_database_status() {
|
||
|
|
local query=$1
|
||
|
|
docker exec $DB_CONTAINER psql -U postgres -d query_orchestration -t -c "$query" 2>/dev/null | tr -d ' ' || echo ""
|
||
|
|
}
|
||
|
|
|
||
|
|
get_recent_batch_id() {
|
||
|
|
local batch_id=$(check_database_status "SELECT id FROM batch_uploads ORDER BY created_at DESC LIMIT 1;")
|
||
|
|
echo "$batch_id"
|
||
|
|
}
|
||
|
|
|
||
|
|
main() {
|
||
|
|
echo "========================================"
|
||
|
|
echo "Document Processing Status Check"
|
||
|
|
echo "========================================"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Get most recent batch
|
||
|
|
log_info "Finding most recent batch..."
|
||
|
|
BATCH_ID=$(get_recent_batch_id)
|
||
|
|
|
||
|
|
if [[ -z "$BATCH_ID" ]]; then
|
||
|
|
log_error "No batches found in database"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
log_success "Found batch: $BATCH_ID"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Check batch status
|
||
|
|
log_info "Batch Status:"
|
||
|
|
BATCH_STATUS=$(check_database_status "SELECT status FROM batch_uploads WHERE id='$BATCH_ID';")
|
||
|
|
TOTAL_DOCS=$(check_database_status "SELECT total_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||
|
|
PROCESSED_DOCS=$(check_database_status "SELECT processed_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||
|
|
FAILED_DOCS=$(check_database_status "SELECT failed_documents FROM batch_uploads WHERE id='$BATCH_ID';")
|
||
|
|
|
||
|
|
echo " Status: $BATCH_STATUS"
|
||
|
|
echo " Progress: $PROCESSED_DOCS/$TOTAL_DOCS documents processed"
|
||
|
|
if [[ "$FAILED_DOCS" != "0" ]]; then
|
||
|
|
echo " Failed: $FAILED_DOCS documents"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Check individual document counts
|
||
|
|
log_info "Document Breakdown:"
|
||
|
|
TOTAL_IN_DB=$(check_database_status "SELECT COUNT(*) FROM documents WHERE batch_id='$BATCH_ID';")
|
||
|
|
UPLOADED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documententries WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');")
|
||
|
|
CLEANED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documentcleans WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');")
|
||
|
|
EXTRACTED_COUNT=$(check_database_status "SELECT COUNT(*) FROM documenttextextractions WHERE documentid IN (SELECT id FROM documents WHERE batch_id='$BATCH_ID');" 2>/dev/null || echo "0")
|
||
|
|
|
||
|
|
echo " Total documents in DB: $TOTAL_IN_DB"
|
||
|
|
echo " Uploaded to storage: $UPLOADED_COUNT"
|
||
|
|
echo " Passed cleaning: $CLEANED_COUNT"
|
||
|
|
echo " Text extracted: $EXTRACTED_COUNT"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Check current queue depths
|
||
|
|
log_info "Current Queue Depths:"
|
||
|
|
STORE_EVENT=$(check_queue_depth "store_event")
|
||
|
|
DOC_INIT=$(check_queue_depth "document_init")
|
||
|
|
DOC_SYNC=$(check_queue_depth "document_sync")
|
||
|
|
DOC_CLEAN=$(check_queue_depth "document_clean")
|
||
|
|
DOC_TEXT=$(check_queue_depth "document_text")
|
||
|
|
QUERY_SYNC=$(check_queue_depth "query_sync")
|
||
|
|
QUERY_RUNNER=$(check_queue_depth "query_runner")
|
||
|
|
|
||
|
|
echo " store_event: $STORE_EVENT messages"
|
||
|
|
echo " document_init: $DOC_INIT messages"
|
||
|
|
echo " document_sync: $DOC_SYNC messages"
|
||
|
|
echo " document_clean: $DOC_CLEAN messages"
|
||
|
|
echo " document_text: $DOC_TEXT messages"
|
||
|
|
echo " query_sync: $QUERY_SYNC messages"
|
||
|
|
echo " query_runner: $QUERY_RUNNER messages"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Determine where processing stopped and why
|
||
|
|
log_info "Processing Analysis:"
|
||
|
|
|
||
|
|
# Check if client has sync permission
|
||
|
|
CLIENT_ID=$(check_database_status "SELECT client_id FROM batch_uploads WHERE id='$BATCH_ID';")
|
||
|
|
SYNC_BLOCKED=$(check_database_status "SELECT COUNT(*) FROM clients c LEFT JOIN clientcansync cs ON c.clientid = cs.clientid WHERE c.clientid='$CLIENT_ID' AND (cs.cansync IS NULL OR cs.cansync = false);")
|
||
|
|
|
||
|
|
if [[ "$SYNC_BLOCKED" != "0" ]]; then
|
||
|
|
log_warning "Documents blocked at sync stage: Client CanSync flag is false"
|
||
|
|
echo " Solution: Run 'INSERT INTO clientcansync (clientid, cansync) VALUES ('[client_id]', true);'"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Check for MIME type issues by looking at recent logs
|
||
|
|
MIME_ERRORS=$(docker logs deployments-doc_clean_runner-1 2>&1 | grep "invalid_mimetype" | wc -l 2>/dev/null || echo "0")
|
||
|
|
if [[ "$MIME_ERRORS" -gt "0" ]]; then
|
||
|
|
log_warning "Found $MIME_ERRORS MIME type errors in doc clean runner logs"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Determine overall status
|
||
|
|
echo ""
|
||
|
|
if [[ "$BATCH_STATUS" == "completed" ]]; then
|
||
|
|
log_success "✅ Processing completed successfully!"
|
||
|
|
elif [[ "$BATCH_STATUS" == "processing" ]]; then
|
||
|
|
if [[ "$SYNC_BLOCKED" != "0" ]]; then
|
||
|
|
log_warning "⚠️ Processing stopped at document sync (client CanSync issue)"
|
||
|
|
elif [[ "$DOC_CLEAN" != "0" ]] || [[ "$MIME_ERRORS" != "0" ]]; then
|
||
|
|
log_warning "⚠️ Processing stopped at document cleaning (MIME type issue)"
|
||
|
|
elif [[ "$DOC_TEXT" != "0" ]]; then
|
||
|
|
log_info "🔄 Processing at text extraction stage"
|
||
|
|
elif [[ "$QUERY_RUNNER" != "0" ]]; then
|
||
|
|
log_info "🔄 Processing at query execution stage"
|
||
|
|
else
|
||
|
|
log_info "🔄 Processing in progress..."
|
||
|
|
fi
|
||
|
|
elif [[ "$BATCH_STATUS" == "failed" ]]; then
|
||
|
|
log_error "❌ Batch processing failed"
|
||
|
|
else
|
||
|
|
log_warning "⚠️ Unknown batch status: $BATCH_STATUS"
|
||
|
|
fi
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "========================================"
|
||
|
|
}
|
||
|
|
|
||
|
|
# Check if required tools are available
|
||
|
|
if ! command -v aws &> /dev/null; then
|
||
|
|
log_error "AWS CLI not found. Please install it to use this script."
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
if ! docker ps | grep -q $DB_CONTAINER; then
|
||
|
|
log_error "Database container not running: $DB_CONTAINER"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
main "$@"
|