Merged in feature/support-more-types (pull request #219)
support new types for import * tests pass * missing file * bug fixes
This commit is contained in:
Jay Brown
+765
@@ -0,0 +1,765 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test script for multi-type ZIP batch upload functionality
|
||||
# Exercises all supported file types through the full processing pipeline
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Local stack running: task compose:refresh
|
||||
# - Python3 available (stdlib only for runtime, PIL only for JPEG generation)
|
||||
#
|
||||
# REST API operations tested:
|
||||
# - GET /metrics - Service health
|
||||
# - POST /client - Create test client
|
||||
# - POST /client/{CLIENT_ID}/document/batch - Upload ZIP
|
||||
# - GET /client/{CLIENT_ID}/document/batch/{BATCH_ID} - Poll batch status
|
||||
# - POST /client/{CLIENT_ID}/document/batch (invalid) - Reject non-ZIP
|
||||
#
|
||||
# File types tested (valid + invalid for each):
|
||||
# PDF, TIFF, JPEG, PNG, BMP, DOCX, XLSX, PPTX, TXT, EML
|
||||
|
||||
set -e
|
||||
|
||||
# --- Configuration ---
|
||||
BASE_URL="http://localhost:8080"
|
||||
CLIENT_ID="test_alltypes_$(date +%s)"
|
||||
ZIP_FILE="test_all_types_batch.zip"
|
||||
TEMP_DIR="temp_all_types_structure"
|
||||
TOTAL_VALID=10
|
||||
TOTAL_INVALID=10
|
||||
TOTAL_FILES=$((TOTAL_VALID + TOTAL_INVALID))
|
||||
|
||||
# --- Terminal outcome constants ---
|
||||
TERMINAL_OUTCOMES="duplicate init_duplicate failed_open failed_read failed_s3_upload failed_upload invalid_type clean_passed clean_failed sync_skipped"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
log_step() {
|
||||
echo -e "${BLUE}[STEP]${NC} $1"
|
||||
}
|
||||
|
||||
# Check if service is running
|
||||
check_service() {
|
||||
log_step "Checking if queryAPI service is running..."
|
||||
if ! curl -s "$BASE_URL/metrics" > /dev/null 2>&1; then
|
||||
log_error "queryAPI service is not running at $BASE_URL"
|
||||
log_info "Please start the stack with: task compose:refresh"
|
||||
exit 1
|
||||
fi
|
||||
log_info "Service is running"
|
||||
}
|
||||
|
||||
# Create client (idempotent)
|
||||
create_client() {
|
||||
log_step "Creating test client: $CLIENT_ID"
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"id\":\"$CLIENT_ID\",\"name\":\"Test AllTypes $CLIENT_ID\"}" 2>/dev/null || echo "000")
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
|
||||
if [ "$http_code" = "201" ] || [ "$http_code" = "200" ]; then
|
||||
log_info "Client created: $CLIENT_ID"
|
||||
elif [ "$http_code" = "409" ] || [ "$http_code" = "400" ]; then
|
||||
log_info "Client already exists: $CLIENT_ID"
|
||||
else
|
||||
log_warn "Unexpected response when creating client (HTTP $http_code), continuing..."
|
||||
fi
|
||||
}
|
||||
|
||||
# Enable CanSync flag for client
|
||||
enable_client_sync() {
|
||||
log_step "Enabling CanSync flag for client: $CLIENT_ID"
|
||||
|
||||
existing=$(docker exec deployments-db-1 psql -U postgres -d query_orchestration -t -c \
|
||||
"SELECT COUNT(*) FROM clientcansync WHERE clientid='$CLIENT_ID';" 2>/dev/null | tr -d ' ')
|
||||
|
||||
if [ "$existing" = "0" ]; then
|
||||
docker exec deployments-db-1 psql -U postgres -d query_orchestration -c \
|
||||
"INSERT INTO clientcansync (clientid, cansync) VALUES ('$CLIENT_ID', true);" >/dev/null 2>&1
|
||||
else
|
||||
docker exec deployments-db-1 psql -U postgres -d query_orchestration -c \
|
||||
"UPDATE clientcansync SET cansync = true WHERE clientid='$CLIENT_ID';" >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
log_info "CanSync flag enabled"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to enable CanSync flag"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Generate all test files using inline Python3
|
||||
generate_test_files() {
|
||||
log_step "Generating test files for all supported types..."
|
||||
rm -rf "$TEMP_DIR"
|
||||
mkdir -p "$TEMP_DIR/valid" "$TEMP_DIR/invalid"
|
||||
|
||||
# Copy known-good PDF
|
||||
cp ./test.pdf.batch/test.pdf "$TEMP_DIR/valid/test_document.pdf"
|
||||
|
||||
# Generate all other files via Python
|
||||
export TEMP_DIR
|
||||
python3 << 'PYEOF'
|
||||
import struct, zlib, zipfile, os, base64
|
||||
from email.message import EmailMessage
|
||||
|
||||
TEMP_DIR = os.environ["TEMP_DIR"]
|
||||
valid_dir = os.path.join(TEMP_DIR, "valid")
|
||||
invalid_dir = os.path.join(TEMP_DIR, "invalid")
|
||||
|
||||
# ============================
|
||||
# Helper: PNG chunk builder
|
||||
# ============================
|
||||
def png_chunk(chunk_type, data):
|
||||
chunk = chunk_type + data
|
||||
length = struct.pack(">I", len(data))
|
||||
crc = struct.pack(">I", zlib.crc32(chunk) & 0xFFFFFFFF)
|
||||
return length + chunk + crc
|
||||
|
||||
# ============================
|
||||
# VALID FILES
|
||||
# ============================
|
||||
|
||||
# --- PNG (100x100 RGB gradient) ---
|
||||
def create_valid_png(filepath):
|
||||
width, height = 100, 100
|
||||
signature = b'\x89PNG\r\n\x1a\n'
|
||||
ihdr_data = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
ihdr = png_chunk(b'IHDR', ihdr_data)
|
||||
raw = b''
|
||||
for y in range(height):
|
||||
raw += b'\x00' # filter: none
|
||||
for x in range(width):
|
||||
r = (x * 255 // max(width - 1, 1)) & 0xFF
|
||||
g = (y * 255 // max(height - 1, 1)) & 0xFF
|
||||
b = ((x + y) * 127 // max(width + height - 2, 1)) & 0xFF
|
||||
raw += struct.pack('BBB', r, g, b)
|
||||
idat = png_chunk(b'IDAT', zlib.compress(raw))
|
||||
iend = png_chunk(b'IEND', b'')
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(signature + ihdr + idat + iend)
|
||||
|
||||
create_valid_png(os.path.join(valid_dir, "test_screenshot.png"))
|
||||
|
||||
# --- BMP (100x100 24-bit RGB gradient) ---
|
||||
def create_valid_bmp(filepath):
|
||||
width, height = 100, 100
|
||||
row_size = (width * 3 + 3) & ~3
|
||||
pixel_size = row_size * height
|
||||
file_size = 54 + pixel_size
|
||||
fh = struct.pack('<2sIHHI', b'BM', file_size, 0, 0, 54)
|
||||
ih = struct.pack('<IiiHHIIiiII', 40, width, height, 1, 24, 0,
|
||||
pixel_size, 2835, 2835, 0, 0)
|
||||
pixels = b''
|
||||
for y in range(height):
|
||||
row = b''
|
||||
for x in range(width):
|
||||
bv = (x * 255 // max(width - 1, 1)) & 0xFF
|
||||
gv = (y * 255 // max(height - 1, 1)) & 0xFF
|
||||
rv = ((x + y) * 127 // max(width + height - 2, 1)) & 0xFF
|
||||
row += struct.pack('BBB', bv, gv, rv)
|
||||
row += b'\x00' * (row_size - width * 3)
|
||||
pixels += row
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(fh + ih + pixels)
|
||||
|
||||
create_valid_bmp(os.path.join(valid_dir, "test_bitmap.bmp"))
|
||||
|
||||
# --- TIFF (100x100 uncompressed RGB, little-endian) ---
|
||||
def create_valid_tiff(filepath):
|
||||
width, height = 100, 100
|
||||
row_bytes = width * 3
|
||||
strip_size = row_bytes * height
|
||||
pixels = bytearray()
|
||||
for y in range(height):
|
||||
for x in range(width):
|
||||
r = (x * 255 // max(width - 1, 1)) & 0xFF
|
||||
g = (y * 255 // max(height - 1, 1)) & 0xFF
|
||||
b = ((x + y) * 127 // max(width + height - 2, 1)) & 0xFF
|
||||
pixels += struct.pack('BBB', r, g, b)
|
||||
num_entries = 12
|
||||
ifd_offset = 8
|
||||
ifd_size = 2 + num_entries * 12 + 4
|
||||
bps_offset = ifd_offset + ifd_size
|
||||
xres_offset = bps_offset + 6
|
||||
yres_offset = xres_offset + 8
|
||||
strip_offset = yres_offset + 8
|
||||
|
||||
def ifd_entry(tag, dtype, count, value):
|
||||
return struct.pack('<HHII', tag, dtype, count, value)
|
||||
|
||||
header = struct.pack('<2sHI', b'II', 42, ifd_offset)
|
||||
ifd = struct.pack('<H', num_entries)
|
||||
ifd += ifd_entry(256, 3, 1, width) # ImageWidth
|
||||
ifd += ifd_entry(257, 3, 1, height) # ImageLength
|
||||
ifd += ifd_entry(258, 3, 3, bps_offset) # BitsPerSample -> offset
|
||||
ifd += ifd_entry(259, 3, 1, 1) # Compression = none
|
||||
ifd += ifd_entry(262, 3, 1, 2) # PhotometricInterpretation = RGB
|
||||
ifd += ifd_entry(273, 4, 1, strip_offset) # StripOffsets
|
||||
ifd += ifd_entry(277, 3, 1, 3) # SamplesPerPixel = 3
|
||||
ifd += ifd_entry(278, 3, 1, height) # RowsPerStrip
|
||||
ifd += ifd_entry(279, 4, 1, strip_size) # StripByteCounts
|
||||
ifd += ifd_entry(282, 5, 1, xres_offset) # XResolution
|
||||
ifd += ifd_entry(283, 5, 1, yres_offset) # YResolution
|
||||
ifd += ifd_entry(296, 3, 1, 2) # ResolutionUnit = inch
|
||||
ifd += struct.pack('<I', 0) # Next IFD = none
|
||||
bps_data = struct.pack('<HHH', 8, 8, 8)
|
||||
xres = struct.pack('<II', 72, 1)
|
||||
yres = struct.pack('<II', 72, 1)
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(header + ifd + bps_data + xres + yres + bytes(pixels))
|
||||
|
||||
create_valid_tiff(os.path.join(valid_dir, "test_image.tiff"))
|
||||
|
||||
# --- JPEG (base64-encoded minimal valid 100x100 gradient) ---
|
||||
def create_valid_jpeg(filepath):
|
||||
jpeg_b64 = (
|
||||
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAUDBAQEAwUEBAQFBQUGBwwIBwcHBw8LCwkM"
|
||||
"EQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUFBQcGBw4I"
|
||||
"CA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e"
|
||||
"Hh4eHh7/wAARCABkAGQDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcI"
|
||||
"CQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS"
|
||||
"0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1"
|
||||
"dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW"
|
||||
"19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcI"
|
||||
"CQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMz"
|
||||
"UvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0"
|
||||
"dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU"
|
||||
"1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD4/ig9qtxQe1WYoOnFW4oP"
|
||||
"aud1B4bEFaKD2q1FB7Vaig9qtxQe1Q6h9BhsQVYoOnFW4oParMUHTircUHPSodQ+gw2IK0UH"
|
||||
"tVuKD2qzFB7Vbig9qh1D38NiCtFB04q1FB7Vaig6cVbig9ql1D6DDYgqxQe1W4oOnFWYoPar"
|
||||
"cUHtUOoe/hsQVFg46UVqrB8vSip9oeusRoeJxQe1WooParUUHtVuKDpxVuofxbhsQVYoParcU"
|
||||
"HtVmKD2q3FB7VDqH0GGxBWig6cVbig6cVZig9qtxQdOKh1D6DDYgqxQe1W4oParUUHPSrUUH"
|
||||
"tUOoe/hsQVooOnFW4oParMUHtVuKDpxUOofQYbEFaKD2q1FB7Vaig9qtRQe1Q6h7+GxBVWD5e"
|
||||
"lFaiwfL0oqfaHrrEaHicUHtVuKDpxVqKD2q1FB04q3UP4tw2IK0UHtVuKD2qzFB7Vbig9qh1"
|
||||
"D6DDYgqxQe1W4oOnFWooParUUHtUOofQYbEFaKD2q3FB7VZig9qtxQe1Q6h7+GxBVig9qtxQd"
|
||||
"OKtRQe1W4oOnFQ6h9BhsQVYoParcUHtVqKD2q1FB7VDqHv4bEFRYPl6UVqLBx0oqfaHrrEaH"
|
||||
"ikUHtVqKD2q1FB7Vbig9qt1D+LcNiCrFB04q3FB7Vaig9qtRQe1Q6h9BhsQVooParcUHtVmKD"
|
||||
"2q3FB7VDqH0GGxBVig9qtxQe1WooOnFW4oPapdQ9/DYgqxQe1W4oParMUHtVuKD2qHUPoMNi"
|
||||
"CtFB04q1FB7Vaig6cVbig9qh1D6DDYgqLBx0orUWDjpRU+0PWWI0PFIoParUUHtVqKDnpVqKD"
|
||||
"2q3UP4uw2IK0UHtVuKD2qzFB7Vbig9qh1D38NiCtFB7Vaig9qtRQe1W4oPaodQ+gw2IKsUHTi"
|
||||
"rcUHtVmKDpxVuKD2qHUPfw2IK0UHtVqKD2q1FB7Vbig9qh1D6DDYgqxQdOKtxQe1WooParUU"
|
||||
"HtUOofQYbEFRYOOlFaiwcdKKn2h6yxGh4pFB7Vbig9qsxQe1W4oPardQ/i7DYgrRQe1WooP"
|
||||
"arUUHtVqKD2qHUPfw2IK0UHtVuKD2qzFB7Vbig9qh1D6DDYgrRQe1WooParUUHtVuKD2qHUP"
|
||||
"fw2IKsUHtVuKD2q1FB7Vaig9qh1D6DDYgrRQe1W4oParMUHtVuKDpxUOofQYbEFRYOOlFai2"
|
||||
"/HSip9oessRoeIxIvpVqJF9KKK6GfxvhmW4kX0q3Ei8cUUVDPfwzLcSL6VbiRfSiioZ9Bhmy1"
|
||||
"Ei8cVbiRfSiioZ9BhmW4kX0q3Ei+lFFQz38My3Ei+lWokXjiiioZ9BhmWVRcdKKKKk9ZPQ/9k="
|
||||
)
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(base64.b64decode(jpeg_b64))
|
||||
|
||||
create_valid_jpeg(os.path.join(valid_dir, "test_photo.jpeg"))
|
||||
|
||||
# --- DOCX (minimal valid Word document) ---
|
||||
def create_valid_docx(filepath):
|
||||
content_types = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/>
|
||||
</Types>'''
|
||||
rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="word/document.xml"/>
|
||||
</Relationships>'''
|
||||
document = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
|
||||
<w:body>
|
||||
<w:p><w:r><w:t>Test document content for batch upload validation. This document contains meaningful text to pass the empty content check.</w:t></w:r></w:p>
|
||||
<w:p><w:r><w:t>Second paragraph with additional content to ensure validation passes.</w:t></w:r></w:p>
|
||||
</w:body>
|
||||
</w:document>'''
|
||||
doc_rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"/>'''
|
||||
with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr('[Content_Types].xml', content_types)
|
||||
zf.writestr('_rels/.rels', rels)
|
||||
zf.writestr('word/document.xml', document)
|
||||
zf.writestr('word/_rels/document.xml.rels', doc_rels)
|
||||
|
||||
create_valid_docx(os.path.join(valid_dir, "test_report.docx"))
|
||||
|
||||
# --- XLSX (minimal valid Excel spreadsheet) ---
|
||||
def create_valid_xlsx(filepath):
|
||||
content_types = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>
|
||||
<Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>
|
||||
<Override PartName="/xl/sharedStrings.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml"/>
|
||||
</Types>'''
|
||||
rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/>
|
||||
</Relationships>'''
|
||||
workbook = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<sheets>
|
||||
<sheet name="Sheet1" sheetId="1" r:id="rId1"/>
|
||||
</sheets>
|
||||
</workbook>'''
|
||||
wb_rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/>
|
||||
<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings" Target="sharedStrings.xml"/>
|
||||
</Relationships>'''
|
||||
sheet1 = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
|
||||
<sheetData>
|
||||
<row r="1"><c r="A1" t="s"><v>0</v></c><c r="B1" t="s"><v>1</v></c></row>
|
||||
<row r="2"><c r="A2" t="s"><v>2</v></c><c r="B2"><v>42</v></c></row>
|
||||
</sheetData>
|
||||
</worksheet>'''
|
||||
shared_strings = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<sst xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" count="3" uniqueCount="3">
|
||||
<si><t>Name</t></si>
|
||||
<si><t>Value</t></si>
|
||||
<si><t>Test Data</t></si>
|
||||
</sst>'''
|
||||
with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr('[Content_Types].xml', content_types)
|
||||
zf.writestr('_rels/.rels', rels)
|
||||
zf.writestr('xl/workbook.xml', workbook)
|
||||
zf.writestr('xl/_rels/workbook.xml.rels', wb_rels)
|
||||
zf.writestr('xl/worksheets/sheet1.xml', sheet1)
|
||||
zf.writestr('xl/sharedStrings.xml', shared_strings)
|
||||
|
||||
create_valid_xlsx(os.path.join(valid_dir, "test_spreadsheet.xlsx"))
|
||||
|
||||
# --- PPTX (minimal valid PowerPoint presentation) ---
|
||||
def create_valid_pptx(filepath):
|
||||
content_types = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types">
|
||||
<Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/>
|
||||
<Default Extension="xml" ContentType="application/xml"/>
|
||||
<Override PartName="/ppt/presentation.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.presentation.main+xml"/>
|
||||
<Override PartName="/ppt/slides/slide1.xml" ContentType="application/vnd.openxmlformats-officedocument.presentationml.slide+xml"/>
|
||||
</Types>'''
|
||||
rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="ppt/presentation.xml"/>
|
||||
</Relationships>'''
|
||||
presentation = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<p:presentation xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"
|
||||
xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">
|
||||
<p:sldIdLst>
|
||||
<p:sldId id="256" r:id="rId2"/>
|
||||
</p:sldIdLst>
|
||||
</p:presentation>'''
|
||||
pres_rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
|
||||
<Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide" Target="slides/slide1.xml"/>
|
||||
</Relationships>'''
|
||||
slide1 = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<p:sld xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main"
|
||||
xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main">
|
||||
<p:cSld>
|
||||
<p:spTree>
|
||||
<p:nvGrpSpPr><p:cNvPr id="1" name=""/><p:cNvGrpSpPr/><p:nvPr/></p:nvGrpSpPr>
|
||||
<p:grpSpPr/>
|
||||
<p:sp>
|
||||
<p:nvSpPr><p:cNvPr id="2" name="Title"/><p:cNvSpPr/><p:nvPr/></p:nvSpPr>
|
||||
<p:spPr/>
|
||||
<p:txBody>
|
||||
<a:bodyPr/>
|
||||
<a:p><a:r><a:t>Test slide content for batch upload validation.</a:t></a:r></a:p>
|
||||
</p:txBody>
|
||||
</p:sp>
|
||||
</p:spTree>
|
||||
</p:cSld>
|
||||
</p:sld>'''
|
||||
slide1_rels = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"/>'''
|
||||
with zipfile.ZipFile(filepath, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.writestr('[Content_Types].xml', content_types)
|
||||
zf.writestr('_rels/.rels', rels)
|
||||
zf.writestr('ppt/presentation.xml', presentation)
|
||||
zf.writestr('ppt/_rels/presentation.xml.rels', pres_rels)
|
||||
zf.writestr('ppt/slides/slide1.xml', slide1)
|
||||
zf.writestr('ppt/slides/_rels/slide1.xml.rels', slide1_rels)
|
||||
|
||||
create_valid_pptx(os.path.join(valid_dir, "test_presentation.pptx"))
|
||||
|
||||
# --- TXT ---
|
||||
def create_valid_txt(filepath):
|
||||
with open(filepath, 'w', encoding='utf-8') as f:
|
||||
f.write("Test document for batch upload validation.\n")
|
||||
f.write("This file contains multiple lines of meaningful text content.\n")
|
||||
f.write("Line three provides additional content to ensure validation.\n")
|
||||
f.write("The quick brown fox jumps over the lazy dog.\n")
|
||||
|
||||
create_valid_txt(os.path.join(valid_dir, "test_notes.txt"))
|
||||
|
||||
# --- EML ---
|
||||
def create_valid_eml(filepath):
|
||||
msg = EmailMessage()
|
||||
msg['From'] = 'sender@example.com'
|
||||
msg['To'] = 'recipient@example.com'
|
||||
msg['Subject'] = 'Test email for batch upload validation'
|
||||
msg['Date'] = 'Mon, 01 Jan 2024 12:00:00 +0000'
|
||||
msg.set_content(
|
||||
"This is a test email body for batch upload validation.\n"
|
||||
"It contains multiple lines of plain text content.\n"
|
||||
"No attachments are included in this message.\n"
|
||||
)
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(msg.as_string())
|
||||
|
||||
create_valid_eml(os.path.join(valid_dir, "test_email.eml"))
|
||||
|
||||
# ============================
|
||||
# INVALID FILES
|
||||
# ============================
|
||||
def write_text_as(filepath, text):
|
||||
with open(filepath, 'w') as f:
|
||||
f.write(text)
|
||||
|
||||
def write_binary_as(filepath, data):
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
# Wrong magic bytes (plain text content with image/office extensions)
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.pdf"),
|
||||
"This is not a valid PDF file. It has no PDF magic bytes.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.tiff"),
|
||||
"This is not a valid TIFF file. No TIFF header present.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.jpeg"),
|
||||
"This is not a valid JPEG file. Missing SOI marker.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.png"),
|
||||
"This is not a valid PNG file. Missing PNG signature.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.bmp"),
|
||||
"This is not a valid BMP file. Missing BM header.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.docx"),
|
||||
"This is not a valid DOCX file. Not a ZIP archive.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.xlsx"),
|
||||
"This is not a valid XLSX file. Not a ZIP archive.")
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.pptx"),
|
||||
"This is not a valid PPTX file. Not a ZIP archive.")
|
||||
|
||||
# TXT: binary content with null bytes
|
||||
write_binary_as(os.path.join(invalid_dir, "bad_content.txt"),
|
||||
bytes(range(256)) * 4)
|
||||
|
||||
# EML: no RFC822 headers
|
||||
write_text_as(os.path.join(invalid_dir, "bad_content.eml"),
|
||||
"Hello this is just some random text without any email "
|
||||
"headers at all. No From, To, Subject, or Date headers. "
|
||||
"No header-body separator either.")
|
||||
|
||||
print("All test files generated successfully.")
|
||||
PYEOF
|
||||
|
||||
# Count generated files
|
||||
valid_count=$(find "$TEMP_DIR/valid" -type f | wc -l | tr -d ' ')
|
||||
invalid_count=$(find "$TEMP_DIR/invalid" -type f | wc -l | tr -d ' ')
|
||||
log_info "Generated $valid_count valid files and $invalid_count invalid files"
|
||||
|
||||
if [ "$valid_count" -ne "$TOTAL_VALID" ]; then
|
||||
log_error "Expected $TOTAL_VALID valid files, got $valid_count"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$invalid_count" -ne "$TOTAL_INVALID" ]; then
|
||||
log_error "Expected $TOTAL_INVALID invalid files, got $invalid_count"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Create ZIP from generated files
|
||||
create_test_zip() {
|
||||
log_step "Creating ZIP file with all test files..."
|
||||
rm -f "$ZIP_FILE"
|
||||
current_dir=$(pwd)
|
||||
cd "$TEMP_DIR"
|
||||
zip -q -r "$current_dir/$ZIP_FILE" .
|
||||
cd "$current_dir"
|
||||
file_size=$(stat -f%z "$ZIP_FILE" 2>/dev/null || stat -c%s "$ZIP_FILE" 2>/dev/null)
|
||||
log_info "Created $ZIP_FILE ($file_size bytes)"
|
||||
}
|
||||
|
||||
# Upload ZIP batch
|
||||
upload_batch() {
|
||||
log_step "Uploading ZIP batch..."
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@$ZIP_FILE" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
response_body=$(echo "$response" | head -n -1)
|
||||
|
||||
if [ "$http_code" = "202" ]; then
|
||||
BATCH_ID=$(echo "$response_body" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
print(data['batch_id'])
|
||||
")
|
||||
log_info "Batch uploaded successfully"
|
||||
log_info "Batch ID: $BATCH_ID"
|
||||
return 0
|
||||
else
|
||||
log_error "Failed to upload batch (HTTP $http_code)"
|
||||
echo "$response_body"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Phase 1: Wait for batch status != "processing"
|
||||
poll_batch_extraction() {
|
||||
log_step "Waiting for batch extraction to complete (up to 60s)..."
|
||||
local max_wait=60 elapsed=0 interval=3
|
||||
while [ $elapsed -lt $max_wait ]; do
|
||||
response=$(curl -s "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID")
|
||||
status=$(echo "$response" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
print(data.get('status', 'unknown'))
|
||||
")
|
||||
if [ "$status" != "processing" ]; then
|
||||
log_info "Batch extraction finished with status: $status"
|
||||
return 0
|
||||
fi
|
||||
sleep $interval
|
||||
elapsed=$((elapsed + interval))
|
||||
log_info "Batch still extracting... ($elapsed/${max_wait}s)"
|
||||
done
|
||||
log_error "Timed out waiting for batch extraction"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Phase 2: Wait for all document outcomes to reach terminal state
|
||||
poll_pipeline_complete() {
|
||||
log_step "Waiting for all documents to complete pipeline (up to 180s)..."
|
||||
local max_wait=180 elapsed=0 interval=5
|
||||
while [ $elapsed -lt $max_wait ]; do
|
||||
response=$(curl -s "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID")
|
||||
result=$(echo "$response" | python3 -c "
|
||||
import sys, json
|
||||
TERMINAL = {'duplicate','init_duplicate','failed_open','failed_read',
|
||||
'failed_s3_upload','failed_upload','invalid_type',
|
||||
'clean_passed','clean_failed','sync_skipped'}
|
||||
data = json.load(sys.stdin)
|
||||
outcomes = data.get('document_outcomes', [])
|
||||
total = len(outcomes)
|
||||
terminal = sum(1 for o in outcomes if o.get('outcome') in TERMINAL)
|
||||
print(f'{terminal}/{total}')
|
||||
if total > 0 and terminal == total:
|
||||
sys.exit(0)
|
||||
else:
|
||||
sys.exit(1)
|
||||
" 2>/dev/null)
|
||||
exit_code=$?
|
||||
log_info "Pipeline progress: $result documents at terminal state"
|
||||
if [ $exit_code -eq 0 ]; then
|
||||
log_info "All documents have completed pipeline processing"
|
||||
return 0
|
||||
fi
|
||||
sleep $interval
|
||||
elapsed=$((elapsed + interval))
|
||||
done
|
||||
log_error "Timed out waiting for pipeline completion"
|
||||
curl -s "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID" | python3 -m json.tool
|
||||
return 1
|
||||
}
|
||||
|
||||
# Validate final batch results
|
||||
validate_results() {
|
||||
log_step "Validating batch results..."
|
||||
response=$(curl -s "$BASE_URL/client/$CLIENT_ID/document/batch/$BATCH_ID")
|
||||
|
||||
BATCH_RESPONSE="$response" python3 << 'PYEOF'
|
||||
import sys, json, os
|
||||
|
||||
data = json.loads(os.environ["BATCH_RESPONSE"])
|
||||
errors = []
|
||||
|
||||
def check(condition, msg):
|
||||
if not condition:
|
||||
errors.append(msg)
|
||||
|
||||
# --- Count assertions ---
|
||||
check(data["total_documents"] == 20,
|
||||
f"total_documents: expected 20, got {data['total_documents']}")
|
||||
check(data["processed_documents"] == 10,
|
||||
f"processed_documents: expected 10, got {data['processed_documents']}")
|
||||
check(data["failed_documents"] == 10,
|
||||
f"failed_documents: expected 10, got {data['failed_documents']}")
|
||||
check(data["invalid_type_documents"] == 0,
|
||||
f"invalid_type_documents: expected 0, got {data['invalid_type_documents']}")
|
||||
check(data["status"] == "completed",
|
||||
f"status: expected 'completed', got '{data['status']}'")
|
||||
check(data.get("progress_percent") == 100,
|
||||
f"progress_percent: expected 100, got {data.get('progress_percent')}")
|
||||
check(data.get("completed_at") is not None,
|
||||
"completed_at: expected not null")
|
||||
|
||||
# --- Per-file outcome assertions ---
|
||||
valid_files = {
|
||||
"valid/test_document.pdf",
|
||||
"valid/test_image.tiff",
|
||||
"valid/test_photo.jpeg",
|
||||
"valid/test_screenshot.png",
|
||||
"valid/test_bitmap.bmp",
|
||||
"valid/test_report.docx",
|
||||
"valid/test_spreadsheet.xlsx",
|
||||
"valid/test_presentation.pptx",
|
||||
"valid/test_notes.txt",
|
||||
"valid/test_email.eml",
|
||||
}
|
||||
|
||||
invalid_files = {
|
||||
"invalid/bad_content.pdf",
|
||||
"invalid/bad_content.tiff",
|
||||
"invalid/bad_content.jpeg",
|
||||
"invalid/bad_content.png",
|
||||
"invalid/bad_content.bmp",
|
||||
"invalid/bad_content.docx",
|
||||
"invalid/bad_content.xlsx",
|
||||
"invalid/bad_content.pptx",
|
||||
"invalid/bad_content.txt",
|
||||
"invalid/bad_content.eml",
|
||||
}
|
||||
|
||||
outcomes = data.get("document_outcomes", [])
|
||||
outcome_map = {o["filename"]: o for o in outcomes}
|
||||
|
||||
# Assert: all valid files must reach clean_passed with a document_id
|
||||
for f in sorted(valid_files):
|
||||
if f not in outcome_map:
|
||||
errors.append(f"Missing outcome for valid file: {f}")
|
||||
else:
|
||||
o = outcome_map[f]
|
||||
if o["outcome"] != "clean_passed":
|
||||
detail = o.get("error_detail") or o.get("clean_fail_reason") or "none"
|
||||
errors.append(f"{f}: expected clean_passed, got {o['outcome']} (detail: {detail})")
|
||||
if not o.get("document_id"):
|
||||
errors.append(f"{f}: expected document_id to be set")
|
||||
|
||||
# Assert: all invalid files must reach clean_failed with a clean_fail_reason
|
||||
for f in sorted(invalid_files):
|
||||
if f not in outcome_map:
|
||||
errors.append(f"Missing outcome for invalid file: {f}")
|
||||
else:
|
||||
o = outcome_map[f]
|
||||
if o["outcome"] != "clean_failed":
|
||||
detail = o.get("error_detail") or o.get("clean_fail_reason") or "none"
|
||||
errors.append(f"{f}: expected clean_failed, got {o['outcome']} (detail: {detail})")
|
||||
if not o.get("clean_fail_reason"):
|
||||
errors.append(f"{f}: expected clean_fail_reason to be set")
|
||||
|
||||
# Verify all expected files are present in outcomes
|
||||
outcome_filenames = set(outcome_map.keys())
|
||||
missing_valid = valid_files - outcome_filenames
|
||||
missing_invalid = invalid_files - outcome_filenames
|
||||
if missing_valid:
|
||||
errors.append(f"Missing valid files in outcomes: {missing_valid}")
|
||||
if missing_invalid:
|
||||
errors.append(f"Missing invalid files in outcomes: {missing_invalid}")
|
||||
check(len(outcomes) == 20, f"Expected 20 outcomes, got {len(outcomes)}")
|
||||
|
||||
# --- Report ---
|
||||
if errors:
|
||||
print(f"VALIDATION FAILED ({len(errors)} errors):")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
print("\nFull batch status:")
|
||||
print(json.dumps(data, indent=2, default=str))
|
||||
sys.exit(1)
|
||||
else:
|
||||
passed = sum(1 for o in outcomes if o["outcome"] == "clean_passed")
|
||||
failed = sum(1 for o in outcomes if o["outcome"] == "clean_failed")
|
||||
print("All assertions passed!")
|
||||
print(f" Valid files (clean_passed): {passed}")
|
||||
print(f" Invalid files (clean_failed): {failed}")
|
||||
print(f" Total documents: {data['total_documents']}")
|
||||
print(f" Progress: {data.get('progress_percent')}%")
|
||||
sys.exit(0)
|
||||
PYEOF
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "Validation FAILED"
|
||||
return 1
|
||||
fi
|
||||
log_info "Validation PASSED"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Test invalid ZIP upload
|
||||
test_invalid_zip_upload() {
|
||||
log_step "Testing invalid file upload (should fail with 400)..."
|
||||
|
||||
echo "This is not a ZIP file" > invalid.txt
|
||||
|
||||
response=$(curl -s -w "\n%{http_code}" -X POST "$BASE_URL/client/$CLIENT_ID/document/batch" \
|
||||
-F "archive=@invalid.txt" 2>/dev/null)
|
||||
|
||||
http_code=$(echo "$response" | tail -n1)
|
||||
rm -f invalid.txt
|
||||
|
||||
if [ "$http_code" = "400" ]; then
|
||||
log_info "Invalid file correctly rejected (HTTP 400)"
|
||||
return 0
|
||||
else
|
||||
log_warn "Expected HTTP 400 for invalid file, got HTTP $http_code"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Cleanup
|
||||
cleanup() {
|
||||
log_info "Cleaning up..."
|
||||
rm -f "$ZIP_FILE" invalid.txt
|
||||
rm -rf "$TEMP_DIR"
|
||||
log_info "Cleanup complete"
|
||||
}
|
||||
|
||||
# Main test flow
|
||||
main() {
|
||||
log_step "Starting multi-type ZIP batch upload test..."
|
||||
log_info "Testing file types: PDF, TIFF, JPEG, PNG, BMP, DOCX, XLSX, PPTX, TXT, EML"
|
||||
log_info "Using client ID: $CLIENT_ID"
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
check_service
|
||||
generate_test_files
|
||||
create_test_zip
|
||||
create_client
|
||||
enable_client_sync
|
||||
|
||||
if upload_batch; then
|
||||
poll_batch_extraction
|
||||
poll_pipeline_complete
|
||||
validate_results
|
||||
|
||||
test_invalid_zip_upload
|
||||
|
||||
log_info "All tests completed successfully!"
|
||||
else
|
||||
log_error "Batch upload failed, skipping remaining tests"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user