From 0c3636f13b0ad8faf70270f81087a5dde4b682f1 Mon Sep 17 00:00:00 2001 From: Rahul Ailaboina Date: Thu, 30 Apr 2026 01:15:33 +0000 Subject: [PATCH] Merged in hotfix/fileextension_issue (pull request #989) Hotfix/fileextension issue * fixed strip_ext issue * black format Approved-by: Katon Minhas --- src/document_classification/main.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/document_classification/main.py b/src/document_classification/main.py index c6d21df..1cee58b 100644 --- a/src/document_classification/main.py +++ b/src/document_classification/main.py @@ -588,10 +588,11 @@ def _process_file_for_pre_doczy_report(filename, file_text, json_folder): def _normalize_filename(filename): - """Strip .txt extension to get the base name for matching against PDFs.""" + """Strip .txt then any source extension (incl. double-extensions like + `.PDF.txt` produced by Textract) so TXT names align with their PDF base.""" if filename.lower().endswith(".txt"): - return filename[:-4] - return filename + filename = filename[:-4] + return _strip_ext(filename) _STRIPPABLE_EXTS = ( @@ -810,21 +811,32 @@ def generate_pre_doczy_report(input_dict, run_timestamp): # `inv_fname not in results` guard prevents double-counting. if pdf_filenames: no_txt_count = 0 + matched_txt_count = 0 + matched_pdf_bases = set() for pdf_name in sorted(pdf_filenames): base_name = _strip_ext(pdf_name) if pdf_name not in txt_base_names and base_name not in txt_base_names: results[pdf_name] = {**na_placeholder, "STATUS": "TEXTRACT_FAILED"} no_txt_count += 1 + else: + matched_txt_count += 1 + matched_pdf_bases.add(base_name) # Add non-PDF files from the bucket (files with other extensions) for non_pdf_name in sorted(non_pdf_filenames): if non_pdf_name not in results: results[non_pdf_name] = {**na_placeholder, "STATUS": "INVALID"} + orphan_txt_count = sum( + 1 + for k in txt_base_names + if k not in matched_pdf_bases and k not in pdf_filenames + ) logging.info( f"PDF reconciliation: {len(pdf_filenames)} total PDFs, " - f"{len(txt_base_names)} with TXT, " + f"{matched_txt_count} matched a TXT, " f"{no_txt_count} without TXT (TEXTRACT_FAILED), " + f"{len(txt_base_names)} TXTs loaded ({orphan_txt_count} orphan — no PDF), " f"{len(non_pdf_filenames)} non-PDF files" )