From ac857b9953e0e1218a26774bc1eac0a94f4a279b Mon Sep 17 00:00:00 2001 From: Rahul Ailaboina Date: Fri, 17 Apr 2026 20:44:34 +0000 Subject: [PATCH] Merged in bugfix/post_doczy_report_lob (pull request #973) Post_doczy_report_lob_fix * Post_doczy_report_lob_fix Approved-by: Katon Minhas --- src/document_classification/main.py | 83 +++++++++++++++++++---------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/src/document_classification/main.py b/src/document_classification/main.py index 08c2852..c6d21df 100644 --- a/src/document_classification/main.py +++ b/src/document_classification/main.py @@ -1199,12 +1199,8 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol if doczy_row is not None: doczy_tin = doczy_row.get("PROV_GROUP_TIN", None) doczy_filename_tin = doczy_row.get("FILENAME_TIN", None) - doczy_lob = doczy_row.get("AARETE_DERIVED_LOB", None) or doczy_row.get( - "LOB", None - ) - doczy_eff_dt = doczy_row.get( - "AARETE_DERIVED_EFFECTIVE_DT", None - ) or doczy_row.get("EFFECTIVE_DT", None) + doczy_lob = doczy_row.get("AARETE_DERIVED_LOB", None) + doczy_eff_dt = doczy_row.get("AARETE_DERIVED_EFFECTIVE_DT", None) doczy_signed_count = doczy_row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None) has_doczy_tin = _is_doczy_field_present( @@ -1376,19 +1372,38 @@ def generate_post_doczy_report(input_dict, doczy_results_df, run_timestamp): os.makedirs(json_folder, exist_ok=True) logging.info(f"Post-Doczy Report JSON output folder: {json_folder}") - # Build a case-insensitive lookup dict from Doczy results: filename -> row dict - # Doczy postprocessing strips .txt from FILE_NAME, so we try multiple - # matching strategies: exact, with .txt appended, and case-insensitive. + # Build a case-insensitive lookup dict from Doczy results: filename -> row dict. + # A contract may span multiple Doczy rows (one per exhibit/reimbursement term); + # for AARETE_DERIVED_LOB we aggregate distinct values across all rows so the + # report reflects every LOB the contract covers rather than a single row's value. doczy_lookup = {} if "FILE_NAME" in doczy_results_df.columns: + lob_by_file = {} + if "AARETE_DERIVED_LOB" in doczy_results_df.columns: + for _, row in doczy_results_df.iterrows(): + fname = str(row.get("FILE_NAME", "")).strip() + if not fname: + continue + lob_val = row.get("AARETE_DERIVED_LOB", None) + if not _is_doczy_field_present(lob_val): + continue + cleaned = _clean_list_field(lob_val) + if not cleaned: + continue + for part in (p.strip() for p in cleaned.split(",")): + if part: + lob_by_file.setdefault(fname, set()).add(part) + for _, row in doczy_results_df.iterrows(): fname = str(row.get("FILE_NAME", "")).strip() if fname: row_dict = row.to_dict() - # Store with original case and lowercased for case-insensitive matching + if fname in lob_by_file: + row_dict["AARETE_DERIVED_LOB"] = ", ".join( + sorted(lob_by_file[fname]) + ) doczy_lookup[fname] = row_dict doczy_lookup[fname.lower()] = row_dict - # Also store with .txt suffix since input_dict keys include .txt if not fname.lower().endswith(".txt"): doczy_lookup[fname + ".txt"] = row_dict doczy_lookup[fname.lower() + ".txt"] = row_dict @@ -1640,24 +1655,42 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None): ) # Group by FILE_NAME to produce one row per contract file. - # Uses first non-null value per field (a contract may have multiple rows - # with different values across exhibits/reimbursement terms). - # Note: .first() is order-dependent — result depends on row ordering in the - # Doczy output. If exhibits have conflicting values (e.g., different TINs), - # the first encountered row wins. + # A contract may have multiple Doczy rows (one per exhibit/reimbursement term). + # For AARETE_DERIVED_LOB we collect the set of distinct values across all rows + # so the report reflects every LOB covered by the contract. For other fields + # we take the first non-null value (order-dependent on Doczy row ordering). + # Base `LOB` / `EFFECTIVE_DT` columns are intentionally ignored — only the + # AArete-derived values are canonical. contract_level_cols = [ "FILE_NAME", "FILENAME_TIN", "PROV_GROUP_TIN", "AARETE_DERIVED_LOB", - "LOB", "AARETE_DERIVED_EFFECTIVE_DT", - "EFFECTIVE_DT", "NUM_SIGNED_SIGNATORY_LINE_COUNT", ] available_cols = [c for c in contract_level_cols if c in doczy_df.columns] + + def _agg_lob(series): + values = set() + for v in series.dropna(): + cleaned = _clean_list_field(v) + if not cleaned: + continue + for part in (p.strip() for p in cleaned.split(",")): + if part: + values.add(part) + return ", ".join(sorted(values)) if values else None + + agg_rules = {c: "first" for c in available_cols if c != "FILE_NAME"} + if "AARETE_DERIVED_LOB" in agg_rules: + agg_rules["AARETE_DERIVED_LOB"] = _agg_lob + contracts_df = ( - doczy_df[available_cols].groupby("FILE_NAME", sort=False).first().reset_index() + doczy_df[available_cols] + .groupby("FILE_NAME", sort=False) + .agg(agg_rules) + .reset_index() ) logging.info( @@ -1689,20 +1722,14 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None): tin_where = ", ".join(tin_where_parts) # ── LOB ── - # Use AARETE_DERIVED_LOB first; fall back to LOB if derived is empty/NaN + # Pre-aggregated above: comma-joined distinct AARETE_DERIVED_LOB values + # across all rows for this contract. doczy_lob_raw = row.get("AARETE_DERIVED_LOB", None) - if not _is_doczy_field_present(doczy_lob_raw): - doczy_lob_raw = row.get("LOB", None) has_doczy_lob = _is_doczy_field_present(doczy_lob_raw) - cleaned_lob = _clean_list_field(doczy_lob_raw) if has_doczy_lob else None - - lob_value = cleaned_lob if cleaned_lob else "" + lob_value = str(doczy_lob_raw).strip() if has_doczy_lob else "" # ── Effective Date ── - # Use AARETE_DERIVED_EFFECTIVE_DT first; fall back to EFFECTIVE_DT doczy_eff_dt = row.get("AARETE_DERIVED_EFFECTIVE_DT", None) - if not _is_doczy_field_present(doczy_eff_dt): - doczy_eff_dt = row.get("EFFECTIVE_DT", None) # Validate the date is actually parseable (not NaT/nan) has_doczy_eff_dt = False