jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/post_doczy_report_lob (pull request #973)

Browse Source 
Post_doczy_report_lob_fix

* Post_doczy_report_lob_fix


Approved-by: Katon Minhas
This commit is contained in:
Rahul Ailaboina
2026-04-17 20:44:34 +00:00
committed by Katon Minhas
parent 73f345da86
commit ac857b9953
1 changed files with 55 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/document_classification/main.py
+55 -28
View File
@@ -1199,12 +1199,8 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol
if doczy_row is not None:
doczy_tin = doczy_row.get("PROV_GROUP_TIN", None)
doczy_filename_tin = doczy_row.get("FILENAME_TIN", None)
doczy_lob = doczy_row.get("AARETE_DERIVED_LOB", None) or doczy_row.get(
"LOB", None
)
doczy_eff_dt = doczy_row.get(
"AARETE_DERIVED_EFFECTIVE_DT", None
) or doczy_row.get("EFFECTIVE_DT", None)
doczy_lob = doczy_row.get("AARETE_DERIVED_LOB", None)
doczy_eff_dt = doczy_row.get("AARETE_DERIVED_EFFECTIVE_DT", None)
doczy_signed_count = doczy_row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None)
has_doczy_tin = _is_doczy_field_present(
@@ -1376,19 +1372,38 @@ def generate_post_doczy_report(input_dict, doczy_results_df, run_timestamp):
os.makedirs(json_folder, exist_ok=True)
logging.info(f"Post-Doczy Report JSON output folder: {json_folder}")
# Build a case-insensitive lookup dict from Doczy results: filename -> row dict
# Doczy postprocessing strips .txt from FILE_NAME, so we try multiple
# matching strategies: exact, with .txt appended, and case-insensitive.
# Build a case-insensitive lookup dict from Doczy results: filename -> row dict.
# A contract may span multiple Doczy rows (one per exhibit/reimbursement term);
# for AARETE_DERIVED_LOB we aggregate distinct values across all rows so the
# report reflects every LOB the contract covers rather than a single row's value.
doczy_lookup = {}
if "FILE_NAME" in doczy_results_df.columns:
lob_by_file = {}
if "AARETE_DERIVED_LOB" in doczy_results_df.columns:
for _, row in doczy_results_df.iterrows():
fname = str(row.get("FILE_NAME", "")).strip()
if not fname:
continue
lob_val = row.get("AARETE_DERIVED_LOB", None)
if not _is_doczy_field_present(lob_val):
continue
cleaned = _clean_list_field(lob_val)
if not cleaned:
continue
for part in (p.strip() for p in cleaned.split(",")):
if part:
lob_by_file.setdefault(fname, set()).add(part)
for _, row in doczy_results_df.iterrows():
fname = str(row.get("FILE_NAME", "")).strip()
if fname:
row_dict = row.to_dict()
# Store with original case and lowercased for case-insensitive matching
if fname in lob_by_file:
row_dict["AARETE_DERIVED_LOB"] = ", ".join(
sorted(lob_by_file[fname])
)
doczy_lookup[fname] = row_dict
doczy_lookup[fname.lower()] = row_dict
# Also store with .txt suffix since input_dict keys include .txt
if not fname.lower().endswith(".txt"):
doczy_lookup[fname + ".txt"] = row_dict
doczy_lookup[fname.lower() + ".txt"] = row_dict
@@ -1640,24 +1655,42 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None):
)
# Group by FILE_NAME to produce one row per contract file.
# Uses first non-null value per field (a contract may have multiple rows
# with different values across exhibits/reimbursement terms).
# Note: .first() is order-dependent — result depends on row ordering in the
# Doczy output. If exhibits have conflicting values (e.g., different TINs),
# the first encountered row wins.
# A contract may have multiple Doczy rows (one per exhibit/reimbursement term).
# For AARETE_DERIVED_LOB we collect the set of distinct values across all rows
# so the report reflects every LOB covered by the contract. For other fields
# we take the first non-null value (order-dependent on Doczy row ordering).
# Base `LOB` / `EFFECTIVE_DT` columns are intentionally ignored — only the
# AArete-derived values are canonical.
contract_level_cols = [
"FILE_NAME",
"FILENAME_TIN",
"PROV_GROUP_TIN",
"AARETE_DERIVED_LOB",
"LOB",
"AARETE_DERIVED_EFFECTIVE_DT",
"EFFECTIVE_DT",
"NUM_SIGNED_SIGNATORY_LINE_COUNT",
]
available_cols = [c for c in contract_level_cols if c in doczy_df.columns]
def _agg_lob(series):
values = set()
for v in series.dropna():
cleaned = _clean_list_field(v)
if not cleaned:
continue
for part in (p.strip() for p in cleaned.split(",")):
if part:
values.add(part)
return ", ".join(sorted(values)) if values else None
agg_rules = {c: "first" for c in available_cols if c != "FILE_NAME"}
if "AARETE_DERIVED_LOB" in agg_rules:
agg_rules["AARETE_DERIVED_LOB"] = _agg_lob
contracts_df = (
doczy_df[available_cols].groupby("FILE_NAME", sort=False).first().reset_index()
doczy_df[available_cols]
.groupby("FILE_NAME", sort=False)
.agg(agg_rules)
.reset_index()
)
logging.info(
@@ -1689,20 +1722,14 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None):
tin_where = ", ".join(tin_where_parts)
# ── LOB ──
# Use AARETE_DERIVED_LOB first; fall back to LOB if derived is empty/NaN
# Pre-aggregated above: comma-joined distinct AARETE_DERIVED_LOB values
# across all rows for this contract.
doczy_lob_raw = row.get("AARETE_DERIVED_LOB", None)
if not _is_doczy_field_present(doczy_lob_raw):
doczy_lob_raw = row.get("LOB", None)
has_doczy_lob = _is_doczy_field_present(doczy_lob_raw)
cleaned_lob = _clean_list_field(doczy_lob_raw) if has_doczy_lob else None
lob_value = cleaned_lob if cleaned_lob else ""
lob_value = str(doczy_lob_raw).strip() if has_doczy_lob else ""
# ── Effective Date ──
# Use AARETE_DERIVED_EFFECTIVE_DT first; fall back to EFFECTIVE_DT
doczy_eff_dt = row.get("AARETE_DERIVED_EFFECTIVE_DT", None)
if not _is_doczy_field_present(doczy_eff_dt):
doczy_eff_dt = row.get("EFFECTIVE_DT", None)
# Validate the date is actually parseable (not NaT/nan)
has_doczy_eff_dt = False