jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in DAIP2-1595-optimize-signature-fields (pull request #928)

Browse Source 
removed 2 signature related fields

* removed 2 signature related fields

* Merged dev into DAIP2-1595-optimize-signature-fields

* Merged dev into DAIP2-1595-optimize-signature-fields


Approved-by: Katon Minhas
This commit is contained in:
Mayank Aamseek
2026-04-02 18:11:21 +00:00
committed by Katon Minhas
parent 0253d7dc4a
commit a9098ab2e8
6 changed files with 21 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/constants/investment_columns.py
-2
View File
@@ -49,9 +49,7 @@ FIELD_FORMAT_MAPPING = {
"AARETE_DERIVED_TERMINATION_DT": "str",
"AUTO_RENEWAL_IND": "str",
"AUTO_RENEWAL_TERM": "str",
"NUM_AVAILABLE_SIGNATORY_LINE_COUNT": "str",
"NUM_SIGNED_SIGNATORY_LINE_COUNT": "str",
"AARETE_DERIVED_SIGNATORY_COMPLETE_IND": "str",
"EXHIBIT_TITLE": "str",
"EXHIBIT_PAGE": "str",
"REIMB_PROV_TIN": "list[str]",
src/document_classification/main.py
+6 -20
View File
@@ -822,9 +822,6 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol
doczy_eff_dt = doczy_row.get(
"AARETE_DERIVED_EFFECTIVE_DT", None
) or doczy_row.get("EFFECTIVE_DT", None)
doczy_signature = doczy_row.get(
"AARETE_DERIVED_SIGNATORY_COMPLETE_IND", None
)
doczy_signed_count = doczy_row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None)
has_doczy_tin = _is_doczy_field_present(
@@ -832,10 +829,9 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol
) or _is_doczy_field_present(doczy_filename_tin)
has_doczy_lob = _is_doczy_field_present(doczy_lob)
has_doczy_eff_dt = _is_doczy_field_present(doczy_eff_dt)
has_doczy_signature = _is_doczy_field_present(doczy_signature) or (
_is_doczy_field_present(doczy_signed_count)
and str(doczy_signed_count).strip() not in ("0", "0.0")
)
has_doczy_signature = _is_doczy_field_present(doczy_signed_count) and str(
doczy_signed_count
).strip() not in ("0", "0.0")
else:
has_doczy_tin = False
has_doczy_lob = False
@@ -844,7 +840,6 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol
doczy_tin = None
doczy_lob = None
doczy_eff_dt = None
doczy_signature = None
# ── 2. What's in the raw document (regex search) ──
tin_in_filename = _search_tin_in_filename(filename)
@@ -914,7 +909,7 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol
)
# Signature value: prefer Doczy extracted
signature_doczy_display = str(doczy_signature) if has_doczy_signature else ""
signature_doczy_display = "Y" if has_doczy_signature else ""
# Pricing readiness: all 4 fields must be "Extracted"
pricing_ready = all(
@@ -1227,7 +1222,7 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None):
- TIN (Y/N) and where it was found
- LOB (Insurance Type value)
- Effective Date (Y/N), where found, and actual date value
- Signature Indicator (Y/N)
- Signature Count (0,1,2,...)
Output columns:
Contract Name | TIN | TIN Where | LOB | EFF Date | Where Eff |
@@ -1275,7 +1270,6 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None):
"LOB",
"AARETE_DERIVED_EFFECTIVE_DT",
"EFFECTIVE_DT",
"AARETE_DERIVED_SIGNATORY_COMPLETE_IND",
"NUM_SIGNED_SIGNATORY_LINE_COUNT",
]
available_cols = [c for c in contract_level_cols if c in doczy_df.columns]
@@ -1346,20 +1340,12 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None):
where_eff = "In contract" if has_doczy_eff_dt else ""
# ── Signature ──
doczy_signature = row.get("AARETE_DERIVED_SIGNATORY_COMPLETE_IND", None)
doczy_signed_count = row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None)
# Check signatory indicator — treat "N", "No", "False" as no signature
sig_negative_values = ("N", "NO", "FALSE", "0", "0.0")
has_signature_ind = (
_is_doczy_field_present(doczy_signature)
and str(doczy_signature).strip().upper() not in sig_negative_values
)
has_signed_count = _is_doczy_field_present(doczy_signed_count) and str(
doczy_signed_count
).strip() not in ("0", "0.0")
has_signature = has_signature_ind or has_signed_count
signature_ind = "Y" if has_signature else "N"
signature_ind = "Y" if has_signed_count else "N"
report_rows.append(
{
src/pipelines/shared/postprocessing/postprocess.py
+5 -3
View File
@@ -80,6 +80,11 @@ def standard_postprocess(df, constants: Constants):
postprocessing_funcs.format_rate_fields_with_commas
)
# format numeric fields
df = postprocessing_funcs.format_numeric_fields(
df, ["NUM_SIGNED_SIGNATORY_LINE_COUNT"]
)
for col in df.columns:
if "_IND" in col:
df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field)
@@ -109,9 +114,6 @@ def standard_postprocess(df, constants: Constants):
df, constants.VALID_UNIT_OF_MEASURE
)
# Add AARETE_DERIVED_SIGNATORY_COMPLETE_IND based on 2 signatory fields
df = postprocessing_funcs.add_aarete_derived_signatory_complete_ind(df)
# update reimb_pct_rate
df = postprocessing_funcs.fill_empty_reimb_pct_rate(df)
src/pipelines/shared/postprocessing/postprocessing_funcs.py
+10 -53
View File
@@ -224,6 +224,16 @@ def normalize_currency(value: str) -> str:
return text
def format_numeric_fields(df: pd.DataFrame, numeric_fields: list[str]) -> pd.DataFrame:
"""
Format specified numeric fields
"""
for field in numeric_fields:
if field in df.columns:
df[field] = pd.to_numeric(df[field], errors="coerce")
return df
def flatten_singleton_string_list(list_str: str) -> str:
"""Take in a list within a string, and if it has a single element, return that element.
If it has multiple elements, return the list as a string.
@@ -1013,59 +1023,6 @@ def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
return df
def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame:
"""
Add a derived indicator column to the DataFrame indicating whether all signatory lines are signed.
Creates column:
AARETE_DERIVED_SIGNATORY_COMPLETE_IND = 'Y' if
((NUM_AVAILABLE_SIGNATORY_LINE_COUNT <= NUM_SIGNED_SIGNATORY_LINE_COUNT) and
(NUM_SIGNED_SIGNATORY_LINE_COUNT >= 1))
else 'N'.
Parameters
----------
df : pd.DataFrame
Input DataFrame containing the required columns.
Returns
-------
pd.DataFrame
DataFrame with the new indicator column added.
"""
try:
required_cols = [
"NUM_AVAILABLE_SIGNATORY_LINE_COUNT",
"NUM_SIGNED_SIGNATORY_LINE_COUNT",
]
# Validate required columns
for col in required_cols:
if col not in df.columns:
return df
# Ensure consistent numeric types
for col in required_cols:
df[col] = pd.to_numeric(df[col], errors="coerce")
# Create indicator column
df["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"] = (
(
df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"].le(
df["NUM_SIGNED_SIGNATORY_LINE_COUNT"]
)
)
& (df["NUM_SIGNED_SIGNATORY_LINE_COUNT"].ge(1))
).map({True: "Y", False: "N"})
return df
except Exception:
# Return df unchanged on any exception
return df
def standardize_file_name(file_name: str) -> str:
"""
Standardizes the given file name by applying a series of regex replacements
src/prompts/investment_prompts.json
-7
View File
@@ -839,12 +839,5 @@
"relationship": "one_to_n",
"field_type": "facility_adjustment_breakout",
"prompt": "Are adjustments for Graduate Medical Education (GME) included in the reimbursement? Return 'Y' or 'N'."
},
{
"field_name": "NUM_AVAILABLE_SIGNATORY_LINE_COUNT",
"relationship": "one_to_one",
"field_type": "smart_chunked",
"prompt": "Count the total number of signature lines in the document. A signature line refers to any line or section intended for a party to sign, typically accompanied by but not limited to a date, designation, or name of the signee or entity. Include all distinct signature lines for individuals and entities. If none are present, return 0.",
"retrieval_question": "Identify all sections in the document that require a signature from any party. This includes any designated space or instruction for signing, such as fields labeled Signature, Date, Name, Designation, Title, or Entity details. Return every distinct signature-related section for all parties involved, including signature blocks, acknowledgment areas, and authorization sections."
}
]
src/tests/test_postprocess.py
-15
View File
@@ -11,7 +11,6 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import (
clean_na_values,
deduplicate_provider_columns,
fill_claim_type_from_title,
add_aarete_derived_signatory_complete_ind,
flatten_singleton_string_list,
format_as_json_list,
format_rate_fields_with_commas,
@@ -735,20 +734,6 @@ class TestPostprocessFunctions(unittest.TestCase):
# Empty list should be treated as empty and filled with mode or inferred
assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M"
def test_add_aarete_derived_signatory_complete_ind(self):
"""Test the derivation of the signatory complete indicator for fully signed, partially signed, and zero-signature scenarios."""
df = pd.DataFrame(
{
"NUM_AVAILABLE_SIGNATORY_LINE_COUNT": [2, 2, 0],
"NUM_SIGNED_SIGNATORY_LINE_COUNT": [2, 1, 0],
}
)
result = add_aarete_derived_signatory_complete_ind(df)
self.assertEqual(
result["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"].tolist(), ["Y", "N", "N"]
)
def test_clean_na_values_string_placeholders(self):
"""Test clean_na_values removes placeholder strings when they're the only value."""
# Test with string placeholders