diff --git a/src/constants/investment_columns.py b/src/constants/investment_columns.py index 5241854..6c088ae 100644 --- a/src/constants/investment_columns.py +++ b/src/constants/investment_columns.py @@ -49,9 +49,7 @@ FIELD_FORMAT_MAPPING = { "AARETE_DERIVED_TERMINATION_DT": "str", "AUTO_RENEWAL_IND": "str", "AUTO_RENEWAL_TERM": "str", - "NUM_AVAILABLE_SIGNATORY_LINE_COUNT": "str", "NUM_SIGNED_SIGNATORY_LINE_COUNT": "str", - "AARETE_DERIVED_SIGNATORY_COMPLETE_IND": "str", "EXHIBIT_TITLE": "str", "EXHIBIT_PAGE": "str", "REIMB_PROV_TIN": "list[str]", diff --git a/src/document_classification/main.py b/src/document_classification/main.py index 53fb5ec..1924263 100644 --- a/src/document_classification/main.py +++ b/src/document_classification/main.py @@ -822,9 +822,6 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol doczy_eff_dt = doczy_row.get( "AARETE_DERIVED_EFFECTIVE_DT", None ) or doczy_row.get("EFFECTIVE_DT", None) - doczy_signature = doczy_row.get( - "AARETE_DERIVED_SIGNATORY_COMPLETE_IND", None - ) doczy_signed_count = doczy_row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None) has_doczy_tin = _is_doczy_field_present( @@ -832,10 +829,9 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol ) or _is_doczy_field_present(doczy_filename_tin) has_doczy_lob = _is_doczy_field_present(doczy_lob) has_doczy_eff_dt = _is_doczy_field_present(doczy_eff_dt) - has_doczy_signature = _is_doczy_field_present(doczy_signature) or ( - _is_doczy_field_present(doczy_signed_count) - and str(doczy_signed_count).strip() not in ("0", "0.0") - ) + has_doczy_signature = _is_doczy_field_present(doczy_signed_count) and str( + doczy_signed_count + ).strip() not in ("0", "0.0") else: has_doczy_tin = False has_doczy_lob = False @@ -844,7 +840,6 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol doczy_tin = None doczy_lob = None doczy_eff_dt = None - doczy_signature = None # ── 2. What's in the raw document (regex search) ── tin_in_filename = _search_tin_in_filename(filename) @@ -914,7 +909,7 @@ def _process_file_for_post_doczy_report(filename, file_text, doczy_row, json_fol ) # Signature value: prefer Doczy extracted - signature_doczy_display = str(doczy_signature) if has_doczy_signature else "" + signature_doczy_display = "Y" if has_doczy_signature else "" # Pricing readiness: all 4 fields must be "Extracted" pricing_ready = all( @@ -1227,7 +1222,7 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None): - TIN (Y/N) and where it was found - LOB (Insurance Type value) - Effective Date (Y/N), where found, and actual date value - - Signature Indicator (Y/N) + - Signature Count (0,1,2,...) Output columns: Contract Name | TIN | TIN Where | LOB | EFF Date | Where Eff | @@ -1275,7 +1270,6 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None): "LOB", "AARETE_DERIVED_EFFECTIVE_DT", "EFFECTIVE_DT", - "AARETE_DERIVED_SIGNATORY_COMPLETE_IND", "NUM_SIGNED_SIGNATORY_LINE_COUNT", ] available_cols = [c for c in contract_level_cols if c in doczy_df.columns] @@ -1346,20 +1340,12 @@ def generate_post_doczy_report_from_excel(excel_path, run_timestamp=None): where_eff = "In contract" if has_doczy_eff_dt else "" # ── Signature ── - doczy_signature = row.get("AARETE_DERIVED_SIGNATORY_COMPLETE_IND", None) doczy_signed_count = row.get("NUM_SIGNED_SIGNATORY_LINE_COUNT", None) - # Check signatory indicator — treat "N", "No", "False" as no signature - sig_negative_values = ("N", "NO", "FALSE", "0", "0.0") - has_signature_ind = ( - _is_doczy_field_present(doczy_signature) - and str(doczy_signature).strip().upper() not in sig_negative_values - ) has_signed_count = _is_doczy_field_present(doczy_signed_count) and str( doczy_signed_count ).strip() not in ("0", "0.0") - has_signature = has_signature_ind or has_signed_count - signature_ind = "Y" if has_signature else "N" + signature_ind = "Y" if has_signed_count else "N" report_rows.append( { diff --git a/src/pipelines/shared/postprocessing/postprocess.py b/src/pipelines/shared/postprocessing/postprocess.py index 20ba887..ddaef17 100644 --- a/src/pipelines/shared/postprocessing/postprocess.py +++ b/src/pipelines/shared/postprocessing/postprocess.py @@ -80,6 +80,11 @@ def standard_postprocess(df, constants: Constants): postprocessing_funcs.format_rate_fields_with_commas ) + # format numeric fields + df = postprocessing_funcs.format_numeric_fields( + df, ["NUM_SIGNED_SIGNATORY_LINE_COUNT"] + ) + for col in df.columns: if "_IND" in col: df[col] = df[col].apply(postprocessing_funcs.normalize_indicator_field) @@ -109,9 +114,6 @@ def standard_postprocess(df, constants: Constants): df, constants.VALID_UNIT_OF_MEASURE ) - # Add AARETE_DERIVED_SIGNATORY_COMPLETE_IND based on 2 signatory fields - df = postprocessing_funcs.add_aarete_derived_signatory_complete_ind(df) - # update reimb_pct_rate df = postprocessing_funcs.fill_empty_reimb_pct_rate(df) diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index d7f8a95..b258626 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -224,6 +224,16 @@ def normalize_currency(value: str) -> str: return text +def format_numeric_fields(df: pd.DataFrame, numeric_fields: list[str]) -> pd.DataFrame: + """ + Format specified numeric fields + """ + for field in numeric_fields: + if field in df.columns: + df[field] = pd.to_numeric(df[field], errors="coerce") + return df + + def flatten_singleton_string_list(list_str: str) -> str: """Take in a list within a string, and if it has a single element, return that element. If it has multiple elements, return the list as a string. @@ -1013,59 +1023,6 @@ def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame: return df -def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame: - """ - Add a derived indicator column to the DataFrame indicating whether all signatory lines are signed. - - Creates column: - AARETE_DERIVED_SIGNATORY_COMPLETE_IND = 'Y' if - ((NUM_AVAILABLE_SIGNATORY_LINE_COUNT <= NUM_SIGNED_SIGNATORY_LINE_COUNT) and - (NUM_SIGNED_SIGNATORY_LINE_COUNT >= 1)) - else 'N'. - - Parameters - ---------- - df : pd.DataFrame - Input DataFrame containing the required columns. - - Returns - ------- - pd.DataFrame - DataFrame with the new indicator column added. - """ - - try: - required_cols = [ - "NUM_AVAILABLE_SIGNATORY_LINE_COUNT", - "NUM_SIGNED_SIGNATORY_LINE_COUNT", - ] - - # Validate required columns - for col in required_cols: - if col not in df.columns: - return df - - # Ensure consistent numeric types - for col in required_cols: - df[col] = pd.to_numeric(df[col], errors="coerce") - - # Create indicator column - df["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"] = ( - ( - df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"].le( - df["NUM_SIGNED_SIGNATORY_LINE_COUNT"] - ) - ) - & (df["NUM_SIGNED_SIGNATORY_LINE_COUNT"].ge(1)) - ).map({True: "Y", False: "N"}) - - return df - - except Exception: - # Return df unchanged on any exception - return df - - def standardize_file_name(file_name: str) -> str: """ Standardizes the given file name by applying a series of regex replacements diff --git a/src/prompts/investment_prompts.json b/src/prompts/investment_prompts.json index f505c19..b2a13db 100644 --- a/src/prompts/investment_prompts.json +++ b/src/prompts/investment_prompts.json @@ -839,12 +839,5 @@ "relationship": "one_to_n", "field_type": "facility_adjustment_breakout", "prompt": "Are adjustments for Graduate Medical Education (GME) included in the reimbursement? Return 'Y' or 'N'." - }, - { - "field_name": "NUM_AVAILABLE_SIGNATORY_LINE_COUNT", - "relationship": "one_to_one", - "field_type": "smart_chunked", - "prompt": "Count the total number of signature lines in the document. A signature line refers to any line or section intended for a party to sign, typically accompanied by but not limited to a date, designation, or name of the signee or entity. Include all distinct signature lines for individuals and entities. If none are present, return 0.", - "retrieval_question": "Identify all sections in the document that require a signature from any party. This includes any designated space or instruction for signing, such as fields labeled Signature, Date, Name, Designation, Title, or Entity details. Return every distinct signature-related section for all parties involved, including signature blocks, acknowledgment areas, and authorization sections." } ] \ No newline at end of file diff --git a/src/tests/test_postprocess.py b/src/tests/test_postprocess.py index ebd227e..8aaab32 100644 --- a/src/tests/test_postprocess.py +++ b/src/tests/test_postprocess.py @@ -11,7 +11,6 @@ from src.pipelines.shared.postprocessing.postprocessing_funcs import ( clean_na_values, deduplicate_provider_columns, fill_claim_type_from_title, - add_aarete_derived_signatory_complete_ind, flatten_singleton_string_list, format_as_json_list, format_rate_fields_with_commas, @@ -735,20 +734,6 @@ class TestPostprocessFunctions(unittest.TestCase): # Empty list should be treated as empty and filled with mode or inferred assert result_df.loc[0, "AARETE_DERIVED_CLAIM_TYPE_CD"] == "M" - def test_add_aarete_derived_signatory_complete_ind(self): - """Test the derivation of the signatory complete indicator for fully signed, partially signed, and zero-signature scenarios.""" - df = pd.DataFrame( - { - "NUM_AVAILABLE_SIGNATORY_LINE_COUNT": [2, 2, 0], - "NUM_SIGNED_SIGNATORY_LINE_COUNT": [2, 1, 0], - } - ) - result = add_aarete_derived_signatory_complete_ind(df) - - self.assertEqual( - result["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"].tolist(), ["Y", "N", "N"] - ) - def test_clean_na_values_string_placeholders(self): """Test clean_na_values removes placeholder strings when they're the only value.""" # Test with string placeholders