From 7038a92d10a43824c9bd9eea223f010288fedc73 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Thu, 21 Aug 2025 15:19:53 +0000 Subject: [PATCH] Merged in bugfix/medicare-advantage-duals (pull request #674) Bugfix/medicare advantage duals * Add logging for debugging in crosswalk utilities and mapping functions * Add debug logging for crosswalk processing and mapping in get_crosswalk_fields * Add logging for dual LOB check prompts and responses in update_lob_for_duals * Enhance dual LOB check logic to ensure PROGRAM and PRODUCT fields are empty before proceeding * Update dual LOB check to use AARETE_DERIVED fields for validation * Refactor logging in fill_na_mapping and get_crosswalk_fields for clarity and conciseness * Remove debug logging from get_lob_relationship for cleaner output * Remove debug logging from apply_crosswalk for cleaner execution Approved-by: Katon Minhas --- .../src/investment/aarete_derived.py | 14 +++++++---- .../src/investment/one_to_n_funcs.py | 24 ++++++++++++------- .../src/investment/postprocessing_funcs.py | 11 ++++++++- fieldExtraction/src/utils/crosswalk_utils.py | 10 +++++--- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index d7c0fc5..d500b59 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -1,3 +1,4 @@ +import logging import os from collections import defaultdict @@ -34,22 +35,25 @@ def fill_na_mapping(answer_dicts): for answer_dict in answer_dicts: # Fill AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): - answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field( + filled_value = fill_na_from_field( answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "constants/mappings/crosswalk_program_lob.json", ) + + answer_dict["AARETE_DERIVED_LOB"] = filled_value answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive" # Fill AARETE_DERIVED_LOB from PRODUCT if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): - answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field( + filled_value = fill_na_from_field( answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "constants/mappings/crosswalk_product_lob.json", ) + answer_dict["AARETE_DERIVED_LOB"] = filled_value answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive" return answer_dicts @@ -75,7 +79,8 @@ def get_crosswalk_fields(answer_dicts): from_field_value = answer_dict.get(from_field) if from_field_value == "N/A": # Special handling for N/A - answer_dict[to_field.field_name] = "N/A" + to_field_value = "N/A" + answer_dict[to_field.field_name] = to_field_value elif not string_utils.is_empty(from_field_value): if from_field_value in crosswalk_mapping.values(): to_field_value = from_field_value @@ -92,6 +97,7 @@ def get_crosswalk_fields(answer_dicts): answer_dict[to_field.field_name] = to_field_value just_mapped.append(to_field.field_name) else: # Other is_empty cases besides N/A - answer_dict[to_field.field_name] = "" + to_field_value = "" + answer_dict[to_field.field_name] = to_field_value return answer_dicts diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index a13cf26..0be0191 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -1388,11 +1388,17 @@ def get_lob_relationship(answer_dicts, exhibit_dict, filename): Helper function to prompt LLM for LOB relationship based on the field and exhibit text. """ programs = answer_dict.get(field) - dynamic_primary_values = f"LOB: {answer_dict.get("AARETE_DERIVED_LOB")}\nPROGRAM: {programs}" + dynamic_primary_values = ( + f"LOB: {answer_dict.get("AARETE_DERIVED_LOB")}\nPROGRAM: {programs}" + ) prompt = prompt_templates.LOB_RELATIONSHIP(exhibit_text, dynamic_primary_values) - llm_answer_raw = llm_utils.invoke_claude(prompt, model_id="sonnet_latest", filename=filename) - llm_answer_final = string_utils.extract_text_from_delimiters(llm_answer_raw, Delimiter.PIPE) + llm_answer_raw = llm_utils.invoke_claude( + prompt, model_id="sonnet_latest", filename=filename + ) + llm_answer_final = string_utils.extract_text_from_delimiters( + llm_answer_raw, Delimiter.PIPE + ) return llm_answer_final for answer_dict in answer_dicts: @@ -1401,13 +1407,13 @@ def get_lob_relationship(answer_dicts, exhibit_dict, filename): exhibit_text = exhibit_dict.get(answer_dict.get("EXHIBIT_PAGE")) # Process for program if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")): - answer_dict["LOB_PROGRAM_RELATIONSHIP"] = prompt_lob_relationship("AARETE_DERIVED_PROGRAM", exhibit_text) + answer_dict["LOB_PROGRAM_RELATIONSHIP"] = prompt_lob_relationship( + "AARETE_DERIVED_PROGRAM", exhibit_text + ) # Process for Product if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PRODUCT")): - answer_dict["LOB_PRODUCT_RELATIONSHIP"] = prompt_lob_relationship("AARETE_DERIVED_PRODUCT", exhibit_text) + answer_dict["LOB_PRODUCT_RELATIONSHIP"] = prompt_lob_relationship( + "AARETE_DERIVED_PRODUCT", exhibit_text + ) return answer_dicts - - - - diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index 1eedfc3..4d36214 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -1,5 +1,6 @@ import ast import hashlib +import logging import os import re from datetime import datetime @@ -581,11 +582,19 @@ def update_lob_for_duals(answer_dicts): # Get SERVICE_TERM value service_term = answer_dict.get("SERVICE_TERM", "").lower() # Check if the SERVICE_TERM contains "Medicare" and "Medicaid" - case insensitive - if "medicare" in service_term and "medicaid" in service_term: + # AND check that there's no explicit PROGRAM or PRODUCT that would override this logic + if ( + "medicare" in service_term + and "medicaid" in service_term + and string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")) + and string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PRODUCT")) + ): prompt = prompt_templates.DUAL_LOB_CHECK(service_term) + logging.debug(f"Prompt for dual LOB check: {prompt}") claude_answer_raw = llm_utils.invoke_claude( prompt, model_id="legacy_sonnet", filename="", max_tokens=200 ) + logging.debug(f"Claude response for dual LOB check: {claude_answer_raw}") claude_answer_extracted = string_utils.extract_text_from_delimiters( claude_answer_raw, Delimiter.PIPE ) diff --git a/fieldExtraction/src/utils/crosswalk_utils.py b/fieldExtraction/src/utils/crosswalk_utils.py index 5446672..f56b112 100644 --- a/fieldExtraction/src/utils/crosswalk_utils.py +++ b/fieldExtraction/src/utils/crosswalk_utils.py @@ -1,4 +1,5 @@ import ast +import logging import src.utils.string_utils as string_utils @@ -61,8 +62,11 @@ def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "") -> str final_mapped = [v for v in mapped_values if v and not string_utils.is_empty(v)] if len(final_mapped) == 1: - return final_mapped[0] + result = final_mapped[0] + return result elif len(final_mapped) > 1: - return "|".join(final_mapped) + result = "|".join(final_mapped) + return result else: - return default if default else original_val + result = default if default else original_val + return result