From 7a44c8575d95c5ef0aedaebe3a6f5c999b326bc1 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 22 May 2025 19:53:15 +0000 Subject: [PATCH] Merged in bugfix/remaining-dynamic-issues (pull request #543) Bugfix/remaining dynamic issues * Add Care1st-->Medicaid mapping for Centene * Streamlined fill_na_mapping * Update prompt for reimbursement-level dynamic fields - remove unknown * Merge branch 'main' into bugfix/remaining-dynamic-issues * Remove test Approved-by: Alex Galarce --- .../mappings/crosswalk_product_lob.json | 2 +- .../crosswalk/mappings/crosswalk_program.json | 2 +- .../src/investment/aarete_derived.py | 30 +++++++------------ .../src/investment/dynamic_funcs.py | 6 ++-- .../src/investment/one_to_n_funcs.py | 3 +- .../src/prompts/investment_prompts.json | 4 +-- .../src/prompts/investment_prompts.py | 3 +- 7 files changed, 20 insertions(+), 30 deletions(-) diff --git a/fieldExtraction/crosswalk/mappings/crosswalk_product_lob.json b/fieldExtraction/crosswalk/mappings/crosswalk_product_lob.json index d4558f8..77615e9 100644 --- a/fieldExtraction/crosswalk/mappings/crosswalk_product_lob.json +++ b/fieldExtraction/crosswalk/mappings/crosswalk_product_lob.json @@ -10,7 +10,7 @@ "Medicare Select" : "Medicare" }, "Centene" : { - + "Care1st" : "Medicaid" }, "Aetna" : { diff --git a/fieldExtraction/crosswalk/mappings/crosswalk_program.json b/fieldExtraction/crosswalk/mappings/crosswalk_program.json index 2c3fc38..181981d 100644 --- a/fieldExtraction/crosswalk/mappings/crosswalk_program.json +++ b/fieldExtraction/crosswalk/mappings/crosswalk_program.json @@ -31,7 +31,7 @@ "Medicare-Medicaid Coordinated Plan (MMCP)" : "MMCP", "Medicare-Medicaid Plan (MMP)" : "MMP", "Program of All-Inclusive Care for the Elderly (PACE)" : "PACE", - "Medicare Advantage (MA)" : "MA", + "Medicare Advantage" : "MA", "Medicare Part C" : "MA", "Supplemental" : "Supp", "Medigap" : "Supp", diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 868374b..3872d1f 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -11,24 +11,12 @@ import os from collections import defaultdict def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path): - if not string_utils.is_empty(results_dict.get(to_field)): - return results_dict.get(to_field) - from_str = results_dict.get(from_field) if string_utils.is_empty(from_str): return results_dict.get(to_field) - MAPPING_DICT = CrosswalkBuilder().from_json(crosswalk_path).mapping - - from_list = [s.strip() for s in from_str.split('|')] - to_list = [] - for from_value in from_list: - if from_value: - to_value = MAPPING_DICT.get(from_value) - if to_value: - to_list.append(to_value) - - return '|'.join(list(set(to_list))) if to_list else results_dict.get(to_field) + to_value = apply_crosswalk(from_str, MAPPING_DICT) + return to_value if to_value else results_dict.get(to_field) def fill_na_mapping(answer_dicts): """ @@ -40,16 +28,14 @@ def fill_na_mapping(answer_dicts): Returns: pd.DataFrame: DataFrame with filled missing values """ - for answer_dict in answer_dicts: # Fill AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM - answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "crosswalk/mappings/crosswalk_program_lob.json") + if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): + answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "crosswalk/mappings/crosswalk_program_lob.json") # Fill AARETE_DERIVED_LOB from PRODUCT - answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "crosswalk/mappings/crosswalk_product_lob.json") - - # Add other mappable fields below - + if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")): + answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "crosswalk/mappings/crosswalk_product_lob.json") return answer_dicts @@ -57,6 +43,7 @@ def get_crosswalk_fields(answer_dicts): crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True) just_mapped = [] for to_field in crosswalk_fields.fields: + from_field = to_field.base_field if from_field in just_mapped: continue @@ -71,10 +58,13 @@ def get_crosswalk_fields(answer_dicts): from_field_value = answer_dict.get(from_field) if not string_utils.is_empty(from_field_value): to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping) + if not to_field_value: to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping_reverse) + answer_dict[to_field.field_name] = to_field_value just_mapped.append(to_field.field_name) + return answer_dicts def get_aarete_derived(aarete_derived_fields: FieldSet, diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index 3940b2d..fd97390 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -21,6 +21,8 @@ def prompt_dynamic(text, field_prompts, filename): """ prompt = investment_prompts.EXHIBIT_LEVEL(text, field_prompts) llm_answer_raw = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename) # Returns dictionary of lists + + llm_answer_final = string_utils.universal_json_load(llm_answer_raw) return llm_answer_final @@ -69,12 +71,13 @@ def get_dynamic_answers(exhibit_chunk, exhibit_header, filename, dynamic_fields exhibit_header_results = prompt_dynamic(text=exhibit_header, field_prompts=dynamic_fields.print_prompt_dict(), filename=filename) - + # Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields for field_name, answer in exhibit_header_results.items(): if not string_utils.is_empty(answer): field = dynamic_fields.get_field(field_name) field.update_valid_values(answer) + field.prompt = field.prompt + ". Ensure ALL values that apply to the specific reimbursement term are included." reimbursement_level_fields.add_field(field) dynamic_fields.remove_field(field_name) @@ -83,7 +86,6 @@ def get_dynamic_answers(exhibit_chunk, exhibit_header, filename, dynamic_fields exhibit_chunk_results = prompt_dynamic(text=exhibit_chunk, field_prompts=dynamic_fields.print_prompt_dict(), filename=filename) - # Process Exhibit Chunk Results - update dynamic fields and add to reimbursement level fields for field_name, answer in exhibit_chunk_results.items(): if not string_utils.is_empty(answer): diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 254fdda..9c13f57 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -104,11 +104,10 @@ def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> lis def get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename): field_prompts = reimbursement_level_fields.get_prompt_dict() prompt = investment_prompts.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts) - # print(prompt) # Debugging line llm_answer_raw = llm_utils.invoke_claude( prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=25000 ) - # print(llm_answer_raw) # Debugging line + llm_answer_final = string_utils.universal_json_load(llm_answer_raw) return llm_answer_final diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index ae21690..d837aa6 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -243,7 +243,7 @@ "field_name": "REIMB_PROV_NAME", "relationship": "one_to_n", "field_type": "dynamic", - "prompt": "Identify any Provider Names that this reimbursement term applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'UNKNOWN'." + "prompt": "Identify any Provider Names that this reimbursement term applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'N/A'." }, { "field_name": "AARETE_DERIVED_EFFECTIVE_DT", @@ -326,7 +326,7 @@ "field_name": "AARETE_DERIVED_PROV_TYPE", "relationship": "one_to_n", "field_type": "TBD", - "prompt": "What Provider Type does the Exhibit apply to? Choose ONLY from the following: {valid_values}. If a provider type is mentioned but doesn't match any values in the list, write 'UNKNOWN'. If no provider type is mentioned at all, write 'N/A'.", + "prompt": "What Provider Type does the Exhibit apply to? Choose ONLY from the following: {valid_values}. If no provider type form the list is mentioned, write 'N/A'.", "valid_values": "VALID_PROV_TYPE" }, { diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 3d699b4..bf79ff2 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -349,8 +349,7 @@ Here are the attributes to be included in the dictionary, and instructions on ho For each field, return ALL correct answers found. There may be more than one correct answer. If there are multiple, return them in a pipe-separated list ("|"). If a list of valid values is provided, ONLY answer with the EXACT value from that list. -If there are no correct answers for a given field because the requested information doesn't apply to this context, write 'N/A' as the answer for that field. -If there are no correct answers for a given field because the requested information should exist but cannot be clearly identified, write 'UNKNOWN' as the answer for that field. +If there are no correct answers for a given field, write 'N/A' as the answer for that field. Here is the text to analyze: