From 098c10b9ca008eaaafd04d2a998f98bd86cb7152 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 22 May 2025 15:04:19 +0000 Subject: [PATCH] Merged in bugfix/dynamic-primary-issues (pull request #541) Bugfix/dynamic primary issues * Update reimbursement exhibit detection * Remove prints * Update prompt * Decrease header size * Prompt update - only return dynamic 1:1 values if applied to entire contract * Testing * pipe-delimited * Merged main into bugfix/dynamic-primary-issues * Fix Product->LOB mapping * Remove test * remove commented crosswalk code * Remove print Approved-by: Alex Galarce --- fieldExtraction/crosswalk/crosswalk_utils.py | 6 ++- .../src/investment/aarete_derived.py | 37 ++++++++++--------- .../src/investment/dynamic_funcs.py | 1 + .../src/investment/file_processing.py | 3 +- .../src/investment/one_to_n_funcs.py | 17 +++++---- .../src/prompts/investment_prompts.json | 8 ++-- .../src/prompts/investment_prompts.py | 16 ++++---- 7 files changed, 48 insertions(+), 40 deletions(-) diff --git a/fieldExtraction/crosswalk/crosswalk_utils.py b/fieldExtraction/crosswalk/crosswalk_utils.py index ee03aec..eb3f063 100644 --- a/fieldExtraction/crosswalk/crosswalk_utils.py +++ b/fieldExtraction/crosswalk/crosswalk_utils.py @@ -187,8 +187,10 @@ def apply_crosswalk( values = ast.literal_eval(val) elif "," in val: values = [v.strip() for v in val.split(",")] # Split by comma if present + elif "|" in val: + values = [v.strip() for v in val.split("|")] else: values = [val] - mapped_values = [mapping.get(v.strip(), default) for v in values] - return ",".join(mapped_values) # Join back into a single string + mapped_values = {mapping.get(v.strip(), default) for v in values} + return "|".join(mapped_values) # Join back into a single string diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index fc26d6b..868374b 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -52,26 +52,29 @@ def fill_na_mapping(answer_dicts): return answer_dicts + def get_crosswalk_fields(answer_dicts): crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True) - for answer_dict in answer_dicts: - for to_field in crosswalk_fields.fields: - if string_utils.is_empty(answer_dict.get(to_field.field_name, None)): # If the to_field is not present, or is empty - from_field = to_field.base_field - if from_field in answer_dict.keys(): - if not string_utils.is_empty(answer_dict[from_field]): - from_field_value = str(answer_dict[from_field]).strip() - crosswalk = CrosswalkBuilder() - crosswalk = crosswalk.from_json(path=os.path.join("crosswalk/mappings", to_field.crosswalk)) - crosswalk_mapping = crosswalk.mapping - crosswalk_mapping_reverse = crosswalk.create_reverse_mapping() + just_mapped = [] + for to_field in crosswalk_fields.fields: + from_field = to_field.base_field + if from_field in just_mapped: + continue + + # Generate Crosswalk + crosswalk = CrosswalkBuilder() + crosswalk = crosswalk.from_json(path=os.path.join("crosswalk/mappings", to_field.crosswalk)) + crosswalk_mapping = crosswalk.mapping + crosswalk_mapping_reverse = crosswalk.create_reverse_mapping() - # Apply mapping - crosswalked_answer = apply_crosswalk(from_field_value, crosswalk_mapping) - # If mapping fails, apply the reverse mapping - if not crosswalked_answer: - crosswalked_answer = apply_crosswalk(from_field_value, crosswalk_mapping_reverse) - answer_dict[to_field.field_name] = crosswalked_answer + for answer_dict in answer_dicts: + from_field_value = answer_dict.get(from_field) + if not string_utils.is_empty(from_field_value): + to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping) + if not to_field_value: + to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping_reverse) + answer_dict[to_field.field_name] = to_field_value + just_mapped.append(to_field.field_name) return answer_dicts def get_aarete_derived(aarete_derived_fields: FieldSet, diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index ef0ad2b..3940b2d 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -38,6 +38,7 @@ def get_dynamic_one_to_one_fields(answer_dicts): if field_name in answer_dict and string_utils.is_empty(answer_dict[field_name]): field_to_add = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name=field.base_field) field_to_add.field_type = "full_context" + field_to_add.prompt = field.prompt + ". Only answer if the answer applies to the ENTIRE contract. Otherwise, answer N/A." one_to_one_fields.add_field(field_to_add) continue return one_to_one_fields diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index c176217..24fdcb5 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -68,6 +68,7 @@ def process_file(file_object, all_dataset, run_timestamp): one_to_n_results = pd.DataFrame([{'FILE_NAME' : filename}]) dynamic_one_to_one_fields = FieldSet() print(f"{datetime_str()} No One to N Found, Skipping - {filename}") + ################## ONE TO ONE ################## one_to_one_results = run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields) # Return dict @@ -138,7 +139,6 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset): - ################## IDENTIFY COMPENSATION EXHIBITS ################## # Get all exhibit headers first all_exhibit_headers = {page: one_to_n_funcs.get_exhibit_header(string_utils.get_exhibit_chunk(text_dict, exhibit_chunk_mapping, page), filename) @@ -146,7 +146,6 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi # Then identify the compensation exhibits reimbursement_exhibits = one_to_n_funcs.identify_reimbursement_exhibits(all_exhibit_headers, filename) - # print("Reimbursement Exhibits: ", reimbursement_exhibits) # Debugging line ################## RUN PROMPTS ################## one_to_n_results = [] diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 7feb9e2..7e2ba66 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -48,9 +48,9 @@ def get_exhibit_header(exhibit_chunk, filename): Returns: str: The extracted exhibit header. """ - prompt = investment_prompts.EXHIBIT_HEADER(exhibit_chunk[0:200]) + prompt = investment_prompts.EXHIBIT_HEADER(exhibit_chunk[0:400]) llm_answer_raw = llm_utils.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 1000 + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename ) llm_answer_final = string_utils.extract_text_from_delimiters( llm_answer_raw, Delimiter.PIPE @@ -75,7 +75,6 @@ def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> lis llm_answer_raw = llm_utils.invoke_claude( prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 2000 ) - yaml_content = string_utils.extract_text_from_delimiters( llm_answer_raw, Delimiter.TRIPLE_BACKTICK) @@ -89,10 +88,14 @@ def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> lis try: # Parse the YAML output into a dictionary exhibits_dict = yaml.safe_load(yaml_content) - # YAML parsing has built-in type conversion for "NO" -> False, "YES" -> True - # So we'll check for boolean values and strings (just in case) - return [str(page) for page, value in exhibits_dict.items() # Convert back to string - if value is True or (isinstance(value, str) and value.upper() == "YES")] + + # If any are values are EXPLICIT, return ONLY those that are EXPLICIT + if any(v == 'EXPLICIT' for v in exhibits_dict.values()): + return [str(page) for page, value in exhibits_dict.items() # Convert back to string + if value == 'EXPLICIT'] + else: + return [str(page) for page, value in exhibits_dict.items() # Convert back to string + if value == 'IMPLICIT'] except yaml.YAMLError as e: print(f"Error parsing YAML: {e}") print(f"Raw LLM output: {llm_answer_raw}") diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index b6fc4b3..62d3ce6 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -98,7 +98,7 @@ "field_name" : "LOB", "relationship" : "one_to_n", "field_type" : "dynamic", - "prompt" : "List any Lines of Business explicitly stated in the text. Choose ONLY from the following: {valid_values}", + "prompt" : "List any Lines of Business explicitly stated in the text. Choose ONLY from the following: {valid_values}. If there are multiple answers, return them in a pipe-separated list ('|')", "valid_values" : "VALID_LOBS" }, { @@ -113,7 +113,7 @@ "field_name" : "NETWORK", "relationship" : "one_to_n", "field_type" : "dynamic", - "prompt" : "List any Networks explicitly stated in the text. Choose ONLY from the following: {valid_values}", + "prompt" : "List any Networks explicitly stated in the text. Choose ONLY from the following: {valid_values}. If there are multiple answers, return them in a pipe-separated list ('|')", "valid_values" : "VALID_NETWORKS" }, { @@ -128,7 +128,7 @@ "field_name" : "PRODUCT", "relationship" : "one_to_n", "field_type" : "dynamic", - "prompt" : "List any Products explicitly stated in the text. Look for the brand name of a health plan or product. Do not include the name of the entire healthplan (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list.", + "prompt" : "List any Products explicitly stated in the text. Look for the brand name of a health plan or product. Do not include the name of the entire healthplan (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list. If there are multiple answers, return them in a pipe-separated list ('|')", "valid_values" : "VALID_PRODUCTS" }, { @@ -142,7 +142,7 @@ "field_name" : "PROGRAM", "relationship" : "one_to_n", "field_type" : "dynamic", - "prompt" : "List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'.", + "prompt" : "List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. If there are multiple answers, return them in a pipe-separated list ('|')", "valid_values" : "VALID_PROGRAMS" }, { diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index d01a7e1..7b28dcb 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -291,6 +291,7 @@ You must capture the complete hierarchy of document identifiers present in the t * Full Attachment names and numbers (e.g., "Attachment C: Commercial-Exchange") * Complete Exhibit numbers/letters with full titles (e.g., "Exhibit 1 - Medicare") * Any other relevant subtitles and section identifiers, including but not limited to 'Article', 'Amendment', 'Schedule', 'Addendum', or similar. +* Any words in the header that may specify what type of information is contained, such as "Compensation Schedule" or "Definitions". * Provider/Entity names when listed with exhibit information - Rules for inclusion: @@ -316,23 +317,22 @@ def IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers: dict, yaml_output: b Consider both explicit and implicit indicators of reimbursement information: EXPLICIT INDICATORS: -- Keywords like "compensation", "payment", "rate", "reimbursement", "fee schedule", "pricing", "financial" +- Exhibits that have "Compensation Schedule", "Reimbursement Schedule", "Rates", or similar in the title. The title must somehow explicitly indicate that the Exhibit contains payment terms. IMPLICIT INDICATORS: - Sections titled "COVERED SERVICES" for specific insurance products typically include payment terms because they define what services are reimbursable and how -- Attachments specifying plan types (Medicare, Medicaid, Commercial) usually contain reimbursement details -- References to specific programs (Medicare Advantage, MMAI) that have distinct payment methodologies +- Attachments specifying plan types (Medicare, Medicaid, Commercial, Marketplace) usually contain reimbursement details +- References to specific programs (Medicare Advantage, CHIP) that have distinct payment methodologies - Attachments that appear to define specific services or service categories - -For each header, return 'YES' if it's likely to contain reimbursement information, or 'NO' if it's not. +For each header, return 'EXPLICIT' if there is an EXPLICIT INDICATOR, 'IMPLICIT' if there is an IMPLICIT INDICATOR, or 'NO' if the header indicates that the exhibit is unlikely to contain reimbursement information. Headers to review: {"\n".join(formatted_headers)} -Return your answer as a {"JSON object" if not yaml_output else "YAML list"} where keys are PAGE NUMBERS and values are 'YES' or 'NO'. +Return your answer as a {"JSON object" if not yaml_output else "YAML list"} where keys are PAGE NUMBERS and values are 'EXPLICIT', 'IMPLICIT', or 'NO'. Feel free to explain your reasoning, but place your final {"JSON" if not yaml_output else "YAML"} output between triple backticks. -If you're uncertain about a header, err on the side of inclusion (YES). +If you're uncertain about a header, err on the side of inclusion ('IMPLICIT'). """ @@ -346,7 +346,7 @@ Here are the attributes to be included in the dictionary, and instructions on ho {fields} [GENERAL INSTRUCTIONS] -For each field, return ALL correct answers found. There may be more than one correct answer. If there are multiple, return them in a comma-separated list. +For each field, return ALL correct answers found. There may be more than one correct answer. If there are multiple, return them in a pipe-separated list ("|"). If a list of valid values is provided, ONLY answer with the EXACT value from that list. If there are no correct answers for a given field because the requested information doesn't apply to this context, write 'N/A' as the answer for that field.