From 0e2622f1569a6daffefae0cc66d2d73e2951102a Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 10 Dec 2025 20:01:52 +0000 Subject: [PATCH] Merged in feature/1toN-Optimization (pull request #799) Feature/1toN Optimization * Update config - remove Haiku 3 * Merged in optimization/trigger-cap (pull request #792) Optimization/trigger cap * trigger cap breakout added * Merge remote-tracking branch 'origin/feature/1toN-Optimization' into optimization/trigger-cap * carveout breakout added Approved-by: Katon Minhas * Merged in bugfix/utah_issue_fixes (pull request #791) Bugfix/utah issue fixes to feature/1toN-Optimization * updated valid values for AARETE_DERIVED_REIMB_METHOD * removed example reimbursements * removed reciprocatory agreements section reimbs * Merged feature/1toN-Optimization into bugfix/utah_issue_fixes * update reimb primary Approved-by: Katon Minhas * Update unit tests * Update llm_utils * Merge branch 'feature/deprecate-haiku-3' into feature/1toN-Optimization * fix over-filtering of lesser of * Merged in bugfix/UT-methodology-breakout (pull request #794) Bugfix/UT methodology breakout * updated valid values for AARETE_DERIVED_REIMB_METHOD * removed example reimbursements * prompt update * prompt update * Merged feature/1toN-Optimization into bugfix/nv_issue_fixes * add service term in mb prompts * Merged feature/1toN-Optimization into bugfix/UT-methodology-breakout * print statement removed * Merge branch 'bugfix/UT-methodology-breakout' of https://bitbucket.org/aarete/doczy.ai into bugfix/UT-methodology-breakout * primary prompt update * remove duplicate prompt Approved-by: Katon Minhas * Merge branch 'main' into feature/1toN-Optimization * Merge branch 'main' into feature/1toN-Optimization * Update preprocessing to make the exhibit_chunk_mapping start at first page * Merge row_funcs.py changes from feature/cross-exhibit-dynamic * Merged in bugfix/UT-grouper-issues (pull request #796) Bugfix/UT grouper issues * updated valid values for AARETE_DERIVED_REIMB_METHOD * removed example reimbursements * prompt update * prompt update * Merged feature/1toN-Optimization into bugfix/nv_issue_fixes * add service term in mb prompts * Merged feature/1toN-Optimization into bugfix/UT-methodology-breakout * print statement removed * Merge branch 'bugfix/UT-methodology-breakout' of https://bitbucket.org/aarete/doczy.ai into bugfix/UT-methodology-breakout * primary prompt update * prompt update * Merge remote-tracking branch 'origin/feature/1toN-Optimization' into bugfix/UT-grouper-issues * removed temp changes * removed temp changes * Update reimb primary Approved-by: Katon Minhas * Merged in bugfix/validation_fixes (pull request #795) bugfix/validation_fixes to feature/1toN-Optimization * updated validation of clean claims reimbursement * Merged feature/1toN-Optimization into bugfix/validation_fixes Approved-by: Katon Minhas * Re-add dynamic codes and reimb-info * Re-structure empty reimbursement prompt * Fix lesser of check overfiltering * Merged in bugfix/generic-methodology-breakout (pull request #797) Bugfix/generic methodology breakout * fee schedule fix * carveout+special case prompt changed * fee schedule changes * prompt changes and column order Approved-by: Katon Minhas * pytest fix * remove prints * Update mapping * Update tests * Update test * Address PR review comments: add logging, fix return types, fix variable propagation, and add tests - Add warning log in code_funcs.py try/except block for grouper code parsing failures - Add warning log in hybrid_smart_chunking_funcs.py except block for JSON parsing failures - Add return type annotation to lesser_of_distribution function - Fix first_reimbursement_page variable propagation issue by returning updated value - Add comprehensive tests for simplify_exhibit, chunk_by_exhibit, and lesser_of_distribution functions * Merged in bugfix/prov_info_json (pull request #802) Bugfix/prov info json to feature/1toN-Optimization * prov_info_json added * remove print statements Approved-by: Katon Minhas Approved-by: Siddhant Medar --- fieldExtraction/.gitignore | 3 + fieldExtraction/constants/constants.py | 1 + .../constants/mappings/crosswalk_program.json | 1 - .../constants/mappings/valid_carveouts.json | 6 +- fieldExtraction/src/codes/code_funcs.py | 16 +- .../src/investment/aarete_derived.py | 5 +- .../src/investment/dynamic_funcs.py | 245 +-- .../src/investment/file_processing.py | 175 ++- .../investment/hybrid_smart_chunking_funcs.py | 31 +- .../src/investment/lesser_of_funcs.py | 102 -- .../src/investment/one_to_n_funcs.py | 463 ++---- .../src/investment/one_to_one_funcs.py | 19 +- fieldExtraction/src/investment/preprocess.py | 44 +- .../src/investment/preprocessing_funcs.py | 252 +-- .../src/investment/prompt_calls.py | 191 +-- fieldExtraction/src/investment/row_funcs.py | 70 +- fieldExtraction/src/investment/table_funcs.py | 683 -------- .../parent_child_preprocessing.py | 3 +- fieldExtraction/src/prompts/fieldset.py | 10 +- .../src/prompts/investment_prompts.json | 157 +- .../src/prompts/prompt_templates.py | 559 +++---- fieldExtraction/src/utils/crosswalk_utils.py | 7 +- fieldExtraction/src/utils/logging_utils.py | 3 + fieldExtraction/tests/test_crosswalk_utils.py | 12 +- fieldExtraction/tests/test_dynamic.py | 34 +- .../tests/test_exhibit_chunk_mapping.py | 43 - fieldExtraction/tests/test_one_to_n.py | 1396 ++++++++--------- .../tests/test_preprocessing_funcs.py | 146 +- .../tests/test_preprocessing_integration.py | 135 -- fieldExtraction/tests/test_table_funcs.py | 1080 ------------- 30 files changed, 1664 insertions(+), 4228 deletions(-) delete mode 100644 fieldExtraction/src/investment/lesser_of_funcs.py delete mode 100644 fieldExtraction/src/investment/table_funcs.py delete mode 100644 fieldExtraction/tests/test_exhibit_chunk_mapping.py delete mode 100644 fieldExtraction/tests/test_preprocessing_integration.py delete mode 100644 fieldExtraction/tests/test_table_funcs.py diff --git a/fieldExtraction/.gitignore b/fieldExtraction/.gitignore index 2321e80..04c255f 100644 --- a/fieldExtraction/.gitignore +++ b/fieldExtraction/.gitignore @@ -90,3 +90,6 @@ output_consolidated/ *.zip *.xlsx *.xlsb + +# Shell scripts (debugging/utility scripts) +*.sh diff --git a/fieldExtraction/constants/constants.py b/fieldExtraction/constants/constants.py index 7a8bed2..ab66359 100644 --- a/fieldExtraction/constants/constants.py +++ b/fieldExtraction/constants/constants.py @@ -186,6 +186,7 @@ class Constants: .list ) + #################################### Carveout and Special Case #################################### self.VALID_CARVEOUTS = ( CrosswalkBuilder() .from_json(path="constants/mappings/valid_carveouts.json") diff --git a/fieldExtraction/constants/mappings/crosswalk_program.json b/fieldExtraction/constants/mappings/crosswalk_program.json index 9512e5a..d44a7c5 100644 --- a/fieldExtraction/constants/mappings/crosswalk_program.json +++ b/fieldExtraction/constants/mappings/crosswalk_program.json @@ -6,7 +6,6 @@ "mapping": { "Children's Health Insurance Program (CHIP)" : "CHIP", "Children's Health Insurance Program Perinate (CHIP-P)" : "CHIPP", - "KidsFirst" : "CHIP", "Aged, Blind, and Disabled (ABD)" : "ABD", "Alternative Benefit Plan (ABP)" : "ABP", "Community First Choice (CFC)" : "CFC", diff --git a/fieldExtraction/constants/mappings/valid_carveouts.json b/fieldExtraction/constants/mappings/valid_carveouts.json index 09ef36f..9892ec9 100644 --- a/fieldExtraction/constants/mappings/valid_carveouts.json +++ b/fieldExtraction/constants/mappings/valid_carveouts.json @@ -3,11 +3,11 @@ "description": "Valid carveout types and descriptions" }, "mapping": { - "BASE_COVERED_SERVICES": "Describes foundational payment methodologies for standard covered services, including primary coverage determinations. This refers to payment methodologies that establish the fundamental reimbursement structure, not exceptions or adjustments to it. Examples:\n- 'covered services'\n- 'Covered Services not included in the Service Categories in Table 1 shall be reimbursed at 100% of the Medicare fee schedule'\n- Primary coverage methodologies in dual products (e.g., 'services primary to Medicare', 'services primary to Medicaid')\n- Base reimbursement rates that form the foundation of the payment structure", + "BASE_COVERED_SERVICES": "Base reimbursement rates that form the foundation of the payment structure. Describes foundational payment methodologies for standard covered services, including primary coverage determinations. This refers to payment methodologies that establish the fundamental reimbursement structure, not exceptions or adjustments to it. Examples: 'covered services' 'Covered Services not included...', 'services primary to Medicare', etc.", "PAYMENT_CARVEOUT": "Describes payments for specific services, procedures, level designations, or specific DRG or codes that are carved out from the base/foundational payment methodology with distinct payment terms. These are exceptions or specialty payments that differ from the payments for standard/ base services. Examples include 'Radiology covered services', 'Anesthesia covered services', 'Pharmaceuticals' and 'HEMATOCRIT (CPT CODE: 85014)'. Do NOT classify as payment carveout if the service represents a foundational coverage determination (such as primary payer methodologies in dual products) rather than a different payment to standard one.", "UNLISTED_CODE": "Describes payments for codes that are not listed on a fee schedule. This is similar to DEFAULT_TERM. Only populate with Y if the Service or Methodology specifically mentions Codes.", "ADDITION": "Describes payments that include a supplemental amount added to or above the base reimbursement rate. Populate with Y if the Methodology describes any additional payments, supplemental amounts, or add-on fees beyond the standard reimbursement.", - "TRIGGER_CAP": "Describes a new payment methodology which becomes applicable/ gets triggered after a defined quantitative threshold is reached. Unlike supplemental payments, this refers to fundamentally changed new reimbursement methodology applicable only beyond the threshold. Valid thresholds can be based on dollar amounts, number of days, visits, units, or other measurable metrics (but not qualitative thresholds, e.g., 'where there is no Marketplace Medicare Rate'). Look for language indicating a clear change in the payment methodology when a defined quantitative threshold is exceeded, such as 'when charges exceed X, pay Y', or 'after 3 days, payment changes to $1,200 per diem'. Do NOT classify as TRIGGER_CAP if:\n - it is outlier threshold value\n - it is stop-loss provision \n - the methodology is just the threshold value", + "TRIGGER_CAP": "Describes a new payment methodology which becomes applicable/ gets triggered after a defined quantitative threshold is reached. Unlike supplemental payments, this refers to fundamentally changed new reimbursement methodology applicable only beyond the threshold. Valid thresholds can be based on dollar amounts, number of days, visits, units, or other measurable metrics (but not qualitative thresholds, e.g., 'where there is no Marketplace Medicare Rate'). Look for language indicating a clear change in the payment methodology when a defined quantitative threshold is exceeded, such as 'when charges exceed X, pay Y', or 'after 3 days, payment changes to $1,200 per diem'. Do NOT classify as TRIGGER_CAP if the term better fits the criteria for Outlier, Stop-Loss, or simple thresholds.", "NEVER_EVENT": "Indicates that Never Events (e.g. sepsis acquired while inpatient) will not be covered. This may be phrased as 'no liability' or 'no payment'.", "EXPERIMENTAL_INVESTIGATIONAL": "Describes payments for experimental and investigational procedures.", "NOT_MEDICALLY_NECESSARY": "Describes payments for procedures deemed not medically necessary (or medically unnecessary)", @@ -17,6 +17,6 @@ "PROFESSIONAL_COMPONENT": "If Modifier 26 (professional component) is present.", "DIAGNOSTIC_PROCEDURES": "Describes payments for diagnostic procedures.", "READMISSIONS": "Describes payment for subsequent admission with the same diagnostic category of codes as the initial admission.", - "DEFAULT_TERM": "Indicates whether this service/methodology serves as a default/fallback payment rate when other payment method does not apply. The default/fallback payment rate may be a lower rate. It may be specific to certain services or applicable to all services. Do not classify as default if:\n- The methodology is the primary/base reimbursement approach, even if defined by exclusion\n- No default/fallback language is present\n\nExamples: \n- 'For any services not specifically listed in Exhibits A-D, reimbursement shall be 50% of billed charges.' \n- 'If there is no payment rate under the Medicare Fee Schedule, alternative rate e.g. Medicare Rate applies.'" + "DEFAULT_TERM": "Indicates whether this service/methodology serves as a default/fallback payment rate when other payment method does not apply. The default/fallback payment rate may be a lower rate. It may be specific to certain services or applicable to all services. Do not classify as default if the methodology is the primary/base reimbursement approach, even if defined by exclusion OR if no default/fallback language is present. Examples: 'For any services not specifically listed in Exhibits A-D, reimbursement shall be 50% of billed charges.', 'If there is no payment rate under the Medicare Fee Schedule, alternative rate e.g. Medicare Rate applies.'" } } \ No newline at end of file diff --git a/fieldExtraction/src/codes/code_funcs.py b/fieldExtraction/src/codes/code_funcs.py index 575f8a1..fffd78f 100644 --- a/fieldExtraction/src/codes/code_funcs.py +++ b/fieldExtraction/src/codes/code_funcs.py @@ -1,3 +1,4 @@ +import logging import os import re @@ -142,6 +143,7 @@ def code_category(service, proc_category, hcpcs_level2_mapping, filename): for answer in claude_answer_final: if "0000" in answer: code_answer_dict["PROCEDURE_CD"].append(answer) + code_answer_dict["PROCEDURE_CD_DESC"].append("") elif answer in hcpcs_level2_mapping.values(): code = list(hcpcs_level2_mapping.keys())[ list(hcpcs_level2_mapping.values()).index(answer) @@ -455,9 +457,17 @@ def fill_grouper_cd_desc(answer_dict, constants: Constants): grouper_cd = eval(grouper_cd) if "[" in grouper_cd else grouper_cd for code in grouper_cd: - code_without_severity = int( - code.split("-")[0] - ) # Remove severity level if present + try: + code_without_severity = int( + code.split("-")[0] + ) # Remove severity level if present + except Exception as e: + logging.warning( + f"Failed to parse grouper code '{code}' as integer: {type(e).__name__}: {str(e)}. " + f"Using original code value." + ) + code_without_severity = code + if grouper_type == "APR-DRG" and ( code_without_severity in constants.GROUPER_APR_DRG_MAPPING ): diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 593e34d..757049f 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -86,10 +86,13 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): individual_from_field_value = individual_from_field_value.strip() if individual_from_field_value in crosswalk.mapping.keys(): + # Value is a key in the mapping - map it to the target value to_field_answer_list.append(crosswalk.mapping.get( individual_from_field_value )) - elif from_field_value in crosswalk.mapping.values(): + elif individual_from_field_value in crosswalk.mapping.values(): + # Value is already in the target format (e.g., "CHIP" when target is "CHIP") + # Keep it as-is for AARETE_DERIVED fields, or use reverse mapping for others if "AARETE_DERIVED" in to_field_name: to_field_answer_list.append(individual_from_field_value) else: diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index 83728a6..68f61f4 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -9,89 +9,12 @@ from src.prompts import prompt_templates from src.prompts.fieldset import Field, FieldSet -def add_full_context_field(one_to_one_fields, field_to_add, answer_dicts): - field_to_add.field_type = "full_context" - field_to_add.relationship = "one_to_one" - - # Special Handling for Program and Product: - # If there are multiple non-empty AARETE_DERIVED_LOB values in the contract, don't ask for Program or Product full context - if field_to_add.field_name in ["PROGRAM", "PRODUCT"]: - lob_values = [ - answer_dict.get("AARETE_DERIVED_LOB", "") for answer_dict in answer_dicts - ] - non_empty_lobs = [lob for lob in lob_values if not string_utils.is_empty(lob)] - if len(non_empty_lobs) > 1: - return one_to_one_fields - - if field_to_add.field_name == "CLAIM_TYPE_CD": - field_to_add.prompt = ( - field_to_add.prompt - + prompt_templates.FULL_CONTEXT_CLAIM_TYPES_ADDITIONAL_INSTRUCTION() - ) - - field_to_add.prompt = ( - field_to_add.prompt - + prompt_templates.FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(field_to_add.allow_na) - ) - - one_to_one_fields.add_field(field_to_add) - return one_to_one_fields - - -def get_dynamic_one_to_one_fields(answer_dicts): - """ - Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts - """ - # These fields get passed to 1:1 if ALL of the answers are empty - all_empty_fields = FieldSet( - config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True - ).combine(FieldSet( - config.FIELD_JSON_PATH, field_type="special_case_primary" - )) - - any_empty_fields = FieldSet() # Empty for now because there are no any_empty_fields - - one_to_one_fields = FieldSet() # Empty FieldSet to populate if criteria are met - - # Handle ANY empty fields - for field in any_empty_fields.fields: - field_name = field.field_name - for answer_dict in answer_dicts: - if string_utils.is_empty(answer_dict.get(field_name)): - one_to_one_fields = add_full_context_field( - one_to_one_fields, field, answer_dicts - ) - continue - - # Handle ALL empty fields - for field in all_empty_fields.fields: - field_name = field.field_name - if "REIMB" in field_name: # Don't do this for the REIMB_ fields - continue - if all( - [ - string_utils.is_empty(answer_dict.get(field_name)) - for answer_dict in answer_dicts - ] - ): - field_to_add = Field.load_from_file( - file_path=config.FIELD_JSON_PATH, - field_name=field.base_field if field.base_field else field.field_name - ) # Try base_field first, fall back to field_name if base_field is None/empty - one_to_one_fields = add_full_context_field( - one_to_one_fields, field_to_add, answer_dicts - ) - return one_to_one_fields - - def dynamic_primary( exhibit_text: str, - exhibit_header: str, exhibit_level_answers: dict, constants: Constants, filename: str, - dynamic_fields: FieldSet, - reimbursement_level_fields: FieldSet, + dynamic_primary_fields: FieldSet, ): """ Processes dynamic primary fields one-by-one from exhibit header and chunk text. @@ -107,12 +30,13 @@ def dynamic_primary( - exhibit_level_answer_dict: Dictionary of exhibit-level answers - reimbursement_level_fields: FieldSet of fields assigned to reimbursement level """ - if not dynamic_fields.contains_fields(): + if not dynamic_primary_fields.contains_fields(): return {}, FieldSet() - dynamic_to_reimbursement_level_fields = FieldSet() + dynamic_reimbursement_fields = FieldSet() exhibit_level_answer_dict = {} - for field in dynamic_fields.fields: + + for field in dynamic_primary_fields.fields: exhibit_text_answer = prompt_calls.prompt_dynamic_primary( exhibit_text, field, @@ -123,19 +47,16 @@ def dynamic_primary( # If there is at least ONE Non-N/A answers in the Exhibit if not string_utils.is_empty(exhibit_text_answer): field.update_valid_values(exhibit_text_answer + " | N/A") - dynamic_to_reimbursement_level_fields.add_field(field) - dynamic_fields.remove_field(field.field_name) + dynamic_reimbursement_fields.add_field(field) + dynamic_primary_fields.remove_field(field.field_name) # If the field is not found, add to Exhibit-Level answers (the answer will be N/A or similar) else: exhibit_level_answer_dict[field.field_name] = exhibit_text_answer # Update exhibit_level_answers.update(exhibit_level_answer_dict) - reimbursement_level_fields.combine( - dynamic_to_reimbursement_level_fields, inplace=True - ) - return exhibit_level_answers, reimbursement_level_fields + return exhibit_level_answers, dynamic_reimbursement_fields def dynamic( @@ -145,7 +66,7 @@ def dynamic( constants: Constants, filename: str, dynamic_fields: FieldSet, - reimbursement_level_fields: FieldSet, + dynamic_reimbursement_fields: FieldSet, ): """ Processes dynamic (code and provider info) fields from exhibit header and chunk text. @@ -166,30 +87,28 @@ def dynamic( dynamic_to_reimbursement_level_fields = FieldSet() exhibit_level_answer_dict = {} - # Check Exhibit Header - exhibit_header_results = prompt_calls.prompt_dynamic( - text=exhibit_header, - field_prompts=dynamic_fields.print_prompt_dict(constants), - filename=filename, - ) - # Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields - for field_name, answer in exhibit_header_results.items(): - if ( - not string_utils.is_empty(answer) - and field_name in dynamic_fields.list_fields() - ): - field = dynamic_fields.get_field(field_name) - field.update_valid_values(answer) - if field.reimb_level_prompt: - field.prompt = field.reimb_level_prompt - else: + if not string_utils.is_empty(exhibit_header): + # Check Exhibit Header + exhibit_header_results = prompt_calls.prompt_dynamic( + text=exhibit_header, + field_prompts=dynamic_fields.print_prompt_dict(constants), + filename=filename, + ) + # Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields + for field_name, answer in exhibit_header_results.items(): + if ( + not string_utils.is_empty(answer) + and field_name in dynamic_fields.list_fields() + ): + field = dynamic_fields.get_field(field_name) + field.update_valid_values(answer) field.prompt = ( field.prompt + ". Ensure ALL values that apply to the specific reimbursement term are included." ) - reimbursement_level_fields.add_field(field) - dynamic_fields.remove_field(field_name) + dynamic_reimbursement_fields.add_field(field) + dynamic_fields.remove_field(field_name) # Check Exhibit Text if dynamic_fields.contains_fields(): @@ -214,8 +133,116 @@ def dynamic( # Update exhibit_level_answers.update(exhibit_level_answer_dict) - reimbursement_level_fields.combine( + dynamic_reimbursement_fields.combine( dynamic_to_reimbursement_level_fields, inplace=True ) - return exhibit_level_answers, reimbursement_level_fields + return exhibit_level_answers, dynamic_reimbursement_fields + + +def dynamic_assignment(reimbursement_primary_answers: list[dict[str, str]], dynamic_reimbursement_fields: FieldSet, exhibit_text_simplified: str, page_num: str, constants: Constants, filename: str) -> list[dict[str, str]]: + + # Process each answer and field individually + answer_dicts = [] + + # TODO: Cache here + # instruction, prompt = prompt_templates.DYNAMIC_ASSIGNMENT + + # TODO: Multithread this loop + for answer_dict in reimbursement_primary_answers: + service_term = answer_dict['SERVICE_TERM'] + reimb_term = answer_dict['REIMB_TERM'] + for dynamic_field in dynamic_reimbursement_fields.fields: + dynamic_field_answer = prompt_calls.prompt_dynamic_assignment(service_term, reimb_term, dynamic_field, exhibit_text_simplified, page_num, constants, filename) + answer_dict.update(dynamic_field_answer) + answer_dicts.append(answer_dict) + + return answer_dicts + + +def add_one_to_one_field(one_to_one_fields, field_to_add, answer_dicts, constants: Constants): + field_to_add.field_type = "smart_chunked" + field_to_add.relationship = "one_to_one" + + # Special Handling for Program, Product, and Network: + # If any LOB value has been found, skip PROGRAM, PRODUCT, and NETWORK + # Only search for these fields when NO LOB has been found + # NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB + # is populated later via crosswalk and may be derived from PROGRAM/PRODUCT + if field_to_add.field_name in ["PROGRAM", "PRODUCT", "NETWORK"]: + lob_values = [ + answer_dict.get("LOB", "") for answer_dict in answer_dicts + ] + unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)]) + if len(unique_lobs) > 0: + return one_to_one_fields + + if field_to_add.field_name == "CLAIM_TYPE_CD": + field_to_add.prompt = ( + field_to_add.prompt + + prompt_templates.FULL_CONTEXT_CLAIM_TYPES_ADDITIONAL_INSTRUCTION() + ) + + field_to_add.prompt = ( + field_to_add.prompt + + prompt_templates.FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(field_to_add.allow_na) + ) + + if field_to_add.valid_values: + field_to_add.keywords = field_to_add.get_valid_values(constants) + + one_to_one_fields.add_field(field_to_add) + return one_to_one_fields + + +def get_dynamic_one_to_one_fields(answer_dicts: list[dict[str, str]], constants: Constants): + """ + Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts + """ + # These fields get passed to 1:1 if ALL of the answers are empty + all_empty_fields = FieldSet( + config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True + ) + + one_to_one_fields = FieldSet() # Empty FieldSet to populate if criteria are met + + # Check if any LOB value has been found - if so, skip PROGRAM, PRODUCT, NETWORK + # Only search for PROGRAM, PRODUCT, NETWORK when NO LOB has been found + # NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB + # is populated later via crosswalk and may be derived from PROGRAM/PRODUCT + lob_values = [ + answer_dict.get("LOB", "") for answer_dict in answer_dicts + ] + unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)]) + has_lob = len(unique_lobs) > 0 + + # Handle ALL empty fields + for field in all_empty_fields.fields: + field_name = field.field_name + if "REIMB" in field_name: # Don't do this for the REIMB_ fields + continue + + empty_count = sum( + 1 for answer_dict in answer_dicts + if string_utils.is_empty(answer_dict.get(field_name)) + ) + total_count = len(answer_dicts) + + if empty_count == total_count: + # Skip PROGRAM, PRODUCT, NETWORK if any LOB has been found + # Only search for these fields when NO LOB has been found + if has_lob and field_name in ["PROGRAM", "PRODUCT", "NETWORK"]: + continue + + field_to_add = Field.load_from_file( + file_path=config.FIELD_JSON_PATH, + field_name=field.base_field if field.base_field else field.field_name + ) # Try base_field first, fall back to field_name if base_field is None/empty + one_to_one_fields = add_one_to_one_field( + one_to_one_fields, field_to_add, answer_dicts, constants + ) + + return one_to_one_fields + + + diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 2a9c849..50248dd 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -2,19 +2,8 @@ import logging import pandas as pd import src.codes.code_funcs as code_funcs -import src.investment.aarete_derived as aarete_derived -import src.investment.dynamic_funcs as dynamic_funcs -import src.investment.one_to_n_funcs as one_to_n_funcs -import src.investment.one_to_one_funcs as one_to_one_funcs -import src.investment.postprocess as postprocess -import src.investment.postprocessing_funcs as postprocessing_funcs -import src.investment.preprocess as preprocess -import src.investment.hybrid_smart_chunking_funcs as hybrid_smart_chunking_funcs -import src.investment.row_funcs as row_funcs -import src.investment.tin_npi_funcs as tin_npi_funcs -import src.utils.io_utils as io_utils -import src.utils.logging_utils as logging_utils -import src.utils.string_utils as string_utils +from src.investment import preprocessing_funcs, aarete_derived, dynamic_funcs, one_to_n_funcs, one_to_one_funcs, postprocess, postprocessing_funcs, preprocess, hybrid_smart_chunking_funcs, row_funcs, tin_npi_funcs +from src.utils import io_utils, logging_utils, string_utils from constants.constants import Constants from src import config from src.prompts.fieldset import FieldSet @@ -41,14 +30,11 @@ def process_file(file_object, constants: Constants, run_timestamp): contract_text = preprocess.clean_text(contract_text) text_dict, top_sheet_dict = preprocess.split_text(contract_text) text_dict, header, footer = preprocess.find_headers_and_footers(text_dict) - text_dict = preprocess.clean_tables( - text_dict, constants.EXHIBIT_HEADER_MARKERS, filename - ) # ONE TO N PROCESSING one_to_n_results = pd.DataFrame() # Initialize empty DataFrame for one_to_n_results if process_one_to_n: - exhibit_dict, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking( + exhibit_chunk_mapping, all_exhibit_headers = preprocess.one_to_n_exhibit_chunking( text_dict, constants.EXHIBIT_HEADER_MARKERS, filename ) logging.info(f"{datetime_str()} Preprocessing Complete - {filename}") @@ -56,18 +42,20 @@ def process_file(file_object, constants: Constants, run_timestamp): one_to_n_results = pd.DataFrame([{"FILE_NAME": filename}]) # Initialize here if string_utils.contains_reimbursement(contract_text): - one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts( - filename, exhibit_dict, all_exhibit_headers, constants + one_to_n_results, dynamic_one_to_one_fields, first_reimbursement_page = run_one_to_n_prompts( + text_dict, exhibit_chunk_mapping, all_exhibit_headers, constants, filename ) if not one_to_n_results.empty: one_to_n_results["FILE_NAME"] = filename one_to_n_results = postprocessing_funcs.generate_reimb_ids(one_to_n_results) logging.info(f"{datetime_str()} One to N Complete - {filename}") else: + first_reimbursement_page = '1' logging.info(f"{datetime_str()} No Reimbursement Found, Skipping - {filename}") final_results = one_to_n_results # Set as final results else: + first_reimbursement_page = '1' logging.info(f"{datetime_str()} Fields not configured for One to N, Skipping - {filename}") # ONE TO ONE PROCESSING @@ -78,6 +66,7 @@ def process_file(file_object, constants: Constants, run_timestamp): text_dict, top_sheet_dict, dynamic_one_to_one_fields, + first_reimbursement_page, constants, ) one_to_one_results["FILE_NAME"] = filename @@ -113,6 +102,7 @@ def process_file(file_object, constants: Constants, run_timestamp): io_utils.write_local(final_df, filename, "", "individual") logging.info(f"{datetime_str()} Writing Complete - {filename}") + return final_df @@ -120,10 +110,11 @@ def run_one_to_one_prompts( filename: str, contract_text: str, text_dict: dict[str, str], - top_sheet_dict: dict[str,str], + top_sheet_dict, dynamic_one_to_one_fields: FieldSet, + first_reimbursement_page: str, constants: Constants, -)->dict[str, str]: +): ################## INITIALIZE FIELDS ################## one_to_one_fields = FieldSet( @@ -140,10 +131,11 @@ def run_one_to_one_prompts( hybrid_smart_chunked_answers_dict = hybrid_smart_chunking_funcs.run_hybrid_smart_chunked_fields( one_to_one_fields, constants, contract_text, filename, text_dict ) + ################## RUN FULL CONTEXT PROMPTS ################## full_context_answers_dict = one_to_one_funcs.run_full_context_fields( - one_to_one_fields, contract_text, constants, filename + one_to_one_fields, contract_text, text_dict, first_reimbursement_page, constants, filename ) ################## COMBINE ANSWERS ################ # Prefer HSC answers over full_context when HSC value is not N/A @@ -174,102 +166,113 @@ def run_one_to_one_prompts( return one_to_one_results[0] -def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants): +def run_one_to_n_prompts(text_dict: dict[str, str], + exhibit_chunk_mapping: dict[str, list[str]], + all_exhibit_headers: dict[str, str], + constants: Constants, + filename: str): one_to_n_results = [] - seen_pairs = set() + first_reimbursement_page = '1' - for exhibit_page, exhibit_text in exhibit_dict.items(): + #################################### PROCESS EACH EXHIBIT SEPARATELY #################################### + for exhibit_page, exhibit_page_nums in exhibit_chunk_mapping.items(): - ############################### Initialize Fields ############################### - exhibit_level_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="exhibit_level" - ) - reimbursement_level_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="reimbursement_level" - ) - dynamic_primary_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="dynamic_primary" - ) - dynamic_reimb_info_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="dynamic_reimb_info" - ) - dynamic_code_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="dynamic_code" - ) - special_case_primary_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="special_case_primary" - ) - - ############################### Get Exhibit Header ############################### - exhibit_header = one_to_n_funcs.get_exhibit_header( - all_exhibit_headers, exhibit_page - ) + exhibit_text_original = "\n".join([text_dict[page_num] for page_num in exhibit_page_nums]) + exhibit_header = all_exhibit_headers.get(exhibit_page, "") ############################### Get Exhibit Level ############################### - exhibit_level_answers, reimbursement_level_fields = ( + exhibit_level_answers, dynamic_reimbursement_fields = ( one_to_n_funcs.exhibit_level( - exhibit_text, + exhibit_text_original, exhibit_header, exhibit_page, - exhibit_level_fields, - dynamic_primary_fields, - dynamic_code_fields, - dynamic_reimb_info_fields, - reimbursement_level_fields, constants, filename, ) - ) + ) # dict, FieldSet logging.debug(f"Exhibit Level Answers for {filename}, Page {exhibit_page}: {exhibit_level_answers}") - ############################### Get Reimbursement-Level ############################### - reimbursement_primary_answers, special_case_primary_answers = ( - one_to_n_funcs.reimbursement_level( - exhibit_text, - reimbursement_level_fields, - special_case_primary_fields, - exhibit_level_answers["EXHIBIT_LESSER_OF_STATEMENT"], - seen_pairs, + all_exhibit_reimbursements, all_exhibit_special_case = [], [] + #################################### PAGE-BY-PAGE REIMBURSEMENT PRIMARY #################################### + for page_num in exhibit_page_nums: + reimbursement_level_answers, special_case_answers, first_reimbursement_page = process_page_one_to_n( + text_dict, + exhibit_page_nums, exhibit_page, + page_num, + first_reimbursement_page, + dynamic_reimbursement_fields, filename, - constants, + constants ) - ) - logging.debug(f"Reimbursement Primary Answers for {filename}, Page {exhibit_page}: {reimbursement_primary_answers}") - - ################################ Get Breakouts ############################### - reimbursement_level_answers, special_case_breakout_answers = ( - one_to_n_funcs.breakout( - reimbursement_primary_answers, - special_case_primary_answers, - special_case_primary_fields, - filename, - constants, - ) - ) + all_exhibit_reimbursements += reimbursement_level_answers + all_exhibit_special_case += special_case_answers ################################ Combine Answers ############################### all_exhibit_rows = row_funcs.combine_one_to_n( - exhibit_text, - reimbursement_level_answers, + exhibit_text_original, + all_exhibit_reimbursements, + all_exhibit_special_case, exhibit_level_answers, - special_case_breakout_answers, - filename, + filename ) # returns list of dicts ################################ Mapping and Cleaning ############################### - all_exhibit_rows = one_to_n_funcs.one_to_n_cleaning(all_exhibit_rows, exhibit_text, constants, filename) + all_exhibit_rows = one_to_n_funcs.one_to_n_cleaning(all_exhibit_rows, exhibit_text_original, constants, filename) ################################ Add to Total ############################### one_to_n_results += all_exhibit_rows ################## Add N/A Dynamic or Exhibit to One-to-One ################## dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields( - one_to_n_results + one_to_n_results, constants ) ################## CONVERT TO DF ################## one_to_n_df = pd.DataFrame(one_to_n_results) - return one_to_n_df, dynamic_one_to_one_fields + return one_to_n_df, dynamic_one_to_one_fields, first_reimbursement_page + + +def process_page_one_to_n(text_dict: dict[str, str], exhibit_page_nums: list[str], exhibit_page: str, page_num: str, first_reimbursement_page: str, dynamic_reimbursement_fields: FieldSet, filename: str, constants: Constants): + + page_text = text_dict[page_num] + exhibit_text_simplified = preprocessing_funcs.simplify_exhibit(text_dict, exhibit_page_nums, page_num) + + ############################### Reimbursement Primary ############################### + reimbursement_level_answers = one_to_n_funcs.reimbursement_level( + page_text, + constants, + filename + ) + + if not reimbursement_level_answers: + return [], [], first_reimbursement_page + + # Track the first page with reimbursements + if first_reimbursement_page == '1': + first_reimbursement_page = exhibit_page + + logging.debug(f"Reimbursement Primary Answers for {filename}, Page {page_num}: {reimbursement_level_answers}") + + ################################ Carveouts and Special Case ################################ + reimbursement_level_answers, special_case_answers = one_to_n_funcs.carveout_and_special_case( + reimbursement_level_answers, constants, filename + ) # Breaks the reimbursement answers into reimbursments (regular lines) and special cases (distributed across exhibit) + + ############################### Dynamic Assignment ############################### + reimbursement_level_answers = dynamic_funcs.dynamic_assignment(reimbursement_level_answers, dynamic_reimbursement_fields, exhibit_text_simplified, page_num, constants, filename) + + ############################### Lesser of Distribution ############################### + reimbursement_level_answers = one_to_n_funcs.lesser_of_distribution(reimbursement_level_answers, exhibit_text_simplified, page_num, constants, filename) + + ################################ Get Breakouts ############################### + reimbursement_level_answers, special_case_answers = one_to_n_funcs.breakout( + reimbursement_level_answers, + special_case_answers, + filename, + constants, + ) # list[dict[str, str]], list[dict[str, str]] + + return reimbursement_level_answers, special_case_answers, first_reimbursement_page \ No newline at end of file diff --git a/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py b/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py index d89e853..e168340 100644 --- a/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py +++ b/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py @@ -241,9 +241,9 @@ def run_hybrid_smart_chunked_fields( ) continue - llm_prompt = field.get_prompt( + llm_prompt = field.get_prompt_dict( constants - ) # Actual extraction prompt from investment_prompts.json + ) # Returns dict of {field_name : prompt} prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE( context, llm_prompt ) @@ -256,18 +256,29 @@ def run_hybrid_smart_chunked_fields( raw = llm_utils.invoke_claude( prompt, "sonnet_latest", filename, max_tokens=8192 ) + try: + parsed = string_utils.universal_json_load( + raw + ) # returns dict - parsed = string_utils.extract_text_from_delimiters( - raw, Delimiter.PIPE - ) + val = parsed.get(field.field_name) + if isinstance(val, list): + parsed[field.field_name] = "|".join(map(str, val)) + elif isinstance(val, str): + parsed[field.field_name] = val + answers_dict[field.field_name] = parsed[field.field_name] - answers_dict[field.field_name] = parsed if parsed else "N/A" - - if answers_dict[field.field_name] == "N/A": - field.field_type = "full_context" + if answers_dict[field.field_name] == "N/A": + field.field_type = "full_context" + logging.warning( + f"Obtained N/A for {field.field_name}, marking as full_context" + ) + except Exception as e: logging.warning( - f"Obtained N/A for {field.field_name}, marking as full_context" + f"Failed to parse JSON response for field {field.field_name}: " + f"{type(e).__name__}: {str(e)}. Setting field value to N/A." ) + answers_dict[field.field_name] = "N/A" except Exception as e: # Mark field for full_context retry and preserve error info diff --git a/fieldExtraction/src/investment/lesser_of_funcs.py b/fieldExtraction/src/investment/lesser_of_funcs.py deleted file mode 100644 index 94fe15a..0000000 --- a/fieldExtraction/src/investment/lesser_of_funcs.py +++ /dev/null @@ -1,102 +0,0 @@ -import logging -import re - -import src.investment.prompt_calls as prompt_calls -from src.utils import string_utils - - -def has_explicit_lesser_of_carveout(reimb_term: str) -> bool: - """Check for explicit 'lesser of' carve-out language in the reimbursement term. - - This function uses a list of keywords to identify if the reimbursement term contains explicit language indicating a 'lesser of' carve-out. - - Args: - reimb_term (str): The reimbursement term to check. - - Returns: - bool: True if 'lesser of' carve-out language is found, False otherwise. - """ - - carveout_patterns = ["not subject to.*lesser", "not bound by.*lesser"] - reimb_lower = reimb_term.lower() - return any(re.search(pattern, reimb_lower) for pattern in carveout_patterns) - - -def create_exhibit_lesser_of_row( - original_row: dict, exhibit_lesser_of: str, filename: str -) -> dict: - """Create enhanced row with exhibit lesser-of constraint applied, avoiding redundancy with substring checks. - - Args: - original_row (dict): Original row dictionary containing reimbursement details. - exhibit_lesser_of (str): Exhibit lesser-of statement to apply. - filename (str): The name of the file being processed, for logging. - - Returns: - dict: Enhanced row dictionary with exhibit lesser-of constraint applied. - """ - enhanced_row = original_row.copy() - - # Get original REIMB_TERM - original_reimb = original_row.get("REIMB_TERM", "") - - # Skip if exhibit_lesser_of is empty or N/A - if not exhibit_lesser_of or string_utils.is_empty(exhibit_lesser_of): - return enhanced_row - - # FIRST-ORDER DEDUPLICATION: Check if the exhibit constraint is already present (case-insensitive) - if exhibit_lesser_of.lower() in original_reimb.lower(): - logging.debug( - f"Exhibit constraint '{exhibit_lesser_of}' already present in REIMB_TERM: {original_reimb}" - ) - return enhanced_row # No need to modify if already present - - # SECOND-ORDER DEDUPLICATION: Use LLM to check if the exhibit constraint is redundant - if prompt_calls.is_exhibit_lesser_of_redundant( - original_reimb, exhibit_lesser_of, filename - ): - logging.debug( - f"LLM determined that exhibit constraint '{exhibit_lesser_of}' is semantically redundant with REIMB_TERM: {original_reimb}" - ) - return enhanced_row # No need to modify if redundant - - # Apply the exhibit lesser-of constraint - enhanced_reimb = f"lesser of ({original_reimb}) or ({exhibit_lesser_of})" - enhanced_row["REIMB_TERM"] = enhanced_reimb - - logging.debug( - f"Enhanced REIMB_TERM with exhibit lesser-of: {enhanced_row['REIMB_TERM']}" - ) - return enhanced_row - - -def apply_exhibit_lesser_of( - rows: list[dict], exhibit_lesser_of: str, filename: str -) -> list[dict]: - """Apply exhibit lesser-of constraint to rows that don't explicitly have a lesser-of carve-out. - - Args: - rows (list[dict]): The rows to which the constraint should be applied. - exhibit_lesser_of (str): The exhibit lesser-of statement to apply. - filename (str): The name of the file being processed, for logging. - - Returns: - list[dict]: The rows with the exhibit lesser-of constraint applied. - """ - enhanced_rows = [] - - for row in rows: - reimb_term = row.get("REIMB_TERM", "") - - # Check if row explicitly carves out from lesser-of - if has_explicit_lesser_of_carveout(reimb_term): - logging.debug( - f"Skipping exhibit lesser-of application for row with explicit carve-out: {row}" - ) - enhanced_rows.append(row) # Keep as-is - else: - # Apply exhibit lesser-of to rows without explicit carve-out - row = create_exhibit_lesser_of_row(row, exhibit_lesser_of, filename) - enhanced_rows.append(row) - - return enhanced_rows diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index ff1373f..1b131f5 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -1,6 +1,6 @@ import logging -from src.investment import dynamic_funcs, lesser_of_funcs, prompt_calls, aarete_derived, postprocessing_funcs +from src.investment import dynamic_funcs, prompt_calls, aarete_derived, postprocessing_funcs from src.utils import llm_utils, string_utils import src.prompts.prompt_templates as prompt_templates from constants.constants import Constants @@ -8,36 +8,10 @@ from src import config from src.prompts.fieldset import FieldSet -def get_exhibit_header(all_exhibit_headers: dict, exhibit_page: str) -> str: - """ - Retrieves the exhibit header for a given exhibit page. - Args: - all_exhibit_headers (dict): A dictionary mapping exhibit pages to their headers. - exhibit_page (str): The page number of the exhibit being processed. - Returns: - str: The exhibit header if found, otherwise "No Header Found". - """ - - if exhibit_page in all_exhibit_headers.keys(): - exhibit_header = all_exhibit_headers[exhibit_page] - elif ( - ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys() - ): # Take the parent header if it exists - exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])] - else: - exhibit_header = "No Header Found" - return exhibit_header - - def exhibit_level( exhibit_text: str, exhibit_header: str, exhibit_page: str, - exhibit_level_fields: FieldSet, - dynamic_primary_fields: FieldSet, - dynamic_code_fields: FieldSet, - dynamic_reimb_info_fields: FieldSet, - reimbursement_level_fields: FieldSet, constants: Constants, filename: str, ) -> tuple[dict, FieldSet]: @@ -47,10 +21,6 @@ def exhibit_level( Args: exhibit_text (str): The text data to process. exhibit_header (str): The header of the exhibit. - exhibit_level_fields (FieldSet): A FieldSet object containing exhibit-level field prompts. - dynamic_primary_fields (FieldSet): A FieldSet object containing dynamic primary field prompts. - dynamic_code_fields (FieldSet): A FieldSet object containing dynamic code field prompts. - dynamic_reimb_info_fields (FieldSet): A FieldSet object containing dynamic reimbursement info field prompts. reimbursement_level_fields (FieldSet): A FieldSet object containing reimbursement-level field prompts. constants (Constants): A Constants object filename (str): The name of the file @@ -58,112 +28,86 @@ def exhibit_level( dict: All Exhibit-Level answers FieldSet: Updated reimbursement_level_fields with any dynamically added fields """ + exhibit_level_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="exhibit_level" + ) + dynamic_primary_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="dynamic_primary" + ) + dynamic_reimb_info_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="dynamic_reimb_info" + ) + dynamic_code_fields = FieldSet( + config.FIELD_JSON_PATH, field_type="dynamic_code" + ) # True Exhibit Level - these apply to the entire Exhibit exhibit_level_answers = prompt_calls.prompt_exhibit_level( exhibit_text, exhibit_level_fields, constants, filename ) - exhibit_level_answers = prompt_calls.prompt_exhibit_lesser( - exhibit_text, exhibit_level_answers, filename - ) # Dynamic Primary - exhibit_level_answers, reimbursement_level_fields = dynamic_funcs.dynamic_primary( + exhibit_level_answers, dynamic_reimbursement_fields = dynamic_funcs.dynamic_primary( exhibit_text, - exhibit_header, exhibit_level_answers, constants, filename, - dynamic_primary_fields, - reimbursement_level_fields, + dynamic_primary_fields ) # Dynamic Codes - exhibit_level_answers, reimbursement_level_fields = dynamic_funcs.dynamic( + exhibit_level_answers, dynamic_reimbursement_fields = dynamic_funcs.dynamic( exhibit_text, exhibit_header, exhibit_level_answers, constants, filename, dynamic_code_fields, - reimbursement_level_fields, + dynamic_reimbursement_fields ) # Dynamic Reimb Info - exhibit_level_answers, reimbursement_level_fields = dynamic_funcs.dynamic( + exhibit_level_answers, dynamic_reimbursement_fields = dynamic_funcs.dynamic( exhibit_text, exhibit_header, exhibit_level_answers, constants, filename, dynamic_reimb_info_fields, - reimbursement_level_fields, + dynamic_reimbursement_fields ) # add exhibit identifiers exhibit_level_answers["EXHIBIT_PAGE"] = exhibit_page exhibit_level_answers["EXHIBIT_TITLE"] = exhibit_header - return exhibit_level_answers, reimbursement_level_fields + return exhibit_level_answers, dynamic_reimbursement_fields def reimbursement_level( - exhibit_text, - reimbursement_level_fields: FieldSet, - special_case_primary_fields: FieldSet, - exhibit_lesser_of, - seen_pairs, - exhibit_page, - filename: str, + page_text: str, constants: Constants, -): - """ - Processes reimbursement-level fields. - Currently, runs REIMBURSEMENT_LEVEL_PRIMARY prompt only. Future secondary prompts will go in this function - - Args: - exhibit_text (str): The text data to process. - filename (str): The name of the file - reimbursement_level_fields (FieldSet): A FieldSet object containing field prompts. - all_dataset (dict): # TODO: add docstring here - exhibit_page (str): The page number of the exhibit being processed (for a unique identifier) - seen_pairs (set): A set accumulator to track tuples of (base_page, SERVICE_TERM, REIMB_TERM) to avoid duplicate processing. - (i.e. if a exhibit 23.0 [base page 23] has subexhibits 23.0, 23.1, 23.2, etc., - then the pairs could be tracked per base page.) - exhibit_lesser_of (str): The exhibit lesser-of statement, if applicable. - - Returns: - dict: The parsed LLM response as a list of dictionaries with unique identifiers - """ - # Special Case Primary - dict[str, list[str]] - special_case_primary_answers = prompt_calls.prompt_special_case_primary( - special_case_primary_fields, exhibit_text, constants, filename - ) - + filename: str +) -> list[dict[str, str]]: + # Reimbursement Primary reimbursement_primary_answers = prompt_calls.prompt_reimbursement_primary( - reimbursement_level_fields, exhibit_text, constants, filename - ) - reimbursement_primary_answers = clean_reimbursement_primary( - reimbursement_primary_answers, - exhibit_lesser_of, - seen_pairs, - exhibit_page, - constants, - filename, + page_text, filename ) - return reimbursement_primary_answers, special_case_primary_answers + if reimbursement_primary_answers: + reimbursement_primary_answers = clean_reimbursement_primary( + reimbursement_primary_answers, constants, filename + ) + + return reimbursement_primary_answers def clean_reimbursement_primary( - reimbursement_primary_answers: list[dict], - exhibit_lesser_of: str, - seen_pairs: set, - exhibit_page: str, + reimbursement_primary_answers: list[dict[str, str]], constants: Constants, filename: str, -) -> list[dict]: +) -> list[dict[str, str]]: """ Cleans and processes reimbursement primary answers through several steps: 1. Split compound reimbursements @@ -172,87 +116,62 @@ def clean_reimbursement_primary( 4. Apply exhibit lesser-of statement if it exists Args: - reimbursement_primary_answers (list[dict]): List of dictionaries containing reimbursement details. - exhibit_lesser_of (str): The exhibit lesser-of statement, if applicable. - seen_pairs (set): A set accumulator to track tuples of (base_page, SERVICE_TERM, REIMB_TERM) to avoid duplicate processing. + reimbursement_primary_answers (list[str]): List of dictionaries containing reimbursement details. constants (Constants): A Constants object - exhibit_page (str): The page number of the exhibit being processed (for a unique identifier) filename (str): The name of the file being processed, for logging Returns: list[dict]: The cleaned and processed reimbursement primary answers. """ if reimbursement_primary_answers: # If any reimbursements found - # Step 1: Split compound reimbursements FIRST - split_answers = split_compound_reimbursements( - reimbursement_primary_answers, constants.COMPOUND_INDICATORS, filename + + # If needed, add Reimbursement Splitting here + + # If needed, add deduplication here + + # Filter out any non-reimbursement content from split answers + reimbursement_primary_answers = filter_services_without_reimbursements( + reimbursement_primary_answers, filename ) - # Step 2: Filter out any non-reimbursement content from split answers - filtered_answers = filter_services_without_reimbursements( - split_answers, filename - ) - - # Step 3: Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs - deduplicated_answers = deduplicate_with_accumulator( - filtered_answers, seen_pairs, exhibit_page, filename - ) - - # Step 4: Apply exhibit lesser-of statement if it exists - if not string_utils.is_empty(exhibit_lesser_of) and deduplicated_answers: - deduplicated_answers = lesser_of_funcs.apply_exhibit_lesser_of( - deduplicated_answers, exhibit_lesser_of, filename - ) - logging.debug( - f"Applied exhibit lesser-of statement to {len(deduplicated_answers)} reimbursement pairs" - ) - - return deduplicated_answers + return reimbursement_primary_answers else: return [] def breakout( - reimbursement_primary_answers: list[dict], - special_case_primary_answers: dict[str, list[str]], - special_case_primary_fields: FieldSet, + reimbursement_level_answers: list[dict[str, str]], + special_case_answers: list[dict[str, str]], filename: str, - constants: Constants, -) -> tuple[list[dict], dict[str, list]]: + constants: Constants +) -> tuple[list[dict[str, str]], list[dict[str, str]]]: """ Performs breakout processing on reimbursement primary answers and special case primary answers. Args: reimbursement_primary_answers (list[dict]): List of dictionaries containing reimbursement details. - special_case_primary_answers (dict[str, list[str]]): Dictionary containing special case primary answers. - special_case_primary_fields (FieldSet): A FieldSet object containing field prompts for special case primary. filename (str): The name of the file constants (Constants): A Constants object Returns: - tuple: A tuple containing: - - list[dict]: The processed reimbursement primary answers with breakout details. - - dict[str, list]: The processed special case primary answers with breakout details. + - list[dict]: The processed reimbursement primary answers with breakout details. """ - reimbursement_breakout_answers = methodology_breakout( - reimbursement_primary_answers, constants, filename + + reimbursement_level_answers = methodology_breakout( + reimbursement_level_answers, constants, filename ) # Including base methodology breakout, grouper, and fee schedule - # Standard Methodology Breakout - reimbursement_breakout_answers = carveout_check( - reimbursement_breakout_answers, constants, filename - ) # Special Case Breakout special_case_answers = special_case_breakout( - special_case_primary_answers, special_case_primary_fields, filename + special_case_answers, filename ) - return reimbursement_breakout_answers, special_case_answers + return reimbursement_level_answers, special_case_answers -def carveout_check( +def carveout_and_special_case( reimbursement_breakout_answers: list[dict], constants: Constants, filename: str -) -> list[dict]: +) -> tuple[list[dict], list[dict]]: """ - Performs carveout check on reimbursement primary answers. + Performs carveout check on reimbursement primary answers. Determines if the answers are Carveout or Special Case Args: reimbursement_breakout_answers (list[dict]): List of dictionaries containing reimbursement details. @@ -261,31 +180,50 @@ def carveout_check( Returns: list[dict]: The updated reimbursement breakout answers with carveout details. """ - # additional fields besides service and reimb term to pass to LLM for context - additional_fields_for_carveout = ['AARETE_DERIVED_REIMB_METHOD', 'UNIT_OF_MEASURE', 'REIMB_FEE_RATE' - , 'REIMB_PCT_RATE', 'REIMB_CONVERSION_FACTOR', 'FEE_SCHEDULE'] + carveout_definitions = constants.VALID_CARVEOUTS + special_case_definitions = FieldSet(config.FIELD_JSON_PATH, field_type="special_case_check").get_prompt_dict(constants) + + reimbursement_level_answers, special_case_answers = [], [] + for answer_dict in reimbursement_breakout_answers: - service_term = answer_dict.get("SERVICE_TERM") - reimb_term = answer_dict.get("REIMB_TERM") - base_reimb_answer_dict = {key: answer_dict[key] for key in additional_fields_for_carveout if key in answer_dict} - - carveout_answer = prompt_calls.prompt_carveout_check( - service_term, reimb_term, str(base_reimb_answer_dict), constants, filename + service_term, reimb_term = answer_dict.get("SERVICE_TERM"), answer_dict.get("REIMB_TERM") + + reimbursement_categorization = prompt_calls.prompt_carveout_check( + service_term, reimb_term, carveout_definitions, special_case_definitions, filename ) - answer_dict["CARVEOUT_CD"] = carveout_answer + + if reimbursement_categorization in carveout_definitions.keys(): + answer_dict["CARVEOUT_CD"] = reimbursement_categorization + # Standard N/A OR Base response gets CARVEOUT_IND=N + if reimbursement_categorization in ["N/A", "BASE_COVERED_SERVICES", "DEFAULT_TERM"]: + answer_dict["CARVEOUT_IND"] = "N" + else: + answer_dict["CARVEOUT_IND"] = "Y" + # Determine DEFAULT_IND based on specific carveout_answer values + answer_dict["DEFAULT_IND"] = ( + "Y" if reimbursement_categorization in ["DEFAULT_TERM", "UNLISTED_CODE"] else "N" + ) - # Standard N/A OR Base response gets CARVEOUT_IND=N - if carveout_answer in ["N/A", "BASE_COVERED_SERVICES", "DEFAULT_TERM"]: - answer_dict["CARVEOUT_IND"] = "N" - else: - answer_dict["CARVEOUT_IND"] = "Y" + # Carveout breakouts + if reimbursement_categorization == "TRIGGER_CAP": + breakout_answer_dict = prompt_calls.prompt_special_case_breakout( + prompt_templates.TRIGGER_CAP_BREAKOUT, reimb_term, filename + ) + answer_dict.update(breakout_answer_dict) + if reimbursement_categorization == "ADDITION": + breakout_answer_dict = prompt_calls.prompt_special_case_breakout( + prompt_templates.ADDITION_BREAKOUT, reimb_term, filename + ) + answer_dict.update(breakout_answer_dict) - # Determine DEFAULT_IND based on specific carveout_answer values - answer_dict["DEFAULT_IND"] = ( - "Y" if carveout_answer in ["DEFAULT_TERM", "UNLISTED_CODE"] else "N" - ) + reimbursement_level_answers.append(answer_dict) + elif reimbursement_categorization in special_case_definitions.keys(): + answer_dict[reimbursement_categorization] = reimb_term + answer_dict.pop("REIMB_TERM", None) + answer_dict.pop("SERVICE_TERM", None) + special_case_answers.append(answer_dict) - return reimbursement_breakout_answers + return reimbursement_level_answers, special_case_answers def methodology_breakout( @@ -332,8 +270,7 @@ def methodology_breakout_single_row( list[dict]: Updated list with methodology breakout details and fee schedule information. """ - service = answer_dict.get("SERVICE_TERM") - reimbursement_method = answer_dict.get("REIMB_TERM") + reimb_term, service_term = answer_dict.get("REIMB_TERM"), answer_dict.get("SERVICE_TERM") methodology_breakout_questions = FieldSet( config.FIELD_JSON_PATH, field_type="methodology_breakout" ) @@ -345,10 +282,9 @@ def methodology_breakout_single_row( ) # Methodology Breakout: AARETE_DERIVED_REIMB_METHOD, REIMB_FEE_RATE, REIMB_PCT_RATE, LESSER_OF_IND, GREATER_OF_IND, AARETE_DERIVED_PROV_TYPE - if reimbursement_method: + if reimb_term: methodology_breakout_answers = prompt_calls.prompt_methodology_breakout( - service, - reimbursement_method, + service_term, reimb_term, methodology_breakout_questions, constants, filename, @@ -366,8 +302,8 @@ def methodology_breakout_single_row( for methodology_breakout_dict in methodology_breakout_answers: secondary_breakout_dict = methodology_breakout_secondary( methodology_breakout_dict, - service, - reimbursement_method, + service_term, + reimb_term, fs_breakout_questions, grouper_breakout_questions, constants, @@ -382,8 +318,8 @@ def methodology_breakout_single_row( def methodology_breakout_secondary( methodology_breakout_dict: dict, - service: str, - reimbursement_method: str, + service_term: str, + reimb_term: str, fs_breakout_questions: FieldSet, grouper_breakout_questions: FieldSet, constants: Constants, @@ -405,8 +341,8 @@ def methodology_breakout_secondary( methodology_breakout_dict (dict): Results from primary methodology breakout, including the AARETE_DERIVED_REIMB_METHOD field which will drive the branching logic. - service (str): The service term associated with the reimbursement. - reimbursement_method (str): The original reimbursement method text extracted + service_term (str): The service term associated with the reimbursement. + reimb_term (str): The original reimbursement method text extracted (pre-AARETE_DERIVED) fs_breakout_questions (FieldSet): Fee Schedule-specific prompts and field definitions. grouper_breakout_questions (FieldSet): Grouper-specific prompts and field definitions. @@ -428,7 +364,7 @@ def methodology_breakout_secondary( # Prompt for FS breakout, fill grouper with N/A fs_breakout_dict = prompt_calls.prompt_fee_schedule_breakout( methodology_breakout_dict, - reimbursement_method, + reimb_term, fs_breakout_questions, constants, filename, @@ -443,8 +379,8 @@ def methodology_breakout_secondary( # Grouper Breakout elif AARETE_DERIVED_REIMB_METHOD == "Grouper": grouper_breakout_dict = prompt_calls.prompt_grouper_breakout( - service, - reimbursement_method, + service_term, + reimb_term, grouper_breakout_questions, constants, filename, @@ -463,10 +399,9 @@ def methodology_breakout_secondary( def special_case_breakout( - special_case_primary_answers: dict, - special_case_primary_fields: FieldSet, + special_case_answers: list[dict], filename: str, -) -> dict[str, list]: +) -> list[dict]: """Applies specialized breakout templates to special case primary answers. For each special case field (outliers, discounts, premiums, etc.) that has extracted @@ -488,89 +423,23 @@ def special_case_breakout( dict[str, list]: A dictionary containing the processed breakout answers for each special case field. """ - - final_answer_dict = {} - # if not dict and empty special_case_primary_answers, return empty dict - if not isinstance(special_case_primary_answers, dict) and string_utils.is_empty(special_case_primary_answers): - return final_answer_dict - for term_field_name, term_answers in special_case_primary_answers.items(): - - if not term_answers: - continue - - term_field = special_case_primary_fields.get_field(term_field_name) + special_case_fields = FieldSet(config.FIELD_JSON_PATH, field_type="special_case_check") + for answer_dict in special_case_answers: + term_field_name = [field_name for field_name in answer_dict.keys() if "_TERM" in field_name and field_name not in ["SERVICE_TERM","REIMB_TERM",]][0] + term_answer = answer_dict[term_field_name] + term_field = special_case_fields.get_field(term_field_name) breakout_template = term_field.get_breakout_template() - - term_field_answer_list = [] - - for term_answer in term_answers: - breakout_answer_dict = prompt_calls.prompt_special_case_breakout( - breakout_template, term_answer, filename - ) - breakout_answer_dict[term_field_name] = term_answer - term_field_answer_list.append(breakout_answer_dict) - - final_answer_dict[term_field_name] = term_field_answer_list - return final_answer_dict - - -def deduplicate_with_accumulator( - answers: list[dict], seen_pairs: set, exhibit_page: str, filename: str -) -> list[dict]: - """Remove service-reimbursement pairs that have been seen in previous pages. - Includes a base page scope to allow only subexhibits to share pairs. This is because - (a) subexhibits often repeat information due to our table splitting logic, and - (b) different exhibits may share the same (service, reimbursement) pairs, - but in meaningfully different contexts (e.g. MEDICARE FEE SCHEDULE on - exhibit page 23 vs. MARKETPLACE FEE SCHEDULE on exhibit 45). In this case - we do NOT want to deduplicate across exhibits. - - Args: - answers (list[dict]): List of dictionaries with SERVICE_TERM and REIMB_TERM keys. - seen_pairs (set): A set of previously seen (base_page, SERVICE_TERM, REIMB_TERM) pairs. - e.g. {("23", "Routine Vision", "Provider's Billed Charges"), ...} - This is the "accumulator" that persists across multiple calls. It is - modified in-place to add new pairs as they are encountered, and as such - is not returned by the function. - exhibit_page (str): The exhibit page number, for logging. - filename (str): The name of the file being processed, for logging - - Returns: - list[dict]: Deduplicated list of dictionaries. - """ - # Extract base page - base_page = exhibit_page.split(".")[0] # e.g. "23.0" -> "23", "23.0.0" -> "23" - - deduplicated = [] - duplicates_found = 0 - - for answer_dict in answers: - service_term = answer_dict.get("SERVICE_TERM", "").strip() - reimb_term = answer_dict.get("REIMB_TERM", "").strip() - - # Create a tuple key scoped to the base page - pair_key = (base_page, service_term, reimb_term) - - if pair_key not in seen_pairs: - seen_pairs.add(pair_key) # Update accumulator - deduplicated.append(answer_dict) - else: - duplicates_found += 1 - logging.debug( - f"Within-base-page duplicate found in {filename}, base page {base_page}, exhibit {exhibit_page}: {service_term} - {reimb_term}" - ) - - if duplicates_found > 0: - logging.debug( - f"Removed {duplicates_found} duplicate pairs from exhibit {exhibit_page} (base page {base_page}) in {filename}" + breakout_answer_dict = prompt_calls.prompt_special_case_breakout( + breakout_template, term_answer, filename ) - - return deduplicated + answer_dict.update(breakout_answer_dict) + + return special_case_answers def filter_services_without_reimbursements( - answers: list[dict], filename: str -) -> list[dict]: + reimbursement_primary_answers: list[dict[str, str]], filename: str +) -> list[dict[str, str]]: """ Filters out services that lack actual reimbursement methodologies. @@ -598,13 +467,13 @@ def filter_services_without_reimbursements( valid_reimbursements = [] - for answer_dict in answers: + for answer_dict in reimbursement_primary_answers: if prompt_calls.validate_reimbursements_for_llm(answer_dict, filename): valid_reimbursements.append(answer_dict) else: logging.debug(f"LLM filtered out: {answer_dict}") - filtered_count = len(answers) - len(valid_reimbursements) + filtered_count = len(reimbursement_primary_answers) - len(valid_reimbursements) if filtered_count > 0: logging.debug( f"Filtered {filtered_count} services without clear reimbursement methodologies in {filename}." @@ -613,77 +482,6 @@ def filter_services_without_reimbursements( return valid_reimbursements -def split_compound_reimbursements( - answers: list[dict], COMPOUND_INDICATORS, filename: str -) -> list[dict]: - """ - Splits reimbursement terms that contain multiple distinct methodologies. - - The process involves two main steps: - 1. A substring search is performed to identify potential compound reimbursements - using a predefined list of indicators (e.g. "otherwise shall be", "except for", - "for services where there is no", etc.). - 2. For entries flagged as potential compounds, an LLM prompt is invoked to - attempt to split the entry into multiple distinct reimbursement terms. - If the LLM successfully splits the entry into multiple parts, these are added - to the results. If not, the original entry is retained. - - Example: - INPUT: - SERVICE: "Drugs" - REIMBURSEMENT: "lesser of Provider's billed charges or: (i) Fee schedule rate for standard drugs, (ii) WAC + 5% for specialty drugs, not to exceed $50" - - OUTPUT: - [ - {{ - "SERVICE_TERM": "standard drugs", - "REIMB_TERM": "lesser of Provider's billed charges or Fee schedule rate" - }}, - {{ - "SERVICE_TERM": "specialty drugs", - "REIMB_TERM": "lesser of Provider's billed charges or WAC + 5%, not to exceed $50" - }} - ] - - Args: - answers (list[dict]): List of dictionaries containing reimbursement details. - filename (str): The name of the file being processed. - - Returns: - list[dict]: List of dictionaries with split reimbursement terms. - """ - - split_answers = [] - - for answer_dict in answers: - reimb_term = answer_dict.get("REIMB_TERM", "").lower().strip() - - # Check if this looks like a compound reimbursement by using a substring search - has_compound = any(indicator.lower() in reimb_term.lower() for indicator in COMPOUND_INDICATORS) - - if has_compound: - # Use LLM to split the compound reimbursement - split_result = prompt_calls.split_compound_reimbursement_llm( - answer_dict, filename - ) - - if split_result and len(split_result) > 1: - logging.debug( - f"Split compound reimbursement in {filename}: " - f"{answer_dict['SERVICE_TERM']} - {answer_dict['REIMB_TERM']}" - f" into {len(split_result)} parts." - ) - - logging.debug(f"Split results: {split_result}") - - split_answers.extend(split_result) - else: # If LLM didn't split, keep the original - split_answers.append(answer_dict) - else: # If not compound, keep the original - split_answers.append(answer_dict) - - return split_answers - def split_reimb_dates(one_to_n_results: list, filename: str) -> list: """Splits REIMB_DATES field into separate effective and termination date fields. @@ -842,3 +640,18 @@ def one_to_n_cleaning(all_exhibit_rows: list[dict], exhibit_text: str, constants return all_exhibit_rows +def lesser_of_distribution(reimbursement_level_answers: list[dict[str, str]], exhibit_text_simplified: str, page_num: str, constants: Constants, filename: str) -> list[dict[str, str]]: + final_reimbursement_level_answers = [] + for answer_dict in reimbursement_level_answers: + service_term, original_reimb_term = answer_dict['SERVICE_TERM'], answer_dict["REIMB_TERM"] + if "less" in original_reimb_term.lower(): + is_overarching_lesser = prompt_calls.prompt_lesser_of_check(service_term, original_reimb_term, filename) + if not is_overarching_lesser: + final_reimbursement_level_answers.append(answer_dict) + else: + new_reimb_term = prompt_calls.prompt_lesser_of_distribution(answer_dict['SERVICE_TERM'], original_reimb_term, exhibit_text_simplified, page_num, filename) + answer_dict["REIMB_TERM"] = new_reimb_term + final_reimbursement_level_answers.append(answer_dict) + return final_reimbursement_level_answers + + diff --git a/fieldExtraction/src/investment/one_to_one_funcs.py b/fieldExtraction/src/investment/one_to_one_funcs.py index 18e3196..4a29fb7 100644 --- a/fieldExtraction/src/investment/one_to_one_funcs.py +++ b/fieldExtraction/src/investment/one_to_one_funcs.py @@ -1,6 +1,7 @@ import logging import os import re +import json import pandas as pd import src.investment.postprocessing_funcs as postprocessing_funcs @@ -16,7 +17,7 @@ from src.investment.vision_funcs import get_image_array_based_answer from src.prompts.fieldset import Field, FieldSet -def extract_global_lesser_of(contract_text: str, filename: str) -> dict: +def extract_global_lesser_of(text_dict: str, first_reimbursement_page: str, filename: str) -> dict: """Extract global "lesser of" statements that apply contract-wide. Args: @@ -30,12 +31,13 @@ def extract_global_lesser_of(contract_text: str, filename: str) -> dict: global_lesser_of_field = Field.load_from_file( file_path=config.FIELD_JSON_PATH, field_name="GLOBAL_LESSER_OF_STATEMENT" ) + + context = "\n".join([page_text for page_num, page_text in text_dict.items() if int(page_num) < int(first_reimbursement_page)]) + # Process using existing full context logic prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE( - context=contract_text[ - 0 : min(config.MAX_CONTEXT_LENGTH, len(contract_text) - 1) - ], - question=global_lesser_of_field.prompt, + context, + field_prompt=global_lesser_of_field.prompt, ) logging.debug( @@ -180,6 +182,8 @@ def check_and_update_effective_date(answers_dict, text_dict, filename): def run_full_context_fields( one_to_one_fields: FieldSet, contract_text: str, + text_dict: dict[str, str], + first_reimbursement_page: str, constants: Constants, filename: str, ) -> dict[str, str]: @@ -246,6 +250,9 @@ def run_full_context_fields( contract_text, full_context_fields, constants, filename ) + if "PROV_GROUP_NAME_FULL" in full_context_answers_dict: + full_context_answers_dict["PROV_INFO_JSON"] = json.dumps([{"TIN": "UNKNOWN", "NPI": "UNKNOWN", "NAME": full_context_answers_dict["PROV_GROUP_NAME_FULL"], "IS_GROUP": "Y"}]) + full_context_answers_dict["PROV_INFO_JSON_FORMATTED"] = json.dumps([{"TIN": "UNKNOWN", "NPI": "UNKNOWN", "NAME": full_context_answers_dict["PROV_GROUP_NAME_FULL"], "IS_GROUP": "Y"}]) # Normalize PAYER_STATE to two-letter abbreviation if "PAYER_STATE" in full_context_answers_dict: @@ -267,7 +274,7 @@ def run_full_context_fields( for field in separate_fields.fields: if field.field_name == "GLOBAL_LESSER_OF_STATEMENT": # Add global lesser of extraction - global_lesser_of_results = extract_global_lesser_of(contract_text, filename) + global_lesser_of_results = extract_global_lesser_of(text_dict, first_reimbursement_page, filename) full_context_answers_dict.update(global_lesser_of_results) return full_context_answers_dict diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index d6969f4..26e59e5 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -1,6 +1,5 @@ import logging -import src.investment.table_funcs as table_funcs from src import config from src.investment import preprocessing_funcs @@ -68,48 +67,7 @@ def one_to_n_exhibit_chunking( exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit( text_dict, exhibit_pages ) - exhibit_dict = preprocessing_funcs.get_exhibit_dict( - text_dict, exhibit_chunk_mapping - ) - return exhibit_dict, all_exhibit_headers - - -def clean_tables( - text_dict: dict[str, str], EXHIBIT_HEADER_MARKERS: list, filename: str -) -> dict[str, str]: - """ - Process tables with simplified logic: combine continuations, then split by row count. - - Args: - text_dict (dict[str, str]): A dictionary where keys are page numbers and values are the text on those pages. - filename (str): The name of the file being processed. - - Returns: - dict[str, str]: A dictionary with the processed text for each page, with tables split when necessary. - """ - # 1. Extract table info - table_dict = table_funcs.get_table_info(text_dict, EXHIBIT_HEADER_MARKERS) - logging.debug(f"Found tables on pages: {list(table_dict.keys())}") - - # 2. Combine continuous tables (simple version) - text_dict_combined = table_funcs.combine_continuous_tables( - text_dict=text_dict, - table_dict=table_dict, - ) - logging.debug(f"After combination, pages: {list(text_dict_combined.keys())}") - - # 3. Split tables by row count only - text_dict_final = table_funcs.split_tables_by_rows( - text_dict=text_dict_combined, - EXHIBIT_HEADER_MARKERS=EXHIBIT_HEADER_MARKERS, - row_limit=config.TABLE_ROW_LIMIT, - ) - - logging.debug( - f"Final pages after simple table processing: {list(text_dict_final.keys())}" - ) - - return text_dict_final + return exhibit_chunk_mapping, all_exhibit_headers def find_headers_and_footers(text_dict: dict[str, str]) -> tuple[dict, str, str]: diff --git a/fieldExtraction/src/investment/preprocessing_funcs.py b/fieldExtraction/src/investment/preprocessing_funcs.py index afb4f7f..167d9fe 100644 --- a/fieldExtraction/src/investment/preprocessing_funcs.py +++ b/fieldExtraction/src/investment/preprocessing_funcs.py @@ -2,7 +2,7 @@ import logging import re import src.utils.string_utils as string_utils -from src.investment import table_funcs, prompt_calls +from src.investment import prompt_calls def remove_page_indicators(contract_text: str) -> str: """Clean textract output by removing page number indicators in the form of "Page X of Y" @@ -98,7 +98,7 @@ def filter_quick_review(text_dict): - The first dictionary contains pages that do not have the specified keywords. - The second dictionary includes pages that contain any of the specified keywords. """ - keywords = ["QUICK REVIEW", "TOP SHEET", "COVER SHEET", "PROVIDER SUBMISSION FORM"] + keywords = ["QUICK REVIEW", "TOP SHEET", "COVER SHEET", "PROVIDER SUBMISSION FORM", "CONTRACT REVIEW FORM"] regular_pages = {} quick_review_pages = {} @@ -188,25 +188,12 @@ def link_exhibit_pages( final_exhibit_headers[page_num] = page_header first_exhibit = False else: - # Check if this page has a substantial table - has_table = has_substantial_table(text_dict.get(page_num, "")) - # If literally identical, skip prompt (unless it has a table) - if ( - page_header.upper().strip() == previous_header.upper().strip() - and not has_table - ): + if page_header.upper().strip() == previous_header.upper().strip(): exhibit_is_different = "N" elif string_utils.is_empty(previous_header): exhibit_is_different = "Y" previous_header = page_header - elif has_table: - # Always treat pages with substantial tables as different exhibits - logging.debug( - f"Page {page_num} has substantial table, treating as separate exhibit" - ) - exhibit_is_different = "Y" - previous_header = page_header else: exhibit_is_different = prompt_calls.prompt_exhibit_linkage(page_header, previous_header, filename) previous_header = page_header @@ -219,35 +206,6 @@ def link_exhibit_pages( return final_exhibit_pages, final_exhibit_headers -def has_substantial_table(page_text: str) -> bool: - """Check if a page contains a substantial table worth preserving as a separate chunk. - - Args: - page_text (str): The text content of the page to check. - - Returns: - bool: True if the page contains a substantial table, False otherwise. - """ - if not page_text or table_funcs.START_MARKER not in page_text: - return False - - # Count table rows to determine if it's substantial - table_start = page_text.find(table_funcs.START_MARKER) - table_end = page_text.find(table_funcs.END_MARKER) - - if table_start == -1 or table_end == -1: - return False - - table_content = page_text[table_start:table_end] - - # Count rows (each row is a list in the table representation) - row_count = table_content.count("'], ['") # Rough count of table rows - - return ( - row_count >= 5 - ) # Right now, using "5 rows" as "substantial". Alternative: use config.TABLE_ROW_LIMIT - - def chunk_by_exhibit(text_dict: dict, exhibit_pages: list) -> dict: """ Organizes pages into groups based on their association with specific exhibits. @@ -263,141 +221,99 @@ def chunk_by_exhibit(text_dict: dict, exhibit_pages: list) -> dict: exhibit. Returns: - dict: A dictionary mapping each page number in `text_dict` to its corresponding - exhibit identifier. The exhibit identifier is the page number of the first - page in that exhibit as listed in `exhibit_pages`. If there are pages before - the first `exhibit_page`, they are grouped under the exhibit identifier "0". + dict: A dictionary where keys are exhibit identifiers (page numbers from exhibit_pages) + and values are lists of page numbers that belong to that exhibit. + Example: {'1': ['1', '2'], '3': ['3', '4', '5']} """ if len(exhibit_pages) == 0: - return {key: key for key in text_dict.keys()} + return {key: [key] for key in text_dict.keys()} exhibit_chunk_mapping = {} - current_exhibit = "0" + current_exhibit = list(text_dict.keys())[0] + for page_num in text_dict.keys(): if page_num in exhibit_pages: current_exhibit = page_num - exhibit_chunk_mapping[page_num] = current_exhibit - else: - exhibit_chunk_mapping[page_num] = current_exhibit + # Initialize the list for this exhibit if not already present + if current_exhibit not in exhibit_chunk_mapping: + exhibit_chunk_mapping[current_exhibit] = [] + + # Add the page to the current exhibit's list + if current_exhibit not in exhibit_chunk_mapping: + exhibit_chunk_mapping[current_exhibit] = [] + exhibit_chunk_mapping[current_exhibit].append(page_num) + return exhibit_chunk_mapping - -def get_exhibit_dict(text_dict: dict, exhibit_chunk_mapping: dict) -> dict: +def simplify_exhibit(text_dict: dict[str, str], exhibit_page_nums: list[str], current_page_num: str) -> str: """ - Creates a dictionary of exhibit texts, concatenating pages with the same exhibit number, - but separating decimal pages into their own chunks. - + Simplify exhibit text by replacing table content with 'Table X' placeholders on all pages + except the current page being processed. + Args: - text_dict (dict): Dictionary mapping page numbers to page text. - exhibit_chunk_mapping (dict): Dictionary mapping page numbers to exhibit numbers. - + text_dict: Dictionary mapping page numbers to page text + exhibit_page_nums: List of page numbers that belong to this exhibit + current_page_num: The current page being processed (will NOT be simplified) + Returns: - dict: Dictionary mapping exhibit chunk keys to combined text. Format for keys: - "{exhibit}.{index}" - NOTE: If exhibit identifiers contain decimals (e.g., "23.0") output keys will - follow the same pattern (resulting in "23.0.0", "23.0.1", etc.) - - Guidelines: - - Every exhibit chunk contains at most 1 table page in it (i.e. one page with a decimal) - - Tables are inserted in chronological order within the document flow. - - Multiple decimal pages are separated into individual chunks - - Each chunk gets a new key: "{exhibit}.{index}" (e.g. "2.0", "2.1", "2.2", etc.) - - Algorithm: - For each exhibit: - 1. Identify base pages (no decimals) and table pages (with decimals). - 2. For each decimal page, create a chunk containing: - - All base pages that came before the decimal page (in document order) - - The specific decimal page - - All base pages that came after the decimal page (in document order) - - Example 1: - Input mapping {'1.0': '1.0', '2': '2', '3.0': '2', '3.1': '2', '3.2': '2', '4.0': '2'} - - '1.0': text from '1.0' (single page exhibit) - - '2.0': text from '2' + text from '3.0' (base page 2, then table 3.0) - - '2.1': text from '2' + text from '3.1' (base page 2, then table 3.1) - - '2.2': text from '2' + text from '3.2' (base page 2, then table 3.2) - - '2.3': text from '2' + text from '4.0' (base page 2, then table 4.0) - - Example 2: - Input mapping {'1.0': '1.0', '2': '2', '3.0': '2', '3.1': '2', '3.2': '2', '4': '2', '5' : '2'} - - '1.0': text from '1.0' - - '2.0': text from '2' + text from '3.0' + text from '4' + text from '5' - - '2.1': text from '2' + text from '3.1' + text from '4' + text from '5' - - '2.2': text from '2' + text from '3.2' + text from '4' + text from '5' - - '2.3': text from '2' + text from '4' + text from '5' - - Note: Tables 3.0, 3.1, and 3.2 appear between pages 2 and 4 in document flow. - - Other cases: - - No decimal pages: creates single chunk for the exhibit. - - Single decimal page exhibit: uses original page as the chunk key. + str: The exhibit text with tables simplified on all pages except current_page_num """ - exhibit_dict = {} + def replace_tables_with_placeholder(page_text: str, starting_table_num: int) -> tuple[str, int]: + """Replace table blocks with 'Table X' labels and return next table number.""" + if not page_text: + return page_text, starting_table_num - # Group pages by exhibit - exhibit_pages = {} - for page, exhibit in exhibit_chunk_mapping.items(): - if exhibit.split(".")[0] == "0": - continue - if exhibit not in exhibit_pages: - exhibit_pages[exhibit] = [] - exhibit_pages[exhibit].append(page) + start_marker = "-------Table Start--------" + end_marker = "-------Table End--------" + result = [] + pos = 0 + table_num = starting_table_num + + while True: + start_idx = page_text.find(start_marker, pos) + if start_idx == -1: + # No more tables, append remaining text + result.append(page_text[pos:]) + break + + # Append text before table + result.append(page_text[pos:start_idx]) + + # Find end of table + end_idx = page_text.find(end_marker, start_idx + len(start_marker)) + + # Replace table with placeholder + result.append(f"Table {table_num}") + table_num += 1 + + # Move position past the table end marker + if end_idx != -1: + pos = end_idx + len(end_marker) + else: + # No end marker found, stop processing + break + + return "".join(result), table_num - # Process each exhibit - for exhibit in exhibit_pages: - pages = sorted(exhibit_pages[exhibit], key=string_utils.page_key_sort) - - base_pages = [p for p in pages if "." not in p] - decimal_pages = [p for p in pages if "." in p] - - # Special case: single decimal page exhibit - if len(decimal_pages) == 1 and not base_pages: - exhibit_dict[decimal_pages[0]] = text_dict[decimal_pages[0]] - continue - - # Case: No decimal pages - create single chunk with .0 suffix - if not decimal_pages: - exhibit_dict[f"{exhibit}.0"] = "\n".join(text_dict[p] for p in pages) - continue - - # Case: With decimal pages, create chunks for each decimal page - chunk_index = 0 - for decimal_page in decimal_pages: - chunk_key = f"{exhibit}.{chunk_index}" - - # Get decimal page base number for chronological positioning - decimal_base = int(decimal_page.split(".")[0]) - - # Build chunk with pages in chronological order - chunk_parts = [] - - # Add base pages before decimal page - for base_page in base_pages: - if int(base_page) <= decimal_base: - chunk_parts.append(text_dict[base_page]) - - # Add the decimal page itself - chunk_parts.append(text_dict[decimal_page]) - - # Add base pages after decimal page - for base_page in base_pages: - if int(base_page) > decimal_base: - chunk_parts.append(text_dict[base_page]) - - exhibit_dict[chunk_key] = "\n".join(chunk_parts) - chunk_index += 1 - - # Add final chunk with just base pages (if there are base pages after all decimal pages) - if base_pages and decimal_pages: - # Check if there are base pages that come after all decimal pages - max_decimal_base = max(int(dp.split(".")[0]) for dp in decimal_pages) - remaining_base_pages = [ - bp for bp in base_pages if int(bp) > max_decimal_base - ] - - if remaining_base_pages: - chunk_key = f"{exhibit}.{chunk_index}" - # Include ALL base pages in the final chunk - exhibit_dict[chunk_key] = "\n".join(text_dict[bp] for bp in base_pages) - return exhibit_dict + # Process each page in the exhibit + simplified_exhibit_pages = [] + table_counter = 1 + + for page_num in exhibit_page_nums: + page_text = text_dict.get(page_num, "") + + if page_num == current_page_num: + # Don't simplify the current page - keep it as-is + simplified_exhibit_pages.append(page_text) + # But still need to count tables to keep numbering consistent + if "-------Table Start--------" in page_text: + _, table_counter = replace_tables_with_placeholder(page_text, table_counter) + else: + # Simplify other pages by replacing tables with placeholders + if "-------Table Start--------" in page_text: + simplified_page, table_counter = replace_tables_with_placeholder(page_text, table_counter) + simplified_exhibit_pages.append(simplified_page) + else: + simplified_exhibit_pages.append(page_text) + + return "\n".join(simplified_exhibit_pages) \ No newline at end of file diff --git a/fieldExtraction/src/investment/prompt_calls.py b/fieldExtraction/src/investment/prompt_calls.py index 20baa13..1330ac3 100644 --- a/fieldExtraction/src/investment/prompt_calls.py +++ b/fieldExtraction/src/investment/prompt_calls.py @@ -75,14 +75,11 @@ def prompt_dynamic_primary( def prompt_reimbursement_primary( - reimbursement_level_fields: FieldSet, - exhibit_text: str, - constants: Constants, + page_text: str, filename: str, -) -> list[dict]: +) -> list[dict[str, str]]: - field_prompts = reimbursement_level_fields.print_prompt_dict(constants) - instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY(exhibit_text, field_prompts) + instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY(page_text) logging.debug( f"""Running reimbursement primary prompt for {filename} with prompt: {prompt}""" @@ -111,53 +108,15 @@ def prompt_reimbursement_primary( return llm_answer_final -def prompt_special_case_primary( - special_case_primary_fields: FieldSet, - exhibit_text: str, - constants: Constants, - filename: str, -): - field_prompts = special_case_primary_fields.print_prompt_dict(constants) - instruction, prompt = prompt_templates.SPECIAL_CASE_PRIMARY(exhibit_text, field_prompts) - - logging.debug( - f"""Running Special Case Primary prompt for {filename} with prompt: {prompt}""" - ) - llm_answer_raw = llm_utils.invoke_claude( - prompt, - "sonnet_latest", - filename, - max_tokens=25000, - cache=True, - instruction=instruction, - usage_label="SPECIAL_CASE_PRIMARY", - ) - logging.debug(f"""LLM raw output for {filename}: {llm_answer_raw}""") - - # Check for the special "no results" case - if "NO_REIMBURSEMENT_TERMS_FOUND" in llm_answer_raw: - return { - field: [] for field in special_case_primary_fields.list_fields() - } # Return empty list if no reimbursement terms found - - try: - llm_answer_final = string_utils.universal_json_load(llm_answer_raw) - except ValueError as e: - logging.error(f"Error parsing LLM response: {e}") - logging.error(f"Raw LLM output: {llm_answer_raw}") - raise - return llm_answer_final - - def prompt_methodology_breakout( - service: str, - reimbursement_method: str, + service_term: str, + reimb_term: str, methodology_breakout_questions: FieldSet, constants: Constants, filename: str, ): instruction, prompt = prompt_templates.METHODOLOGY_BREAKOUT( - [service, reimbursement_method], + service_term, reimb_term, methodology_breakout_questions.print_prompt_dict(constants), ) logging.debug(f"Methodology Breakout Prompt for {filename}: {prompt}") @@ -172,6 +131,7 @@ def prompt_methodology_breakout( try: methodology_breakout_answers = string_utils.universal_json_load(llm_response) except ValueError as e: + logging.error(f"Error parsing LLM response: {e}") methodology_breakout_answers = [] return methodology_breakout_answers @@ -230,13 +190,13 @@ def prompt_grouper_breakout( def prompt_carveout_check( - service_cd: str, reimb_method: str, base_reimb_answer: str, constants: Constants, filename: str + service_term: str, reimb_term: str, carveout_definitions, special_case_definitions, filename: str ): carveout_prompt = prompt_templates.CARVEOUT_CHECK( - service_cd, - reimb_method, - base_reimb_answer, - constants.VALID_CARVEOUTS, + service_term, + reimb_term, + carveout_definitions, + special_case_definitions ) logging.debug( f"Running carveout check for {filename} with prompt:\n{carveout_prompt}" @@ -249,7 +209,7 @@ def prompt_carveout_check( instruction=prompt_templates.CARVEOUT_CHECK_INSTRUCTION(), usage_label="CARVEOUT_CHECK", ) - logging.debug(f"LLM raw output for carveout check in {filename}: {llm_answer_raw}") + carveout_answer = string_utils.extract_text_from_delimiters( llm_answer_raw, Delimiter.PIPE ) @@ -322,7 +282,7 @@ def prompt_special_case_assignment( special_case_dicts: list, special_case_term: str, filename: str, -): +)->dict[str, str]: # If all non-term values are identical across dictionaries, return first dict non_term_keys = [k for k in special_case_dicts[0].keys() if "_TERM" not in k] @@ -433,7 +393,7 @@ def is_exhibit_lesser_of_redundant( ) -def validate_reimbursements_for_llm(answer_dict: dict, filename: str) -> bool: +def validate_reimbursements_for_llm(answer_dict: dict[str, str], filename: str) -> bool: """Use LLM to determine if a service has actual reimbursement methodology. Args: @@ -443,15 +403,16 @@ def validate_reimbursements_for_llm(answer_dict: dict, filename: str) -> bool: Returns: bool: True if the service has a clear reimbursement methodology, False otherwise. """ - service_term = str(answer_dict.get("SERVICE_TERM", "")).strip() - reimb_term = str(answer_dict.get("REIMB_TERM", "")).strip() + if isinstance(answer_dict, str): + answer_dict = {"SERVICE_TERM": "", "REIMB_TERM": answer_dict} - if string_utils.is_empty(service_term) or string_utils.is_empty(reimb_term): + if string_utils.is_empty(answer_dict.get("SERVICE_TERM", None)) or string_utils.is_empty(answer_dict.get("REIMB_TERM", None)): logging.warning( f"Missing SERVICE_TERM or REIMB_TERM in {filename}: {answer_dict}" ) return False - + + service_term, reimb_term = answer_dict["SERVICE_TERM"], answer_dict["REIMB_TERM"] prompt = prompt_templates.VALIDATE_REIMBURSEMENTS_PROMPT(service_term, reimb_term) logging.debug(f"Prompt for reimbursement validation in {filename}:\n{prompt}") @@ -473,68 +434,7 @@ def validate_reimbursements_for_llm(answer_dict: dict, filename: str) -> bool: return final_answer == "YES" -def split_compound_reimbursement_llm(answer_dict: dict, filename: str) -> list[dict]: - """ - Uses LLM to split a compound reimbursement term into distinct methodologies. - - Args: - answer_dict (dict): The service-reimbursement answer dictionary. - filename (str): The name of the file being processed. - - Returns: - list[dict]: List of dictionaries with split reimbursement terms. - """ - service_term = answer_dict.get("SERVICE_TERM", "").strip() - reimb_term = answer_dict.get("REIMB_TERM", "").strip() - - prompt = prompt_templates.SPLIT_REIMBURSEMENTS_PROMPT(service_term, reimb_term) - try: - logging.debug( - f"Invoking LLM for splitting reimbursements in {filename};\nPrompt:\n{prompt}" - ) - llm_response = llm_utils.invoke_claude( - prompt, - "haiku_latest", - filename, - max_tokens=4096, - cache=True, - instruction=prompt_templates.SPLIT_REIMBURSEMENTS_INSTRUCTION(), - ) - logging.debug( - f"LLM response for splitting reimbursements in {filename}:\n{llm_response}" - ) - - split_data = string_utils.universal_json_load(llm_response) - - if not split_data or len(split_data) == 1: - return [answer_dict] # No split needed, return original - - # Create new answer dictionaries for each split - split_answers = [] - for i, split_item in enumerate(split_data): - new_answer = answer_dict.copy() - - # Update service and reimbursement terms - split_service = split_item.get("SERVICE_TERM", "").strip() - split_methodology = split_item.get("REIMB_TERM", "").strip() - - new_answer["SERVICE_TERM"] = split_service - new_answer["REIMB_TERM"] = split_methodology - - # Add a flag to indicate that this was split - new_answer["COMPOUND_SPLIT_IND"] = "Y" - new_answer["COMPOUND_SPLIT_SEQUENCE"] = str(i + 1) - - split_answers.append(new_answer) - return split_answers - except Exception as e: - logging.error( - f"Error occurred while splitting reimbursement terms in {filename}: {e}" - ) - return [answer_dict] # Return original if error occurs - - -def prompt_dynamic(text, field_prompts, filename): +def prompt_dynamic(text: str, field_prompts, filename): """ Prompts an LLM to process dynamic fields in a given text and extracts structured responses. @@ -816,4 +716,53 @@ def prompt_global_lesser_of_breakout( ) return {"REIMB_TERM": f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"} +def prompt_dynamic_assignment(service_term: str, reimb_term: str, dynamic_field: Field, exhibit_text_simplified: str, page_num: str, constants: Constants, filename: str): + + field_name = dynamic_field.field_name + field_prompt = dynamic_field.get_prompt(constants) + prompt = prompt_templates.DYNAMIC_ASSIGNMENT(service_term, reimb_term, field_name, field_prompt, exhibit_text_simplified, page_num) + + llm_answer_raw = llm_utils.invoke_claude( + prompt, + model_id="sonnet_latest", + filename=filename + ) + + try: + llm_answer_final = string_utils.universal_json_load(llm_answer_raw) + return llm_answer_final + except: + logging.error(f"Failed to parse Dynamic Field: {field_name}. Response: {llm_answer_raw}") + return {field_name: "Extraction Error"} + + +def prompt_lesser_of_distribution(service_term: str, reimb_term: str, exhibit_text_simplified: str, page_num: str, filename: str): + prompt = prompt_templates.LESSER_OF_DISTRIBUTION( + service_term, + reimb_term, + exhibit_text_simplified, + page_num + ) + llm_answer_raw = llm_utils.invoke_claude( + prompt, + "sonnet_latest", + filename + ) + + if "N/A" in llm_answer_raw: + return reimb_term + else: + llm_answer_final = string_utils.extract_text_from_delimiters(llm_answer_raw, Delimiter.PIPE) + return llm_answer_final + +def prompt_lesser_of_check(service_term: str, reimb_term: str, filename: str): + prompt = prompt_templates.LESSER_OF_CHECK(service_term, reimb_term) + llm_answer_raw = llm_utils.invoke_claude(prompt, "sonnet_latest", filename) + + try: + llm_answer_final = string_utils.extract_text_from_delimiters(llm_answer_raw, Delimiter.PIPE) + return llm_answer_final == "Y" + except: + return True # Default to True - prefer extra rows to missing + diff --git a/fieldExtraction/src/investment/row_funcs.py b/fieldExtraction/src/investment/row_funcs.py index 4574672..e79dd93 100644 --- a/fieldExtraction/src/investment/row_funcs.py +++ b/fieldExtraction/src/investment/row_funcs.py @@ -1,58 +1,51 @@ import logging - +from copy import deepcopy import pandas as pd import src.investment.prompt_calls as prompt_calls from constants.constants import Constants from src.investment import one_to_one_funcs from src.utils import string_utils +from src.prompts.fieldset import FieldSet +from src import config def combine_one_to_n( - exhibit_text, - reimbursement_level_answers, - exhibit_level_answers, - special_case_breakout_answers, + exhibit_text: str, + all_exhibit_reimbursements: list[dict[str, str]], + all_exhibit_special_case: list[dict[str, str]], + exhibit_level_answers: dict[str, str], filename, -): +): + # Combine reimbursement_level with exhibit_level - combined_answers = [ - {**reimbursement_dict, **exhibit_level_answers} - for reimbursement_dict in reimbursement_level_answers - ] + # Reimbursement-level values take precedence - only add exhibit-level values that are missing in reimbursement + combined_answers = [] + for reimbursement_dict in all_exhibit_reimbursements: + combined_dict = {**reimbursement_dict} # Start with reimbursement-level (preserves reimbursement values) + # Only add exhibit-level values that don't exist in reimbursement (or are empty/N/A in reimbursement) + for key, value in exhibit_level_answers.items(): + if key not in combined_dict or string_utils.is_empty(combined_dict.get(key)): + combined_dict[key] = value + combined_answers.append(combined_dict) # Initialize final_combined_answers with the current combined list so that it has value even if no special cases are present final_combined_answers = combined_answers - # Combine special cases - for special_case_term, special_case_dicts in special_case_breakout_answers.items(): - - if not special_case_dicts: + special_case_fields = FieldSet(config.FIELD_JSON_PATH, field_type="special_case_check").list_fields() + for field_name in special_case_fields: + special_case_answer_dicts = [answer_dict for answer_dict in all_exhibit_special_case if field_name in answer_dict.keys()] + if not special_case_answer_dicts: continue - - # If only one set of special case answers, apply to all - if len(special_case_dicts) == 1: - final_combined_answers = [ - {**reimbursement_dict, **special_case_dicts[0]} - for reimbursement_dict in combined_answers - ] + elif len(special_case_answer_dicts) == 1: + special_case_answer_dict = special_case_answer_dicts[0] + for reimbursement_dict in final_combined_answers: + reimbursement_dict.update(special_case_answer_dict) else: - final_combined_answers = [ - { - **reimbursement_dict, - **prompt_calls.prompt_special_case_assignment( - exhibit_text, - reimbursement_dict, - special_case_dicts, - special_case_term, - filename, - ), - } - for reimbursement_dict in combined_answers - ] - + for reimbursement_dict in final_combined_answers: + special_case_answer_dict = prompt_calls.prompt_special_case_assignment(exhibit_text, reimbursement_dict, special_case_answer_dicts, field_name, filename) + reimbursement_dict.update(special_case_answer_dict) return final_combined_answers - def merge_one_to_one_into_one_to_n( one_to_n_results: pd.DataFrame, one_to_one_results: dict[str, str], constants: Constants ) -> pd.DataFrame: @@ -157,7 +150,12 @@ def inject_global_lesser_of_rows( # Create new rows for each REIMB_ID that was lacking lesser of logic new_rows = [] + seen_reimb_ids = set() for _, original_row in rows_needing_injection.iterrows(): + reimb_id = original_row.get("REIMB_ID") + if reimb_id in seen_reimb_ids: + continue # Skip if we've already processed this REIMB_ID + seen_reimb_ids.add(reimb_id) new_row = one_to_one_funcs.create_global_lesser_of_row( original_row, global_methodology_breakout ) diff --git a/fieldExtraction/src/investment/table_funcs.py b/fieldExtraction/src/investment/table_funcs.py deleted file mode 100644 index 33923db..0000000 --- a/fieldExtraction/src/investment/table_funcs.py +++ /dev/null @@ -1,683 +0,0 @@ -import ast -import logging -import re -from typing import Any, Dict, List, Pattern, Union - -import pandas as pd -import src.utils.string_utils as string_utils -from constants.constants import Constants -from src import config - -# Table regex patterns -START_PAGE_PATTERN: Pattern[str] = re.compile( - r"^(?!None\n).+\n\{.*?\}", re.DOTALL -) # Regex for start page - shouldn't have literal `None` -CONTINUATION_PAGE_PATTERN: Pattern[str] = re.compile( - r"^None\n\{.*?\}", re.DOTALL -) # Regex for continuation page - has literal `None` present at the start -START_PAGE_NUM_PATTERN: str = r"\n\d+\n?" -CONTINUATION_PAGE_NUM_PATTERN: str = r"\d+\n" -DICTIONARY_PATTERN: str = r"\{.*\}" -LIST_PATTERN: str = r"\[.*\]" - -# Markers for table start and end -START_MARKER: str = "-------Table Start--------" -END_MARKER: str = "-------Table End--------" - - -def convert_str_to_dataframe(text: str, is_continuation: bool = False) -> pd.DataFrame: - """ - Converts a string representation of table data to a pandas DataFrame. - - Supports both old dict format (for backward compatibility) and new list-of-rows format. - - For list format, treats the first row as column headers and the rest as data rows. - For dict format, assumes the keys are column names and values are lists of column data. - - Args: - text (str): The input string containing table data. - is_continuation (bool): Whether this is a continuation table (no header). Defaults to False. - If True, don't treat first row as headers. - - Returns: - pd.DataFrame: A DataFrame representation of the table data with string dtype. - """ - - # Try to parse as list first (new format) - list_match = re.search(LIST_PATTERN, text, re.DOTALL) - if list_match: - list_str = list_match.group(0) - try: - parsed_data = ast.literal_eval(list_str) - if isinstance(parsed_data, list): - if not parsed_data: - return pd.DataFrame() - - # Validate that all elements are lists (handle malformed data) - validated_rows = [] - for row in parsed_data: - if isinstance(row, list): - validated_rows.append( - [str(cell) for cell in row] - ) # Convert all to strings - else: - # Skip malformed rows or convert single values to single-item lists - if isinstance(row, str): - validated_rows.append([row]) - # Skip other types entirely - - if not validated_rows: - return pd.DataFrame() - - # Handle variable row lengths by padding with empty strings - max_cols = ( - max(len(row) for row in validated_rows) if validated_rows else 0 - ) - padded_rows = [ - row + [""] * (max_cols - len(row)) for row in validated_rows - ] - - if is_continuation: - # CONTINUATION: all rows are data, use generic column names - columns = [f"col_{i}" for i in range(max_cols)] - return pd.DataFrame(padded_rows, columns=columns, dtype=str) - else: - # REGULAR: treat first row as headers - if len(padded_rows) > 1: - headers = padded_rows[0] - data_rows = padded_rows[1:] - return pd.DataFrame(data_rows, columns=headers, dtype=str) - elif len(padded_rows) == 1: - # Only one row - treat as headers with no data - headers = padded_rows[0] - return pd.DataFrame(columns=headers, dtype=str) - else: - # Empty list - return empty DataFrame - return pd.DataFrame() - except (SyntaxError, ValueError): - logging.error(f"Invalid list format: {list_str}") - - # Fall back to dict format (old format) - but warn about deprecation - dict_match = re.search(DICTIONARY_PATTERN, text) - if dict_match: - dict_str = dict_match.group(0) - try: - parsed_data = ast.literal_eval(dict_str) - if isinstance(parsed_data, dict): - logging.warning( - "Warning: old table (dict) format detected. Please update to new list-of-rows format by rerunning textract on this file." - ) - # No need to set the headers explicitly, DataFrame will use keys as column names - return pd.DataFrame(parsed_data, dtype=str) - except (SyntaxError, ValueError): - logging.error(f"Invalid dictionary format: {dict_str}") - - return pd.DataFrame() # Return empty DataFrame if no valid format found - - -def convert_str_to_dict(text: str) -> Dict[str, Any]: - """ - Converts a string representation of a dictionary to an actual dictionary. - - This function searches for a dictionary pattern within the given text and - converts the matched string to a dictionary using `ast.literal_eval`. - - Args: - text (str): The input string that potentially contains a dictionary. - - Returns: - dict: The dictionary extracted from the input string. If no dictionary - pattern is found, an empty dictionary is returned. - """ - match = re.search(DICTIONARY_PATTERN, text) - if match: - dict_str: str = match.group(0) - try: - return ast.literal_eval(dict_str) - except (SyntaxError, ValueError): - logging.error(f"Invalid dictionary format: {dict_str}") - return {} - - -class Table: - """ - Represents a table extracted from a document page. - - This class handles table data parsing, combination, and splitting operations - for tables found in OCR-processed documents. - - Attributes: - page_num (str): The page number where the table is located. - metadata (str): Text appearing before the table on the page. - header (str): The table's title/header text. - table (pd.DataFrame): The table data with string dtype. - start_index (int): Starting position of the table in the page text. - end_index (int): Ending position of the table in the page text. - post_table_text (str): Text appearing after the table on the page. - is_continuation (bool): Whether this table continues from a previous page. - - Key Methods: - combine(other): Combine another table into this one with column alignment. - split(row_limit): Split into multiple tables based on row count. - to_list_format(): Convert to list-of-lists format for serialization. - """ - - def __init__( - self, - page_num: str, - metadata: str, - header: str, - table_data: Union[List[List[str]], pd.DataFrame, Dict[str, Any]], - start_index: int, - end_index: int, - post_table_text: str = "", - is_continuation: bool = False, - ): - self.page_num = page_num - self.metadata = metadata - self.header = header - self.start_index = start_index - self.end_index = end_index - self.is_continuation = is_continuation - self.post_table_text: str = post_table_text # Text after the table in the page - - # Handle a few different input types - if isinstance(table_data, str): - self.table = convert_str_to_dataframe( - table_data, is_continuation=is_continuation - ) - elif isinstance(table_data, pd.DataFrame): - self.table = table_data.astype(str) # Ensure DataFrame is of string dtype - elif isinstance(table_data, list): - # Direct list - check if continuation table - if is_continuation: - logging.debug(f"Detected continuation table on page {page_num}.") - # All rows are data rows, no headers - max_cols = max(len(row) for row in table_data) if table_data else 0 - padded_rows = [row + [""] * (max_cols - len(row)) for row in table_data] - columns = [f"col_{i}" for i in range(max_cols)] - self.table = pd.DataFrame(padded_rows, columns=columns, dtype=str) - else: - # Non-continuation table - treat first row as headers - if not table_data: - self.table = pd.DataFrame() - else: - # Handle variable row lengths by padding with empty strings - max_cols = max(len(row) for row in table_data) if table_data else 0 - padded_rows = [ - row + [""] * (max_cols - len(row)) for row in table_data - ] - - if len(padded_rows) > 1: - headers = padded_rows[0] - data_rows = padded_rows[1:] - self.table = pd.DataFrame(data_rows, columns=headers, dtype=str) - elif len(padded_rows) == 1: - # Only one row - treat as headers with no data - headers = padded_rows[0] - self.table = pd.DataFrame(columns=headers, dtype=str) - else: - # Empty list - return empty DataFrame - self.table = pd.DataFrame() - - elif isinstance(table_data, dict): # Old dict format - warn and convert - logging.warning( - "Warning: old table (dict) format detected. Please update to new list-of-rows format by rerunning textract on this file." - ) - self.table = pd.DataFrame(table_data, dtype=str) - else: - self.table = ( - pd.DataFrame() - ) # Default to empty DataFrame if no valid format found - - def to_list_format(self) -> List[List[str]]: - """ - Convert the DataFrame back to list-of-rows format for serialization, with headers - as the first row. - - Returns: - List[List[str]]: The table data as a list of lists. - """ - if self.table.empty: - return [] - - headers = self.table.columns.tolist() - data_rows = self.table.values.tolist() - - # If continuation table with generic column names, don't include headers - if self.is_continuation and all(col.startswith("col_") for col in headers): - return data_rows # Just the data, no headers - - # For all other cases (regular tables), include headers - return [headers] + data_rows - - def combine(self, other: "Table"): - """ - Combine another table into this one with intelligent column handling. - - Handles common OCR data issues like column count mismatches by automatically - padding or truncating the continuation table to match the main table. - - Business Rules: - - Empty tables: Handled gracefully, preserving post-table text - - Column mismatches: Continuation table adjusted to match main table - - Continuation tables: Column names aligned to main table headers - - Post-table text: Accumulated from both tables - - Args: - other: The table to combine into this one (consumed after operation). - - Note: - This method modifies the current table in-place. Column count mismatches - are resolved by padding short rows or truncating long rows in the - continuation table. - """ - - # Handle empty tables - if other.table.empty: - # if other table is empty, just update post_table_text if needed - if other.post_table_text: - self.post_table_text = other.post_table_text - return - - if self.table.empty: - # If this table is empty, just copy the other table - self.table = other.table.copy() - if other.post_table_text: - self.post_table_text = other.post_table_text - return - - # Handle column count mismatches - if self.table.shape[1] != other.table.shape[1]: - main_cols = self.table.shape[1] - other_cols = other.table.shape[1] - - logging.warning( - f"Column count mismatch when combining tables: main table has {main_cols} columns, " - f"continuation table has {other_cols} columns. Adjusting continuation table to match." - ) - - # Create a copy to modify - other_table_adjusted = other.table.copy() - - if other_cols < main_cols: - # Pad with empty columns - for i in range(main_cols - other_cols): - col_name = f"col_{other_cols + i}" - # Ensure unique column name - while col_name in other_table_adjusted.columns: - col_name = f"{col_name}_padded" - other_table_adjusted[col_name] = "" # Add empty column - else: - # Truncate to match main table column count - other_table_adjusted = other_table_adjusted.iloc[:, :main_cols] - - # Ensure column names match (this will be our "other" table for the rest of the method) - other_table_adjusted.columns = self.table.columns - - # Combine the adjusted table - self.table = pd.concat( - [self.table, other_table_adjusted], ignore_index=True - ) - else: - # Column counts match - proceed with normal logic - # If other table has generic column names (continuation table), rename them - if other.is_continuation or ( - list(other.table.columns) != list(self.table.columns) - ): - other_renamed = other.table.copy() - other_renamed.columns = self.table.columns # Use our column names - self.table = pd.concat([self.table, other_renamed], ignore_index=True) - else: - self.table = pd.concat([self.table, other.table], ignore_index=True) - - if other.post_table_text.strip(): # Only if the other table has post_table_text - if self.post_table_text.strip(): - if not other.post_table_text.strip() == self.post_table_text.strip(): - # If both tables have post_table_text AND they're not identical, append the other table's text - self.post_table_text = ( - f"{self.post_table_text}\n{other.post_table_text}" - ) - else: - # Only the other table has post_table_text, so just set it - self.post_table_text = other.post_table_text - - def split(self, row_limit: int) -> List["Table"]: - """Split the table into multiple tables each with at most row_limit rows.""" - if len(self.table) <= row_limit: - return [self] - - tables = [] - for start_row in range(0, len(self.table), row_limit): - end_row = min(start_row + row_limit, len(self.table)) - subset_df = self.table.iloc[start_row:end_row].copy() - - # Create new table with DataFrame directly - new_table = Table( - page_num=self.page_num, - metadata=self.metadata, - header=self.header, - table_data=subset_df, - start_index=self.start_index, # Keep original start index - end_index=self.end_index, # Keep original end index - ) - tables.append(new_table) - - if tables: - # Only the LAST split table gets the post-table text - tables[-1].post_table_text = self.post_table_text - - return tables - - -def get_table_info( - text_dict: Dict[str, str], EXHIBIT_HEADER_MARKERS: list -) -> Dict[str, List[Table]]: - """ - Extracts all of the Table objects from the text_dict. - - Args: - text_dict (Dict[str, str]): A dictionary with keys as page numbers and values as page text. - - Returns: - Dict[str, List[Table]]: A dictionary with keys as page numbers and values as a list of Table objects. - """ - table_dict = {} - for page_num, page_text in text_dict.items(): - tables = [] - start_index = 0 - first_table_start_index = page_text.find(START_MARKER) - - while True: - start_index = page_text.find(START_MARKER, start_index) - if start_index == -1: - break - - end_index = page_text.find(END_MARKER, start_index) - if end_index == -1: - break - - # Don't consider tables that don't have relevant info - table_text = page_text[start_index:end_index] - - if "%" not in table_text and "$" not in table_text: - start_index = end_index + len(END_MARKER) - continue - - metadata = page_text[:first_table_start_index].strip() - header_start = start_index + len(START_MARKER) - - # Look for either [ or { to find start of table data - list_start = page_text.find("[", header_start) - dict_start = page_text.find("{", header_start) - - if list_start != -1 and (dict_start == -1 or list_start < dict_start): - # New list format - # If list starts first, we assume it's a list of rows - header = page_text[header_start:list_start].strip() - table_data = page_text[list_start:end_index].strip() - - # Check if this page starts a new exhibit from the text - exhibit_pattern = "|".join(EXHIBIT_HEADER_MARKERS) - starts_new_exhibit = re.search( - rf"\b({exhibit_pattern})\b", page_text[:100], re.IGNORECASE - ) - - is_continuation = ( - header == "" # No header text - and len(page_text[header_start:list_start].strip()) - == 0 # Nothing between marker and `[` - and not starts_new_exhibit # NOT starting a new exhibit - ) - elif dict_start != -1: - # Old dict format - # If dict starts first, we assume it's a dictionary - header = page_text[header_start:dict_start].strip() - table_data = page_text[dict_start:end_index].strip() - - is_continuation = header == "" - else: - # No table data found - break - - tables.append( - Table( - page_num, - metadata, - header, - table_data, - start_index, - end_index + len(END_MARKER), - is_continuation=is_continuation, - ) - ) - - start_index = end_index + len(END_MARKER) - - if tables: - last_table_end = tables[-1].end_index - # If there's text after the last table, store it as post-table text - post_table_text = page_text[last_table_end:].strip() - - # Store post-table text in the last table - tables[-1].post_table_text = post_table_text - table_dict[page_num] = tables - return table_dict - - -def recreate_page(page_table_list: List[Table]) -> str: - """ - Puts together the tables in the page_table_list to recreate the page text. - - Args: - page_table_list (List[Table]): A list of Table objects. - - Returns: - str: The text of the page with the tables combined. - """ - page_text = "" - for i in range(len(page_table_list)): - t = page_table_list[i] - table_as_list = t.to_list_format() - if i == 0: - page_text += "\n".join( - [t.metadata, START_MARKER, t.header, str(table_as_list), END_MARKER] - ) - else: - page_text += "\n".join( - ["\n", START_MARKER, t.header, str(table_as_list), END_MARKER] - ) - - # Add post-table text if it exists - if t.post_table_text.strip(): - page_text += "\n" + t.post_table_text.strip() - return page_text - - -def split_large_table_with_headers( - table: Table, row_limit: int = config.TABLE_ROW_LIMIT -) -> List[str]: - """Split a large table into smaller chunks while preserving headers. - - Args: - table (Table): The table to split. - row_limit (int): The row limit to use. By default uses config.TABLE_ROW_LIMIT. - - Returns: - List[str]: A list of text chunks representing the split table. - """ - # Use existing table splitting logic - split_tables = split_table_by_rows(table, row_limit) - - chunks = [] - for i, split_table in enumerate(split_tables): - chunk_parts = [] - - # add metadata if it exists - if split_table.metadata: - chunk_parts.append(split_table.metadata) - - # Add tables structure - chunk_parts.extend( - [ - START_MARKER, - split_table.header, # Header replicated in each chunk - str(split_table.to_list_format()), # Convert table to list format - END_MARKER, - ] - ) - - # Add post-table text only to the last chunk - if i == len(split_tables) - 1 and table.post_table_text: - chunk_parts.append(table.post_table_text) - - chunks.append("\n".join(chunk_parts).strip()) - - return chunks - - -def combine_continuous_tables( - text_dict: Dict[str, str], table_dict: Dict[str, List[Table]] -) -> Dict[str, str]: - """Combine continuous tables across pages. - Args: - text_dict (Dict[str, str]): A dictionary with keys as page numbers and values as page text. - table_dict (Dict[str, List[Table]]): A dictionary with keys as page numbers and values as a list of Table objects. - - Returns: - Dict[str, str]: Updated text_dict with combined tables. - """ - pages_to_remove = set() - updated_text_dict = text_dict.copy() - - for page_num, page_text in text_dict.items(): - if page_num in pages_to_remove: - continue - - if page_num not in table_dict or not table_dict[page_num]: - # No tables on this page, nothing to combine - continue - - # Use the existing is_continuation flag from the table object - first_table = table_dict[page_num][0] - - if first_table.is_continuation: - # This is a continuation table, find the main table to combine with - main_table_page = None - for prev_page in sorted( - [p for p in table_dict.keys() if int(p) < int(page_num)], - key=string_utils.page_key_sort, - reverse=True, - ): - if prev_page in table_dict and table_dict[prev_page]: - main_table_page = prev_page - break - - if main_table_page and main_table_page not in pages_to_remove: - # Combine the tables - main_table = table_dict[main_table_page][ - -1 - ] # Last table on the main page - continuation_table = table_dict[page_num][ - 0 - ] # First table on current page - main_table.combine(continuation_table) - - # Update main page with ALL tables from that page - updated_text_dict[main_table_page] = recreate_page( - table_dict[main_table_page] - ) - - # Mark continuation page for removal - pages_to_remove.add(page_num) - # Remove continuation pages (we combined them into the main pages) - for page_num in pages_to_remove: - updated_text_dict.pop(page_num, None) - - return updated_text_dict - - -def split_tables_by_rows( - text_dict: Dict[str, str], EXHIBIT_HEADER_MARKERS: list, row_limit: int -) -> Dict[str, str]: - """Split tables that exceed the row limit, add .0 suffix to pages with tables - - Args: - text_dict (Dict[str, str]): A dictionary with keys as page numbers and values as page text. - row_limit (int): The maximum number of rows allowed in a single table. - - Returns: - Dict[str, str]: Updated text_dict with split tables and .0 suffixes for table pages - """ - result = {} - - for page_num, page_text in text_dict.items(): - # get tables on this page - page_table_dict = get_table_info({page_num: page_text}, EXHIBIT_HEADER_MARKERS) - - if page_num in page_table_dict and page_table_dict[page_num]: - # This page has tables - table = page_table_dict[page_num][0] - - if len(table.table) > row_limit: - # Large table - split it - logging.debug( - f"Splitting table on page {page_num}: {len(table.table)} rows > row_limit ({row_limit})" - ) - split_chunks = split_large_table_with_headers(table, row_limit) - for i, chunk in enumerate(split_chunks): - # Use .0, .1, .2 suffixes for split tables - result[f"{page_num}.{i}"] = chunk.strip() - else: - # Small table - keep it intact - logging.debug( - f"Keeping table on page {page_num}: {len(table.table)} rows <= row_limit ({row_limit})" - ) - result[f"{page_num}.0"] = page_text - else: - # No tables on this page, keep it as is - result[page_num] = page_text - return result - - -def split_table_by_rows(table: Table, row_limit: int) -> List[Table]: - """Split a single table into multiple tables if it exceeds the row limit. - - Args: - table (Table): The table to split. - row_limit (int): The maximum number of rows allowed in a single table. - - Returns: - List[Table]: A list of tables, each with a maximum of row_limit rows. - """ - df = table.table - total_rows = len(df) - if total_rows <= row_limit: - return [table] - - split_tables = [] - for start_idx in range(0, total_rows, row_limit): - end_idx = min(start_idx + row_limit, total_rows) - - # Create subset DataFrame - subset_df = df.iloc[start_idx:end_idx].copy() - - # Create new table object - split_table = Table( - page_num=table.page_num, - metadata=table.metadata, - header=table.header, - table_data=subset_df, - start_index=table.start_index, # Keep original start index - end_index=table.end_index, # Keep original end index - post_table_text="", # Only last split gets post-table text - is_continuation=False, - ) - - split_tables.append(split_table) - - # Add post-table text to the last split - if split_tables: - split_tables[-1].post_table_text = table.post_table_text - - return split_tables diff --git a/fieldExtraction/src/parent_child/parent_child_preprocessing.py b/fieldExtraction/src/parent_child/parent_child_preprocessing.py index c81265e..d061044 100644 --- a/fieldExtraction/src/parent_child/parent_child_preprocessing.py +++ b/fieldExtraction/src/parent_child/parent_child_preprocessing.py @@ -4,7 +4,6 @@ warnings.filterwarnings("ignore") import pandas as pd import nltk import ssl - import re import string import src.config as config @@ -29,6 +28,8 @@ def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_ ) return all_fields_df +# ... rest of the file remains the same + def create_one_to_one_df(df): """ diff --git a/fieldExtraction/src/prompts/fieldset.py b/fieldExtraction/src/prompts/fieldset.py index 8565b4c..10a0330 100644 --- a/fieldExtraction/src/prompts/fieldset.py +++ b/fieldExtraction/src/prompts/fieldset.py @@ -43,11 +43,6 @@ class Field: self.relationship = field_dict["relationship"] self.field_type = field_dict["field_type"] self.prompt = field_dict["prompt"] if "prompt" in field_dict else None - self.reimb_level_prompt = ( - field_dict["reimb_level_prompt"] - if "reimb_level_prompt" in field_dict - else None - ) self.valid_values = ( field_dict["valid_values"] if "valid_values" in field_dict else None ) @@ -203,6 +198,9 @@ class Field: raise AttributeError( f"Breakout template '{self.breakout_template}' not found in prompt_templates module" ) + + def get_valid_values(self, constants: Constants): + return constants.get_constant(self.valid_values) class FieldSet: @@ -293,7 +291,7 @@ class FieldSet: if field.field_name not in [field.field_name for field in self.fields]: self.fields.append(field) - def get_field(self, field_name): + def get_field(self, field_name) -> Field: for field in self.fields: if field.field_name == field_name: return field diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 711374c..917ef4b 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -2,20 +2,58 @@ { "field_name": "SERVICE_TERM", "relationship": "one_to_n", - "field_type": "reimbursement_level", - "prompt": "What is the healthcare Service that is being reimbursed? Include the full Service, including any distinguishing context. Include any codes that are part of the service, including revenue codes and procedure codes. Use only exact text from the document.\n\nEXCLUDE the following non-healthcare services:\n- Administrative fees (copying, records requests, processing fees)\n- Operational costs (supplies, equipment rental)\n- Legal/compliance costs (audits, investigations)\n- General business expenses\n- General payment process or claim processing language (e.g. Clean Claims)\n\nOnly extract services that involve direct patient care, medical procedures, or clinical services. If no healthcare service is being reimbursed, return 'N/A'." + "field_type": "reimbursement_primary", + "prompt": "Describes the medical service or procedure that is being reimbursed. It can be very general (like 'Covered Services') or very specific (like 'CPT 98765')." }, { "field_name": "REIMB_TERM", "relationship": "one_to_n", - "field_type": "reimbursement_level", - "prompt": "Write the full sentence or paragraph in the text describing the reimbursement methodology, including any lesser of language, rates, unit_of_measure and other important information." + "field_type": "reimbursement_primary", + "prompt": "Describes the method by which the price of the Service is calculated. It can be a simple dollar value (like '$100 per unit') or a percentage of some other value (like '100% of the Medicare Fee Schedule')." }, { "field_name": "LESSER_OF_IND", "relationship": "one_to_n", "field_type": "methodology_breakout", - "prompt": "Determine if the reimbursement methodology creates a functional comparison where the smaller/lowest amount from two or more distinct calculations is selected for payment.\n\n**EXPLICIT COMPARATIVE LANGUAGE:**\n• 'lesser of', 'lower of', 'minimum of'\n• 'whichever is less', 'whichever is lower', 'whichever is smaller'\n• 'the smaller of', 'the lowest of'\n\n**FUNCTIONAL COMPARISON THROUGH LIMITING LANGUAGE:**\n• 'not to exceed' + another calculation (creates comparison between calculation and cap)\n• 'up to a maximum of' + base methodology (creates comparison between rate and ceiling)\n• 'capped at' + underlying calculation (creates comparison between rate and limit)\n• 'subject to a limit of' + payment formula (creates comparison between formula result and limit)\n\n**SPECIAL CASE:**\nIf the methodology contains 'At no time shall Plan pay an amount that exceeds Provider's billed charges' (or substantially similar language) AND there is at least one other reimbursement calculation present, treat this as a functional comparison between the other calculation and billed charges.\n\n**WHAT QUALIFIES:**\n• Two or more distinct reimbursement calculations/rates where the smallest is selected\n• Payment methodologies with caps that create effective comparisons\n• Nested or layered structures involving multiple rate options\n\n**WHAT DOES NOT QUALIFY:**\n• Single calculation methods without comparison elements\n• Administrative limitations that don't create payment alternatives\n• Eligibility restrictions or service quantity limits\n• The special case phrase when it appears alone without other calculations\n\n**EXAMPLES:**\n• '80% of Medicare rates, not to exceed $500' → 'Y' (compares 80% calculation vs $500 cap)\n• 'Lesser of billed charges or fee schedule' → 'Y' (explicit comparison)\n• '90% of charges, maximum $1000 per day' → 'Y' (functional comparison)\n• 'Flat rate of $200' → 'N' (single calculation, no comparison)\n\nReturn 'Y' if the methodology creates any functional comparison for payment selection, otherwise 'N'.\n\nAnswer 'Y' for ALL components involved in any comparison structure, including nested or layered scenarios." + "prompt": "Determine if the reimbursement methodology creates a functional comparison where the smaller/lowest amount from two or more distinct calculations is selected for payment. Look for structure like 'lesser of A and B', 'A, not to exceed B', or similar. Return 'Y' if the methodology creates any functional comparison for payment selection, otherwise 'N'. If 'Y', then answer 'Y' for ALL components involved in any comparison structure, including nested or layered scenarios." + }, + { + "field_name": "AARETE_DERIVED_REIMB_METHOD", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'\n- In case of DRG, MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC(Ambulatory Payment Classification)/OPPS(Outpatient Prospective Payment System) or Ambulatory Surgical Center (ASC), return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP)\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'\n- Provider interest rates are not considered 'Flat Rate'.\n- Charge master increase caps or limitations on provider pricing behavior are NOT reimbursement methodologies and should be ignored.\n- Always return 'Grouper' as the priority value if any grouper-related codes or references (e.g., MS-DRG, APR-DRG, DRG, etc.) are present in the SERVICE or METHODOLOGY fields — regardless of any other reimbursement methods (such as case rate, per diem, or lesser-of comparisons) mentioned. \n\nIf no specific rule is matched, map to the most appropriate value from the given valid values.", + "valid_values": "VALID_REIMB_METHODOLOGY" + }, + { + "field_name": "REIMB_FEE_RATE", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "If the reimbursement rate is a flat fee (a specific dollar amount), what is that amount? If no flat fee is mentioned, return 'N/A'." + }, + { + "field_name": "REIMB_PCT_RATE", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "What is the percentage rate of reimbursement? Extract the numeric percentage value (without % symbol). Look for percentages in both numeric form (e.g., '50%', '100%') and written form (e.g., 'fifty percent', 'one hundred percent'). Convert written percentages to numbers (fifty = 50, one hundred = 100, etc.). If the text contains 'plus' followed by a percentage, add that percentage to 100 (e.g., 'plus 5%' = 105). If no percentage is found AND no specific dollar amount is mentioned in the text, return 100 as default. If a specific dollar amount is mentioned, return 'N/A'." + }, + { + "field_name": "REIMB_CONVERSION_FACTOR", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "What is the conversion factor (a specific dollar amount) mentioned in the text? A conversion factor is a dollar amount used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit — RVU) into an actual payment amount. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)." + }, + { + "field_name": "UNIT_OF_MEASURE", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "Extract the unit of measure for flat rate or fee schedule reimbursements. Look for specific units like 'per day', 'per visit', 'per procedure', 'per ASA unit', etc. If a specific unit of measure is found and matches the valid values list, return that exact match. If a unit is mentioned but not in the valid values list, return 'Per Unit'. For flat rate methodologies where no specific unit is mentioned, default to 'Per Unit' regardless of whether procedure codes are present. For non-flat rate methodologies (percentages, bundled payments, etc.) where no unit is explicitly stated, return 'N/A'. Valid values are: {valid_values}. Don't assume units based on context; only extract explicitly stated units, except for the flat rate default rule.", + "valid_values": "VALID_UNIT_OF_MEASURE" + }, + { + "field_name": "FEE_SCHEDULE", + "relationship": "one_to_n", + "field_type": "methodology_breakout", + "prompt": "Write the full name of the Fee Schedule here. Include any secondary part of the fee schedule such as clinical lab, outpatient imaging, DMEPOS, encounter, surgeon, clinical diagnostic lab, parenteral and enteral nutrition (PEN), ASC, Part B Drug, Behavioral health, obstetrical, dental, transportation, etc. CHIP, STAR, and other Programs are not fee schedules. Allowed amounts or allowable charges that are not set by CMS, Medicaid, or Medicare do not constitute a fee schedule. Only use exact text from the Reimbursement Term." }, { "field_name": "GLOBAL_LESSER_OF_STATEMENT", @@ -29,51 +67,14 @@ "field_type": "TBD", "prompt": "If the methodology text indicates that the reimbursement shall be the greater of two or more values, write 'Y' for this field. Otherwise, write 'N'." }, - { - "field_name": "AARETE_DERIVED_REIMB_METHOD", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'\n- In case of DRG, MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC(Ambulatory Payment Classification)/OPPS(Outpatient Prospective Payment System) or Ambulatory Surgical Center (ASC), return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP)\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'\n- Provider interest rates are not considered 'Flat Rate'.\n- Charge master increase caps or limitations on provider pricing behavior are NOT reimbursement methodologies and should be ignored.\n- Always return 'Grouper' as the priority value if any grouper-related codes or references (e.g., MS-DRG, APR-DRG, DRG, etc.) are present in the SERVICE or METHODOLOGY fields — regardless of any other reimbursement methods (such as case rate, per diem, or lesser-of comparisons) mentioned. \n\nIf no specific rule is matched, map to the most appropriate value from the given valid values.", - "valid_values": "VALID_REIMB_TERM_FULL" - }, - { - "field_name": "UNIT_OF_MEASURE", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "Extract the unit of measure for flat rate or fee schedule reimbursements. Look for specific units like 'per day', 'per visit', 'per procedure', 'per ASA unit', etc. If a specific unit of measure is found and matches the valid values list, return that exact match. If a unit is mentioned but not in the valid values list, return 'Per Unit'. For flat rate methodologies where no specific unit is mentioned, default to 'Per Unit' regardless of whether procedure codes are present. For non-flat rate methodologies (percentages, bundled payments, etc.) where no unit is explicitly stated, return 'N/A'. Valid values are: {valid_values}. Don't assume units based on context; only extract explicitly stated units, except for the flat rate default rule.", - "valid_values": "VALID_UNIT_OF_MEASURE" - }, - { - "field_name": "REIMB_FEE_RATE", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "If the reimbursement rate is a flat fee (a specific dollar amount), what is that amount? If no flat fee is mentioned, return 'N/A'." - }, - { - "field_name": "REIMB_PCT_RATE", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "What is the percentage rate of reimbursement? Extract the numeric percentage value (without % symbol).\n\nLook for percentages in both numeric form (e.g., '50%', '100%') and written form (e.g., 'fifty percent', 'one hundred percent').\n\nConvert written percentages to numbers (fifty = 50, one hundred = 100, etc.).\n\nIf the text contains 'plus' followed by a percentage, add that percentage to 100 (e.g., 'plus 5%' = 105).\n\nIf no percentage is found AND no specific dollar amount is mentioned in the text, return 100 as default.\n\nIf a specific dollar amount is mentioned, return 'N/A'." - }, - { - "field_name": "REIMB_CONVERSION_FACTOR", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "What is the conversion factor (a specific dollar amount) mentioned in the text? A conversion factor is a dollar amount used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit — RVU) into an actual payment amount. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)." - }, + { "field_name": "REIMB_DATES", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "Identify the effective and termination date range or duration language for which reimbursement rates apply:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • 'Termination' or 'termination date'\n • 'Expires' or 'expiration date'\n • Date ranges with 'from/through', 'start/end', 'through', or 'until'\n- Also capture duration/term language when mentioned in context of reimbursement rates:\n • Examples: 'Initial Term', 'Renewal Term', 'Extension Term', 'Base Term'\n • 'First year', 'Second year', 'Annual', 'Monthly', 'Quarterly'\n • 'Contract period', 'Agreement term', 'Service period'\n • Any time-based descriptors related to rate applicability\n- Capture date ranges in format: 'start_date - end_date' or single dates\n- For duration language without specific dates, capture the duration term as-is\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)\n- Do NOT include dates written in the footer of the page\n- If only one date is found, capture it as a single date\n- If multiple separate date ranges exist, capture each as a separate entry\n- If no date range, date, duration, or term is found, return 'N/A'", + "prompt": "The date range, delineated by effective and termination dates, for which a Reimbursement Term applies. They may be labelled 'Effective Date' and 'Termination Date', or similar. They may also be presented as a range, like 'Date1 - Date2'. Lastly, there may be language simply describing a term, like 'Initial Term' or 'Renewal Term'. EXCLUDE dates written in the footer of the page. If multiple separate date ranges exist, capture each as a separate entry. Return each date or date range EXACTLY as it's written in the text.", "format": "date_range" }, - { - "field_name": "FEE_SCHEDULE", - "relationship": "one_to_n", - "field_type": "methodology_breakout", - "prompt": "Write the full name of the Fee Schedule here. Include the secondary part of the fee schedule such as clinical lab, outpatient imaging, DMEPOS, encounter, surgeon, clinical diagnostic lab, parenteral and enteral nutrition (PEN), ASC, Part B Drug, Behavioral health, obstetrical, dental, transportation, etc. CHIP and STAR are not fee schedules. Only use exact text from the methodology." - }, { "field_name": "AARETE_DERIVED_FEE_SCHEDULE", "relationship": "one_to_n", @@ -98,9 +99,9 @@ "field_name": "LOB", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "Extract ONLY LOBs that are **explicitly mentioned** in the text. Do NOT extract based on Programs, Products, context, implications, or any other form of inference. Choose LOBs ONLY from the following values: {valid_values}. ", - "reimb_level_prompt": "What LOB applies specifically to this service-reimbursement pair? Extract only LOBs that are directly associated with this particular service or methodology. Choose ONLY from: {valid_values}. If no LOB is specifically associated with this service-reimbursement pair, return 'N/A'.", - "valid_values": "VALID_LOBS" + "prompt": "A distinct, high-level category of health insurance coverage. Choose LOBs ONLY from the following values: {valid_values}. Do NOT extract Programs, Products, or other entities listed. Choose ONLY from the valid values presented. Only return values that are explicitly written - do NOT fill this value out by deducing the answer from sub-Programs that may fall under one of the valid answers. If only the sub-Program is seen, return 'N/A'.", + "valid_values": "VALID_LOBS", + "retrieval_question": "What Line of Business does this contract apply to?" }, { "field_name": "AARETE_DERIVED_LOB", @@ -114,9 +115,9 @@ "field_name": "NETWORK", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "List any Networks explicitly stated in the text. Choose ONLY from the following: {valid_values}.", - "reimb_level_prompt": "What Network applies specifically to this service-reimbursement pair? Look for network references that are directly associated with this particular service or methodology. Choose ONLY from: {valid_values}. If no network is specifically associated with this service-reimbursement pair, return 'N/A'.", - "valid_values": "VALID_NETWORKS" + "prompt": "A type of managed care structures that defines what providers a patient can use. Choose Networks ONLY from the following values: {valid_values}. Only return values that are explicitly written.", + "valid_values": "VALID_NETWORKS", + "retrieval_question" : "What Network (HMO, PPO, etc) does this contract apply to?" }, { "field_name": "AARETE_DERIVED_NETWORK", @@ -130,9 +131,9 @@ "field_name": "PRODUCT", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "List any Products explicitly stated in the text. Look for the brand name of a health plan or product. Do not include the name of the entire healthplan (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list. Use reasonable judgment for obvious synonyms and equivalent terms.", - "reimb_level_prompt": "What Product applies specifically to this service-reimbursement pair? Look for product references that are directly associated with this particular service or methodology. Choose ONLY from: {valid_values}. If no product is specifically associated with this service-reimbursement pair, return 'N/A'.", - "valid_values": "VALID_PRODUCTS" + "prompt": "The brand name of a health plan or product offered by the Payer. Do NOT include the name of the Payer itself (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list. Use reasonable judgment for obvious synonyms and equivalent terms.", + "valid_values": "VALID_PRODUCTS", + "retrieval_question": "What Product does this contract apply to?" }, { "field_name": "AARETE_DERIVED_PRODUCT", @@ -145,9 +146,9 @@ "field_name": "PROGRAM", "relationship": "one_to_n", "field_type": "dynamic_primary", - "prompt": "List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. Choose the answer from the list that most closely matches what is found in the text. Use reasonable judgment for obvious synonyms and equivalent terms (e.g., 'Medicare-Medicaid Program' matches 'Medicare-Medicaid Plan').", - "reimb_level_prompt": "What Program applies specifically to this service-reimbursement pair? Look for program references that are directly associated with this particular service or reimbursement methodology. Choose ONLY from: {valid_values}. Note: 'Medicare' alone does NOT mean 'Medicare Advantage'. Use reasonable judgment for synonyms (e.g., 'Medicare-Medicaid Program' matches 'Medicare-Medicaid Plan'). If no program is specifically associated with this service-reimbursement pair, return 'N/A'.", - "valid_values": "VALID_PROGRAMS" + "prompt": "The brand name of a health plan or product offered by the State or Federal government. List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' on it's own does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. IMPORTANT - CHIP vs CHIPP distinction: 'CHIP' (Children's Health Insurance Program) is a valid standalone value. If you see 'CHIP HMO' or just 'CHIP' (without 'Perinate'), extract 'CHIP'. However, if you see 'CHIP Perinate' or 'CHIP-P' (Children's Health Insurance Program Perinate), extract 'CHIPP' (NOT 'CHIP'). These are two distinct programs - do not confuse them. Choose the answer from the list that most closely matches what is found in the text. Use reasonable judgment for obvious synonyms and equivalent terms (e.g., 'Medicare-Medicaid Program' matches 'Medicare-Medicaid Plan').", + "valid_values": "VALID_PROGRAMS", + "retrieval_question": "What Program(s) does this contract apply to?" }, { "field_name": "AARETE_DERIVED_PROGRAM", @@ -266,19 +267,19 @@ "field_name": "REIMB_PROV_TIN", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "What TIN (Tax ID Number) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable TINs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.)." + "prompt": "The TIN (Tax ID Number) of a healthcare provider. This is a 9-digit number, often of the format XX-XXXXXXX. If there are multiple TINs that apply, provide them ALL in a comma-separated list." }, { "field_name": "REIMB_PROV_NPI", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "What NPI (National Provider Identifier) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable NPIs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.)." + "prompt": "The NPI (National Provider Identifier). This is a 10-digit number. If there are multiple NPIs that apply, provide them all in a comma-separated list." }, { "field_name": "REIMB_PROV_NAME", "relationship": "one_to_n", "field_type": "dynamic_reimb_info", - "prompt": "Identify any Provider Names that this reimbursement applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'N/A'." + "prompt": "The name of a given Provider. This may be the name of a Doctor, Hospital, Clinic, or any other provider entity - not a Payer or insurance plan. If there are multiple Provider Names that apply, provide them all in a comma-separated list." }, { "field_name": "EFFECTIVE_DT", @@ -372,7 +373,7 @@ "field_name": "PROV_TAXONOMY_CD", "relationship": "one_to_n", "field_type": "dynamic_code", - "prompt": "List any Taxonomy Codes explicitly mentioned in the text. These are 10-character alphanumeric codes used to classify providers by type, classification, and area of specialization. They may be listed as 'Taxonomy Code', 'Provider Taxonomy Code', or similar terms. Examples include '207Q00000X', '207R00000X', '207L00000X'." + "prompt": "10-character alphanumeric codes used to classify providers by type, classification, and area of specialization. They may be listed as 'Taxonomy Code', 'Provider Taxonomy Code', or similar terms. Examples include '207Q00000X', '207R00000X', '207L00000X'." }, { "field_name": "PROV_TAXONOMY_CD_DESC", @@ -386,15 +387,15 @@ "base_field": "PROV_SPECIALTY_CD_DESC", "relationship": "one_to_n", "field_type": "dynamic_code", - "prompt": "List any Specialty Codes explicitly written in the text. Specialty Codes are 2-digit codes from 01-99, or may have a letter as the first digit such as A1, C2, etc. They may be listed as 'Specialty Code', 'Provider Specialty', or similar. If no codes are explicitly written in the text, return N/A.", + "prompt": "2-digit codes from 01-99. They may also have a letter as the first digit such as A1, C2, etc. They may be listed as 'Specialty Code', 'Provider Specialty', or similar. If no codes are explicitly written in the text, return N/A.", "crosswalk": "CROSSWALK_PROV_SPECIALTY_CD" }, { "field_name": "PROV_SPECIALTY_CD_DESC", "base_field": "PROV_SPECIALTY_CD", "relationship": "one_to_n", - "field_type": "dynamic_code", - "prompt": "If no Specialty Codes are explicitly written in the text, list any Specialties that apply to the PROVIDER that are mentioned in the text. Choose ONLY from the following: {valid_values}. Do NOT list any of these values if they only apply to a SERVICE, and not explictly to a PROVIDER. If Specialty Codes are explicitly written in the text, return N/A for this field.", + "field_type": "TBD", + "prompt": "", "valid_values": "VALID_PROV_SPECIALTY", "crosswalk": "CROSSWALK_PROV_SPECIALTY_CD" }, @@ -403,15 +404,15 @@ "base_field": "PLACE_OF_SERVICE_CD_DESC", "relationship": "one_to_n", "field_type": "dynamic_code", - "prompt": "List any Place of Service Codes explicitly written in the text. Place of Service Codes are 2-digit numeric codes. They may be listed as 'POS' or 'Place of Service'. If no codes are explicitly written in the text, return N/A.", + "prompt": "2-digit numeric codes. They may be listed as 'POS' or 'Place of Service'. If no codes are explicitly written in the text, return N/A.", "crosswalk": "CROSSWALK_PLACE_OF_SERVICE_CD" }, { "field_name": "PLACE_OF_SERVICE_CD_DESC", "base_field": "PLACE_OF_SERVICE_CD", "relationship": "one_to_n", - "field_type": "dynamic_code", - "prompt": "If no Place of Service Codes are explicitly written in the text, list any Places of Services that apply. Choose ONLY from the following: {valid_values}. If Place of Service Codes are explicitly written in the text, return N/A.", + "field_type": "TBD", + "prompt": "", "valid_values": "VALID_PLACE_OF_SERVICE", "crosswalk": "CROSSWALK_PLACE_OF_SERVICE_CD" }, @@ -420,15 +421,15 @@ "base_field": "BILL_TYPE_CD_DESC", "relationship": "one_to_n", "field_type": "dynamic_code", - "prompt": "List any Bill Type Codes explicitly written in the text. Bill Type Codes are 4-character alphanumeric codes. The first character is often a leading zero, which may or may not be included. The last character may either be a digit 0-9, or the letter X. If no codes are explicitly written in the text, return N/A.", + "prompt": "4-character alphanumeric codes. The first character is often a leading zero, which may or may not be included. The last character may either be a digit 0-9, or the letter X. If no codes are explicitly written in the text, return N/A.", "crosswalk": "CROSSWALK_BILL_TYPE_CD" }, { "field_name": "BILL_TYPE_CD_DESC", "base_field": "BILL_TYPE_CD", "relationship": "one_to_n", - "field_type": "dynamic_code", - "prompt": "If no Bill Type Codes are explicitly written in the text, list any Bill Types that apply to the PROVIDER that are mentioned in the text. Choose ONLY from the following: {valid_values}. Do NOT list any of these values if they only apply to a SERVICE, and not explictly to a PROVIDER. If Bill Type Codes are explicitly written in the text, return N/A.", + "field_type": "TBD", + "prompt": "", "valid_values": "VALID_BILL_TYPE", "crosswalk": "CROSSWALK_BILL_TYPE_CD" }, @@ -561,8 +562,8 @@ { "field_name": "OUTLIER_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", - "prompt": "Identify and extract outlier payment terms. Outlier payments are payment adjustments that provide additional supplemental reimbursement on top of base payments when claims exceed predetermined cost, day, or resource thresholds for unusually high-cost or high-resource cases. These outlier payments apply only to inpatient services and never apply to outpatient services. Extract the exact language that defines conditions and thresholds for outlier payments for inpatient services. Look for specific dollar amounts, day counts, percentages, or other metrics that trigger payments for outlier payments. Also look for any services, codes, or conditions excluded from outlier payments. Look for terms like 'outlier payments', 'outlier threshold', 'unusually high-cost cases', 'extended stay supplements', 'exceptional circumstances payments', 'high-resource case adjustments'. Only extract terms that are explicitly described as outlier payments. Return the exact contract language or 'N/A'.", + "field_type": "special_case_check", + "prompt": "Outlier payments are payment adjustments that provide additional supplemental reimbursement on top of base payments when claims exceed predetermined cost, day, or resource thresholds for unusually high-cost or high-resource cases. These outlier payments apply only to inpatient services and never apply to outpatient services. Extract the exact language that defines conditions and thresholds for outlier payments for inpatient services. Look for specific dollar amounts, day counts, percentages, or other metrics that trigger payments for outlier payments. Also look for any services, codes, or conditions excluded from outlier payments. Look for terms like 'outlier payments', 'outlier threshold', 'unusually high-cost cases', 'extended stay supplements', 'exceptional circumstances payments', 'high-resource case adjustments'. Only extract terms that are explicitly described as outlier payments. Return the exact contract language or 'N/A'.", "breakout_template": "OUTLIER_BREAKOUT" }, { @@ -622,7 +623,7 @@ { "field_name": "STOP_LOSS_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", + "field_type": "special_case_check", "prompt": "Describes financial safety net provisions primarily designed to protect providers against extreme losses. May apply at individual claim or aggregate levels and typically has higher thresholds than standard outlier provisions. Look for provisions specifically labeled as 'Stop Loss' that function separately from the standard payment methodology. (e.g. compensate Provider 120% of the published Medicaid Rate once claim exceeds a payable amount of $300,000)", "breakout_template": "STOP_LOSS_BREAKOUT" }, @@ -683,7 +684,7 @@ { "field_name": "SEQUESTRATION_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", + "field_type": "special_case_check", "prompt": "Describes a government-mandated reduction in federal spending for a period of time. Look for the term 'Sequestration'.", "breakout_template": "SEQUESTRATION_BREAKOUT" }, @@ -702,7 +703,7 @@ { "field_name": "DISCOUNT_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", + "field_type": "special_case_check", "prompt": "Describes a discount applied to the payment. The description must include the term 'Discount' and should specify the discount amount, percentage, or type.", "breakout_template": "DISCOUNT_BREAKOUT" }, @@ -733,8 +734,8 @@ { "field_name": "PREMIUM_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", - "prompt": "Describes a premium to the payment. Look for the term 'Premium'.", + "field_type": "special_case_check", + "prompt": "Describes a premium to the payment. Look for the term 'Premium'. Classify as a premium only if the text explicitly indicates an increase or addition to the standard payment rate, such as 'an additional premium of X%', 'a premium rate of Y% above standard', or 'premium payment terms apply'.", "breakout_template": "PREMIUM_BREAKOUT" }, { @@ -800,7 +801,7 @@ { "field_name": "RATE_ESCALATOR_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", + "field_type": "special_case_check", "prompt": "A rate escalator is a payment mechanism where reimbursement rates automatically increase over time after an initial term or period. Extract the complete sentence or clause from the reimbursement text that describes this rate escalation mechanism, including when it starts and how it applies. Use exact text from the document.", "breakout_template": "RATE_ESCALATOR_BREAKOUT" }, @@ -819,7 +820,7 @@ { "field_name": "FACILITY_ADJUSTMENT_TERM", "relationship": "one_to_n", - "field_type": "special_case_primary", + "field_type": "special_case_check", "prompt": "Extract any language in the Exhibit that relates to Disproportionate Share Hospital ('DSH'), Direct Graduate Medical Education ('GME'), Indirect Medical Education('IME'), New Technology Add-on Payment ('NTAP'), or Uncompensated Care ('UC'). Extract the full sentence or sentences.", "breakout_template": "FACILITY_ADJUSTMENT_BREAKOUT" }, diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 76a254f..9832ec2 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -1,16 +1,9 @@ -import json -from functools import cache -import pandas as pd import src.config as config -import src.utils.llm_utils as llm_utils -from constants.delimiters import Delimiter from src.prompts.fieldset import FieldSet -from src.utils.string_utils import extract_text_from_delimiters PIPE_FORMAT_INSTRUCTIONS = "Briefly explain your answer, then enclose your final answer in |pipes|. If the requested information doesn't apply to this context, return |N/A|. If the information should exist but cannot be clearly identified, return |UNKNOWN|." - def get_cacheable_instructions(constants=None): """ Returns a dictionary of all cacheable instructions for cache warming. @@ -22,7 +15,6 @@ def get_cacheable_instructions(constants=None): Returns: dict: Dictionary mapping cache keys to instruction text """ - from src.prompts.fieldset import FieldSet cacheable_instructions = {} @@ -42,22 +34,6 @@ def get_cacheable_instructions(constants=None): except Exception as e: pass # Skip if fields not available - # SPECIAL_CASE_PRIMARY instruction - if config.FIELDS in ['all', 'one_to_n']: - try: - special_case_fields = FieldSet( - config.FIELD_JSON_PATH, field_type="special_case_primary" - ) - if constants: - field_prompts = special_case_fields.print_prompt_dict(constants) - else: - field_prompts = special_case_fields.print_prompt_dict() - - instruction, _ = SPECIAL_CASE_PRIMARY("", field_prompts) - cacheable_instructions["SPECIAL_CASE_PRIMARY"] = instruction - except Exception as e: - pass - # METHODOLOGY_BREAKOUT instruction if config.FIELDS in ['all', 'one_to_n']: try: @@ -152,8 +128,8 @@ Here is the text to analyze: {context.replace('"', "'")} - Briefly explain your answer, then put the final answer in the properly-formatted JSON dictionary. - """ +Briefly explain your answer, then put the final answer in the properly-formatted JSON dictionary. +""" def EXHIBIT_LEVEL_INSTRUCTION() -> str: @@ -251,283 +227,192 @@ Here is the text to analyze: Briefly explain your answer before putting the final answer in |pipes|. """ +def DYNAMIC_ASSIGNMENT(service_term: str, reimb_term: str, field_name: str, field_prompt: str, exhibit_text_simplified: str, page_num: str): -def REIMBURSEMENT_PRIMARY(context, fields): - instruction = f"""[OBJECTIVE] -Extract reimbursement attributes from this payer-provider contract section. -The output from this prompt will be used to populate downstream fields in the system. + # Set up as prompt, instruction to allow for Page-level caching later + prompt = f"""[OBJECTIVE] +Identify which field value is associated with a given Reimbursement Term. -[A. OUTPUT FORMAT] -Return a JSON array of objects with attributes specified in the prompt. -If no reimbursement terms are found, return the text "NO_REIMBURSEMENT_TERMS_FOUND" instead of an empty JSON array. +[INSTRUCTIONS] +You will be shown a Reimbursement Term, and the section of a contract from which that Reimbursement Term was pulled from. +Closely examine the contract text, find the Reimbursement Term, and decide which field value or values apply to the Reimbursement Term. -[B. KEY EXTRACTION RULES] +CRITICAL CONSIDERATIONS: +1. Look for what {field_name} the Program actually *applies* to. This is not the same as the Schedule used as a basis for calculation. For example, many LOBs use Medicaid as a basis for their calculation. The presence of 'Medicaid' as a rate does not on it's own guarantee that the Reimbursement Term is 'Medicaid'. -1. SERVICE_TERM Extraction: - - Default to 'Covered Services' when no clear service is specified - - Use full plan/program names when explicitly referenced (e.g. 'COMMERCIAL HMO', 'MEDICARE SELECT BENEFIT PROGRAM') - - Include all relevant codes with services (revenue codes, procedure codes, DRG codes, diagnostic codes, etc.) - - When multiple similar service categories exist in the text, include distinguishing context in the SERVICE_TERM. Don't use generic labels like 'Covered Services' for multiple different service types—instead, specify the program, payer, or scope (e.g., 'Medicare Covered Services', 'Medicaid Covered Services', 'Emergency Covered Services') so each term is unambiguously unique. - - While extracting service term from the table, use column name along with the row value to create a meaningful SERVICE_TERM. For example, if the column name is 'HOPOS' and the row value is 'A6544', the SERVICE_TERM should be 'HOPOS A6544'. If there are multiple columns, combine them with a hyphen or space (e.g., 'HOPOS A6544 - Description KNEE PROCEDURES'). +2. USE THE SERVICE_TERM AS PRIMARY CONTEXT: The Service Term shows which programs are mentioned together. If the Service Term explicitly lists specific programs (e.g., "STAR, CHIP HMO, CHIP PERINATE, and STAR+PLUS"), those are the programs that apply to reimbursement terms in that section. Do NOT assign programs that are mentioned in completely separate sections of the contract. -2. REIMB_TERM Completeness: - - Include complete context (preambles, definitions, conditional statements, unit of measure, etc.) - - Always capture full reimbursement methodology text. This may require synthesis from multiple sentences or sections. - - Include ALL "lesser of" and "not to exceed" statements and unit of measure information, even if they are not in the immediate vicinity of the reimbursement term. - - For complex payment structures (e.g., anesthesia), include all components. This may include base units, time-based units (e.g. "per 15 minutes" or "per hour"), different phases of care (e.g. labor vs. surgical time), conversion factors, and per-unit dollar amounts. - - Err on the side of inclusion in terms of language - if it seems relevant, include it. - - While extracting REIMB_TERM from the table, include all relevant columns and rows that contribute to the reimbursement term. For each row, combine the relevant column name and corresponding cell value into a single string. If a row has multiple columns, combine them into a single REIMB_TERM string (e.g., 'HOPOS A6544 - Description KNEE PROCEDURES - Amount $25.00'). +3. LOOK FOR PROXIMITY: Programs must be explicitly mentioned in the SAME sentence or paragraph as the Reimbursement Term. If a reimbursement term appears in a section that lists specific programs (e.g., "STAR, CHIP HMO, CHIP PERINATE, and STAR+PLUS: Covered Services..."), only assign those programs, NOT programs mentioned in separate sections (e.g., "Medicare Advantage (Molina Medicare Options) and MA-SNP..."). -3. Special Cases: - - Stop Loss / Outlier rates: always include these rates when present, even if they are not explicitly labeled as such. - - Ignore anything that is clearly an example, sample, or instruction for the reader. - - Ignore any liability-related language. +4. AVOID CROSS-CONTAMINATION: If the Service Term mentions Medicaid programs (STAR, CHIP, CHIPP, STAR+PLUS, etc.), do NOT assign Medicare programs (MA, MASNP) unless they are explicitly mentioned together in the same context. Similarly, if the Service Term mentions Medicare programs, do NOT assign Medicaid programs unless explicitly mentioned together. -[B1. SPATIAL PROCESSING RULES] -CRITICAL: Apply these rules to handle content that may appear in different positions relative to tables: - -1. **Global Context Scanning**: Before processing any individual table or service, scan the ENTIRE context for: - - "Lesser of" statements that apply broadly - - "Not to exceed" limitations - - Global reimbursement caps - - Percentage multipliers or conversion factors - -2. **Context Inheritance**: When extracting REIMB_TERM for any service: - - Always check if broader reimbursement rules apply to this service - - Include global limitations even if they appear distant from the specific service - - Combine specific rates with applicable global constraints - -3. **Consistent Application**: Ensure that services appearing in the same contract section receive consistent treatment of global rules, regardless of their position relative to shared text. - -4. **EXCEPTION**: When time-period columns contain different rates, this completeness rule does NOT apply. Instead, create separate entries per the time-based handling rules below. - -[C. CRITICAL INSTRUCTIONS] - -LESSER OF LANGUAGE: When a contract contains "lesser of" statements: - - **SPATIAL INDEPENDENCE**: Apply "lesser of" logic regardless of where it appears relative to specific services - - **GLOBAL CAPS**: Statements like "At no time shall Plan pay more than..." apply to ALL services in the section - - **CONSISTENT COMBINATION**: For each service, combine [specific rate] + [applicable global constraints] - -Example: If context contains "Provider shall be paid the lesser of rates in this exhibit or billed charges" followed by multiple tables, EVERY extracted rate should include "lesser of [specific rate] or billed charges" - -EXCEPTION LANGUAGE: When contracts contain exception clauses: - - Exception phrases create carve-outs from SPECIFIC "lesser of" statements only - - Exception items should combine: [their listed rate] + [any global caps that still apply] - - Global caps (like "not to exceed billed charges") remain in effect even for exception items - -CONTEXTUAL COMPLETENESS: The full payment context must be preserved: - - "The maximum compensation shall be..." + "lesser of" statements - - Introductory language + actual rates - - Complete sentence structures describing payment methodologies - - Include sentences like "Except as otherwise provided in this Compensation Schedule" as they provide important context. Since these are often found at the beginning of a section, they may be missed if not explicitly included.' - -[D. PROCESSING METHODOLOGY] -Follow this sequence: -1. **Global Scan**: BEFORE extracting any services, read the ENTIRE context and identify: - - All "lesser of" statements (regardless of position) - - All "not to exceed" caps - - All percentage multipliers - - All conversion factors - Document these constraints before proceeding to service extraction. -2. **Service Extraction**: Extract specific rates for each service/table -3. **Constraint Application**: Apply identified global constraints to each extracted rate -4. **Consistency Check**: Verify that similar services receive similar constraint treatment - -[E. TIME-BASED RATE HANDLING] -When tables contain time-period columns (Initial Term, Renewal Term, Year 1, Year 2, etc.): -- **Different rates across periods**: Create separate entries for each time period -- **Identical rates across periods**: Create single entry with combined time range -- **Apply global constraints**: Include exhibit-level "lesser of" statements consistently - -Example: Service with "90% CMS (Initial)" and "95% CMS (Renewal)" → Two separate entries with respective rates and time periods. - -[F. EXAMPLES] -[EXAMPLE 1] -Text: 'Professional Covered Services is the lesser of: (i) Allowable Charges; or (ii) the "Contracted Rate" percentage found in Table 1. Table 1 - OB and Anesthesia Services : 100% of the Payor's Oregon Health Plan DMAP fee schedule. Radiology Services : 110% of Medicare Fee Schedule' -Answer: "[{{'SERVICE_TERM' : 'OB and Anesthesia Services', 'REIMB_TERM' : 'lesser of (i) Allowable Charges; or (ii) 100% of the Payor's Oregon Health Plan DMAP fee schedule', ...}}, -{{'SERVICE_TERM' : 'Radiology Services', 'REIMB_TERM' : 'lesser of (i) Allowable Charges; or (ii) 110% of Medicare Fee Schedule', ...}}]" - -[EXAMPLE 2] -Here is an example of text with a Lesser of statement with flat dollar amounts that should also be included as a REIMB_TERM answer: -Text: 'Professional Services are paid at the lesser of: (i) Provider's Billed Charges; or (ii) the rates in Schedule A. -Schedule A - -OB Services: -- CPT 59400: Global Obstetric care with vaginal delivery $1,700.00 -- CPT 59618: Attempted vaginal delivery, resulting in cesarean $1,700.00 -Anesthesia Services: $39/unit' - -Answer: "[{{'SERVICE_TERM' : 'OB Services CPT 59400', 'REIMB_TERM' : 'lesser of (i) Provider's Billed Charges; or (ii) $1,700.00', ...}}, -{{'SERVICE_TERM' : 'OB Services CPT 59618', 'REIMB_TERM' : 'lesser of (i) Provider's Billed Charges; or (ii) $1,700.00', ...}}, -{{'SERVICE_TERM' : 'Anesthesia Services', 'REIMB_TERM' : 'lesser of (i) Provider's Billed Charges; or (ii) $39/unit', ...}}]" - -[EXAMPLE 3] -Here is an example of text with a complex payment structure that should be included as a REIMB_TERM answer: -Text: 'Anesthesia Services: -OB Epidural 00955 - Continuous epidural, labor and vaginal delivery: 5 base units, plus up to three units for first hour of labor time, plus two units for each additional hour of labor -OB Epidural 00857 - Continuous epidural, labor and C-Section: 7 base units, plus up to three units for first hour of labor time, plus two units for each additional hour of labor, plus one unit for each fifteen minute interval of surgical time if labor goes into C-Section -OB Epidural 00850 - Planned C-Section: 7 base units, plus one unit for each fifteen minute interval of surgical time' - -Answer: "[{{'SERVICE_TERM' : 'OB Epidural 00955 - Continuous epidural, labor and vaginal delivery', 'REIMB_TERM' : '5 base units, plus up to three units for first hour of labor time, plus two units for each additional hour of labor', ...}}, -{{'SERVICE_TERM' : 'OB Epidural 00857 - Continuous epidural, labor and C-Section', 'REIMB_TERM' : '7 base units, plus up to three units for first hour of labor time, plus two units for each additional hour of labor, plus one unit for each fifteen minute interval of surgical time if labor goes into C-Section', ...}}, -{{'SERVICE_TERM' : 'OB Epidural 00850 - Planned C-Section', 'REIMB_TERM' : '7 base units, plus one unit for each fifteen minute interval of surgical time', ...}}]" - -[EXAMPLE 4] -Here are some examples of language with additional context to be included in the SERVICE_TERM: -Text: 'Covered Services that are Medicare Covered Services and are not Medicaid Covered Services' -Answer: "[{{'SERVICE_TERM' : 'Covered Services that are Medicare Covered Services and are not Medicaid Covered Services', ...}}]" -Text: 'Covered Services rendered to Covered Person who are eligible for Medicare and enrolled in a MMP Plan that may include coverage for both Medicare and Medicaid Covered Services' -Answer: "[{{'SERVICE_TERM' : 'Covered Services rendered to Covered Person who are eligible for Medicare and enrolled in a MMP Plan that may include coverage for both Medicare and Medicaid Covered Services', ...}}]" - -[EXAMPLE 5] -Here is an example with a "lesser of" statement followed by exceptions, but NO additional global caps: -Text: 'Payment for services shall be based on the lesser of 75% of AHCCCS Fee for Service rates or Provider's charges less any applicable Co-Payments, with the exception of those services listed below. -{{'HOPOS': ['A6544', 'L3201'], 'Description': ['COMPRESSION STOCKING', 'ORTHOPEDIC SHOE'], 'Amount': ['$ 25.00', '$ 30.00']}}' - -Answer: "[{{'SERVICE_TERM': 'COMPRESSION STOCKING A6544', 'REIMB_TERM': '$ 25.00', ...}}, -{{'SERVICE_TERM': 'ORTHOPEDIC SHOE L3201', 'REIMB_TERM': '$ 30.00', ...}}]" - -[EXAMPLE 6 - SPATIAL CONSISTENCY] -Text: 'Payment shall be based on the lesser of 75% of AHCCCS rates or Provider's charges, with the exception of services below. At no time shall Plan pay more than Provider's billed charges. -Table 1: A6544: $25.00 -Table 2: B1234: $30.00' - -Answer: "[{{'SERVICE_TERM': 'A6544', 'REIMB_TERM': 'lesser of $25.00 and Provider's billed charges', ...}}, -{{'SERVICE_TERM': 'B1234', 'REIMB_TERM': 'lesser of $30.00 and Provider's billed charges', ...}}]" - -Note: Both services receive the same global cap treatment regardless of table position. - -[G. ANALYSIS CONTEXT]""" - - prompt = f""" -[FIELDS] -Return a JSON array of objects, each containing these attributes: -{fields} +5. "ALL PROGRAMS" AMBIGUITY - CRITICAL: If you see phrases like "for all Health Plan products & programs" in a table or section, this phrase means "all programs in THIS SECTION", NOT "all programs in the entire document". + - If the table appears AFTER a section that says "STAR, CHIP HMO, CHIP PERINATE, and STAR+PLUS" and BEFORE a section that says "Medicare Advantage", the table applies ONLY to the Medicaid programs (STAR, CHIP, CHIPP, STAR+PLUS) mentioned in that section. + - If the table appears AFTER a section that says "Medicare Advantage", the table applies ONLY to the Medicare programs mentioned in that section. + - NEVER assign programs from separate sections just because you see "all programs" - always determine which section the reimbursement term belongs to based on its position in the document. +[CONTEXT] +Here is the section of the contract. Examine this section closely and pick the correct answer or answers for the given Reimbursement Term. [START CONTEXT] -{context.replace('"', "'")} +{exhibit_text_simplified} [END CONTEXT] - -Briefly talk through your reasoning, then return a properly formatted JSON array with all extracted reimbursement items. """ + instruction = f"""[REIMBURSEMENT_TERM] +Here is the Reimbursement Term to analyze. You can find it on page {page_num} of the text. +Service Term: "{service_term}" +Reimbursement Term: "{reimb_term}" - return instruction, prompt - - - -def SPECIAL_CASE_PRIMARY(context, fields): - instruction = f""" -[OBJECTIVE] -Extract relevant excerpts from this payer-provider contract section. The output will be used to populate downstream system fields. +[{field_name} INFORMATION] +Here is the definition and valid values. Use this information to help you find the right answer. +{field_name} : {field_prompt} [OUTPUT FORMAT] -Return a JSON dictionary with attributes specified in the prompt. - -[GUIDELINES FOR EXTRACTION] -- For each attribute, extract all paragraphs or table entries that meet the specified criteria and return them as a list of strings. -- If no items meet the criteria, return an empty list []. -- Some contract sections may not contain any relevant terms — this is acceptable. - -[GUIDELINES FOR GROUPING TERMS OR SPLITTING TERMS] -- When tables are present (marked with "Table Start" and "Table End"): - - Treat each row as a complete term - - Match column headers with their corresponding row values and combine them into readable phrases while preserving exact wording. - - Include any table-level headers, footers, notes, or references (including text appearing before, within, or after the table but not part of a row) with each relevant extracted row, and append such contextual text at the end of the row entry. - - Maintain the original order, spacing, punctuation, and wording — do not rephrase or summarize. -- If multiple related terms are found that logically belong together (e.g., "Outlier Day Threshold" and "Days in Excess of Threshold" for the same service), combine them into a single entry separated by a semicolon (;). -- Ensure that combined entries maintain the exact wording of each individual term. -- If a single term contains multiple distinct concepts that should be separated (e.g., different thresholds for different services), split them into separate entries. -- Ensure that split entries maintain the exact wording of each individual concept. -- Identify text at the end of the term or within in the term that applies to all components and append them to each item as needed for clarity. +Briefly explain your answer, return a properly-formatted JSON object where the key is the field name `{field_name}` and the value is the correct answer. If there is more than one correct answer, return them in a comma-separated list. If there is no correct answer, return 'N/A' + """ + + return prompt + instruction -[GROUPING EXAMPLE] -Given these separate terms: -- "Specialized Neurological Services Outlier Day Threshold - # of Days: 15" -- "Specialized Neurological Services Days in Excess of Threshold - Per Diem: $6,369" -- "Cardiac Services Outlier Day Threshold - # of Days: 12" -- "Cardiac Services Days in Excess of Threshold - Per Diem: $6,368" +def REIMBURSEMENT_PRIMARY(context): + instruction = f"""[OBJECTIVE] +Extract Reimbursement Terms from a section of a payer-provider contract +The output from this prompt will be used to populate downstream fields in the system. -Should be combined into single entries preserving exact wording: -- "Specialized Neurological Services Outlier Day Threshold - # of Days: 15; Specialized Neurological Services Days in Excess of Threshold - Per Diem: $6,369" -- "Cardiac Services Outlier Day Threshold - # of Days: 12; Cardiac Services Days in Excess of Threshold - Per Diem: $6,368" +[REIMBURSEMENT TERMS] +A Reimbursement Term is a statement in the contract describing how a certain Service or class of Services should be paid. Reimbursement Terms ALWAYS consist of two parts: +SERVICE_TERM: Describes the medical service or procedure that is being reimbursed. It can be very general (like 'Covered Services') or very specific (like 'CPT 98765'). +REIMB_TERM: Describes the method by which the price of the Service is calculated. It can be a simple dollar value (like '$100 per unit') or a percentage of some other value (like '100% of the Medicare Fee Schedule'). -[SPLITTING EXAMPLES] +[IMPORTANT INSTRUCTIONS] +- Number one rule: ERR ON THE SIDE OF INCLUSION! +- Ensure that the COMPLETE REIMB_TERM values are captured EXACTLY as they are written in the contract. Return only EXACT word-for-word text from the contract. +- Sometimes, the SERVICE_TERM may not be explicitly written and instead simply implied. When this is the case, default the SERVICE_TERM to "Covered Services". If needed, add additional relevant detail (e.g. "Inpatient Covered Services", "Hospital Covered Services", etc.) +- NEVER return a partial sentence - include the entire sentence even if only part of the sentence is necessary. +- It's ok if the SERVICE_TERM shows up in the REIMB_TERM. Prioritize maintaining complete sentences. +- Do not leave out any relevant information from either the SERVICE_TERM or REIMB_TERM even if it is not present in the vicinity. e.g. answer "$100 per visit" rather than just "$100". If table headers provide ANY relevant information (like code type, provider type, hospital name, etc), include that in the SERVICE_TERM or REIMB_TERM as appropriate. +- Include all SERVICE_TERM and REIMB_TERM pairs mentioned, even if they are redundant or overlapping. There should be at least one entry for each SERVICE_TERM and REIMB_TERM pair mentioned. +- ALWAYS include all relevant percentages and dollar values. If there's even a slight chance that the number is relevant, include it. -Example1: +[EXCLUSIONS] +- NEVER return reimbursement terms that are labeled as Examples or appear in an EXAMPLE section. These are not actual reimbursements and should not be included in the output. +- NEVER return reimbursements from the Reciprocity Agreements section of the contract. These are not actual reimbursements and should not be included in the output. -Given this combined term and common term at the end: -"Type of Transplant | Bone Marrow Transplant Per Case Rate | Outlier Threshold -Standard - Bone Marrow Transplant Autologous | $80,000 | $105,000 -Related: Allogeneic Myeloablative (Standard) Related Blood and Marrow Transplant | $134,000 | $176,000 -Unrelated: Allogeneic Myeloablative (Standard) Unrelated Blood and Marrow Transplant | $157,000 | $206,000 -If Billed Charges exceed the Outlier Threshold, payment is the Bone Marrow Transplant Per Case Rate plus 79% of the Billed Charges that exceed the Outlier Threshold." - -Should be split into separate entries preserving exact wording and appending the common term at the end: -[ - "The per-case rate for a standard autologous bone marrow transplant is $80,000, and the outlier threshold is $105,000. If Billed Charges exceed the Outlier Threshold, payment is the Bone Marrow Transplant Per Case Rate plus 79% of the Billed Charges that exceed the Outlier Threshold.", - "The per-case rate for a standard related allogeneic myeloablative blood and marrow transplant is $134,000, and the outlier threshold is $176,000. If Billed Charges exceed the Outlier Threshold, payment is the Bone Marrow Transplant Per Case Rate plus 79% of the Billed Charges that exceed the Outlier Threshold.", - "The per-case rate for a standard unrelated allogeneic myeloablative blood and marrow transplant is $157,000, and the outlier threshold is $206,000. If Billed Charges exceed the Outlier Threshold, payment is the Bone Marrow Transplant Per Case Rate plus 79% of the Billed Charges that exceed the Outlier Threshold." -] - -Example 2: -Given this combined term: -"Specialized Neurological Services Outlier Day Threshold - # of Days: 15; Specialized Neurological Services Days in Excess of Threshold - Per Diem: $6,369; Cardiac Services Outlier Day Threshold - # of Days: 12; Cardiac Services Days in Excess of Threshold - Per Diem: $6,368" - -Should be split into separate entries preserving exact wording: -[ - "Specialized Neurological Services Outlier Day Threshold - # of Days: 15; Specialized Neurological Services Days in Excess of Threshold - Per Diem: $6,369", - "Cardiac Services Outlier Day Threshold - # of Days: 12; Cardiac Services Days in Excess of Threshold - Per Diem: $6,368" -] - -Example 3: -Given this combined term: -"DRG 1 CRANIOTOMY EX TRAUMA >17 Outlyer Threshold $37,247.14; DRG 2 CRANIOTOMY FOR TRAUMA >17 Outlyer Threshold $75,228.66; DRG 3 CRANIOTOMY <18 Outlyer Threshold $28,628.44;" -Should be split into separate entries preserving exact wording: -[ - "DRG 1 CRANIOTOMY EX TRAUMA >17 Outlyer Threshold $37,247.14", - "DRG 2 CRANIOTOMY FOR TRAUMA >17 Outlyer Threshold $75,228.66", - "DRG 3 CRANIOTOMY <18 Outlyer Threshold $28,628.44" -] - -Keep all original terms exactly as they appear - do not modify the language. +[SPECIAL CASES] +Some types of Reimbursements are special and require additional thought. +- Stop Loss and Outliers - These reimbursements usually consist of multiple rates and/or thresholds. When a Stop-Loss or Outlier is found, include the entire statement (which may be multiple sentences) as a single REIMB_TERM. +- Discounts, Escalators and Premiums - These will also include multiple rates. Capture all relevant numbers [ANALYSIS CONTEXT]""" prompt = f""" -[FIELDS] -Return a JSON dictionary with the following attributes: -{fields} - -[Start Context] +[START CONTEXT] {context.replace('"', "'")} -[End Context] +[END CONTEXT] -Briefly talk through your reasoning, then return a properly formatted JSON dictionary containing lists of all extracted reimbursement items that match the specified criteria.""" +Briefly talk through your reasoning. Then return a properly formatted JSON list of dictionaries with all extracted SERVICE_TERM AND REIMB_TERM values. +""" return instruction, prompt +def LESSER_OF_DISTRIBUTION(service_term: str, reimb_term: str, exhibit_text_simplified: str, page_num: str): + return f"""[OBJECTIVE] +Examine an Exhibit of a Payer-Provider contract. Find any overarching 'Lesser Of' Statements that apply to the given Reimbursement Term. + +['LESSER OF' LANGUAGE] +'Lesser of' language is language in a contract that specifies that a given medical Service should be paid at the lesser of two or more values. Sometimes this is written clearly, other times there will be one sentence that applies to the entire Exhibit, or a portion of the Exhibit. Your job is to find such a statement (if it exists) and determine if it applies to the given Service. + +This involves 3 steps: +[1. IDENTIFY OVERARCHING LESSER OF STATEMENTS] +There may be one or more overarching 'Lesser of' statements. Such statements will include language like: +- '...lesser of the rates listed below and ...' +- 'All services under this exhibit are subject to the lesser of...' +- 'Except as otherwise provided in this Exhibit, the Allowed Amount for Covered Services shall be the lesser of...' +- 'Payment for any service in this exhibit shall not exceed...' +- 'At no time shall Plan pay an amount that exceeds...' + +[2. DETERMINE IF THE STATEMENT APPLIES TO THE SERVICE] +If there are multiple overarching lesser of statements, it is crucial to NOT assign the Service to one that doesn't apply. +Look carefully to ensure that the intended meaning of the text is to apply a Lesser of to the given Service. + +[3. DISTRIBUTE THE SERVICE AND METHODOLOGY INTO THE LESSER OF STATEMENT] +If you find that an overarching 'Lesser of' statement applies to the given Service, you'll need to perform a concatenation or substitution to create one meaningful statement. This is the 'Lesser of' distribution. + +For example, if you see "The Services in this exhibit will be paid at the lesser of the rates listed below and 100% of Billed Charges" +and you determine that the above statement applies to the terms below: +SERVICE: "Radiology Services" +METHODOLOGY:"$100" +then you need to substitute the SERVICE and METHODOLOGY values appropriately, for a final answer of: +"Radiology Services will be paid at the lesser of $100 and 100% of Billed Charges" + +[CONTEXT] +Here is the Service and Methodology to evaluate: +SERVICE: {service_term} +METHODOLOGY: {reimb_term} + +Here is the Exhibit Text (the Service can be found on page {page_num}): +[START TEXT] +{exhibit_text_simplified} +[END TEXT] + +[OUTPUT FORMAT] +If there is no overarching lesser of Statement, return 'N/A' +If there is an overarching lesser of Statement, but it doesn't apply to the Service, return 'N/A'. +If there is an overarching lesser of Statement that applies, return the full Statement with the Service and Methodology appropriately substituted. + +Briefly explain your answer, then put your final answer in pipes. +""" + +def LESSER_OF_CHECK(service_term: str, reimb_term: str): + return f"""[OBJECTIVE] +Examine the following 'lesser of' statement, taken from the Compensation Schedule of a Payer-Provider contract. +Determine if the statement is an overarching 'lesser of' + +[INSTRUCTIONS] +Overarching 'Lesser of' statements are those which do not stand alone as a meaningful statement and instead refer to Services elsewhere in the document. +They may include language like "for all Services in this Exhibit...", "for the rates listed below...", and other language that refers specifically to Services located in other part of the Exhibit. +This is in contrast to a Stand-Alone 'lesser of' statement, which makes sense as a complete statement by itself. + +The distinction between the two options can be subtle. Here are two examples to illustrate: +[Y Example] +Service: "Covered Services" +Reimbursement: "Covered Services in this Exhibit will be paid at the lesser of a) the rates listed below or b) 100% of the Medicare fee schedule. +Answer: "Y" +This is an overarching lesser of because it makes specific references to rates elsewhere in the document. + +[N Example] +Service: "Covered Services" +Reimbursement: "Except as otherwise mentioned, Covered Services in this Exhibit will be paid at the lesser of a) Allowable Charges or b) 100% of the Medicare fee schedule" +Answer: "N" +This is not an overarching lesser of because both rates presented are present in the Reimbursement itself. There are no references to rates elsewhere (the 'except as otherwise mentioned' clause is an exclusion, which is different) + + +[CONTEXT] +Service: {service_term} +Reimbursement: {reimb_term} + +[OUTPUT FORMAT] +- If the Service and Reimbursement Term represent an overarching 'lesser of', return 'Y' +- If the Service and Reimbursement Term are a stand-alone 'lesser of', return 'N' + +Briefly explain your answer, then put your final answer in |pipes|. +""" + ##################################################################################### ################################# BREAKOUT PROMPTS ################################## ##################################################################################### -def METHODOLOGY_BREAKOUT(term, questions): +def METHODOLOGY_BREAKOUT(service_term: str, reimb_term: str, questions: str): """ This prompt is used for the fields with field_type="methodology_breakout" + any other fields we need to run based on the result of special case. All fields run by this prompt will end up becoming their own row, rather than being distributed across the rows of the exhibit """ - if isinstance(term, list): - term = f"SERVICE: {term[0]}\nMETHODOLOGY: {term[1]}" - instruction = f"""[OBJECTIVE] -Analyze a given term from a Payer-Provider contract: +Analyze a given term from a Payer-Provider contract and extract key fields. - The output should always be a JSON list/array of dictionaries, even when there is only one methodology. -For any fields that don't apply to a particular methodology, use "N/A" as the value. - -[KEY EXTRACTION RULES] +[INSTRUCTION] - Always identify **all enforceable reimbursement clauses**, whether they appear as standalone sentences or within a comparative structure. - When a reimbursement methodology includes **multiple enforceable limits**—for example, "100% of billed charges not to exceed $500"—treat **each distinct limit** as a separate reimbursement methodology. Output one dictionary for "100% of billed charges" and another for "$500 cap". - If a clause includes a **"lesser of" (or similar comparative)** structure: @@ -547,11 +432,17 @@ For any fields that don't apply to a particular methodology, use "N/A" as the va - Similarly, for provisions like stop-loss, outlier, trigger cap, discount, premium, rate escalator, different threshold values, and facility adjustments (e.g., DSH, IME, NTAP, UC, GME adjustments), separate rows should not be created as these are not considered reimbursement methodologies and are captured separately. In the case of 'RBRVS', 'RVU', or American Society of Anesthesiology ('ASA'), any dollar amount mentioned refers to a conversion factor, and not to a flat fee rate. +If AARETE_DERIVED_REIMB_METHOD is Grouper, the dollar amount refers to REIMB_CONVERSION_FACTOR. If AARETE_DERIVED_REIMB_METHOD is Flat Rate or similar, the dollar amount refers to a REIMB_FEE_RATE. + +[OUTPUT FORMAT] +The output should always be a JSON list/array of dictionaries, even when there is only one methodology. +For any fields that don't apply to a particular methodology, use "N/A" as the value. [CONTEXT]""" prompt = f"""Here is the text to analyze and respond to: -{term} +Service Term: {service_term} +Reimbursement Term: {reimb_term} [OUTPUT FORMAT] Explain your reasoning as you work through the context. After your explanation, include the heading "FINAL REIMBURSEMENT METHODOLOGY:" followed by a properly formatted JSON list with your final output.""" @@ -865,15 +756,16 @@ Briefly explain your answer, then return your final answer in JSON list format. ##################################################################################### -def ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context, question): - return f"""The following is a contract (or excerpt thereof). Answer the following question: {question} +def ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context: str, field_prompt: dict[str, str]): + return f"""The following is a contract (or excerpt thereof). +Answer the following question: {field_prompt} ## START CONTRACT TEXT ## {context} ## END CONTRACT TEXT ## - {PIPE_FORMAT_INSTRUCTIONS} - """ +Briefly explain your answer, then put the final answer in a properly-formatted JSON dictionary, where the key is the field name and value is the answer. If there is no valid answer, return "N/A". +""" def ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION() -> str: @@ -1031,6 +923,7 @@ Capture the complete hierarchy of document identifiers present in the text. The Rules for Inclusion: * Include ALL text that appears to be part of the header +* Headers MUST appear at the top of the page - do not mistakenly extract Footers, which will be found at the bottom of the page with other text before them. * Full header names and numbers/letters (e.g., "Attachment C: Commercial-Exchange") * Any other relevant subtitles and identifiers * Any words in the header that may specify what type of information is contained, such as "Compensation Schedule" or "Definitions". @@ -1084,27 +977,28 @@ Briefly explain your answer, then enclose your final answer in |pipes|. def VALIDATE_REIMBURSEMENTS_PROMPT(service_term: str, reimb_term: str) -> str: - """Validates whether a service-reimbursement pair represents a complete reimbursement methodology. + """Validates whether a Reimbursement Term represents a complete reimbursement methodology. Args: - service_term (str): The service term to validate. reimb_term (str): The reimbursement term to validate. + service_term (str): The associated service term. Returns: str: A prompt for the AI to determine if the pair is a complete reimbursement methodology. """ - return f"""Analyze service-reimbursement pair from a healthcare contract: + return f"""Analyze a REIMBURSEMENT TERM and SERVICE TERM pair from a healthcare contract: -Determine if the term represents a COMPLETE reimbursement methodology that specifies how payment will be calculated. +Determine if the pair of SERVICE TERM and REIMBURSEMENT TERM represent a COMPLETE reimbursement methodology that specifies how payment will be calculated. VALID entries - return YES: - Specific payment rates (percentages, dollar amounts) - Fee schedule references with rates - Calculation formulas - Payment limitations with amounts -- "Lesser of" or "greater of" statements with rates +- "Lesser of" or "greater of" statements with clear and specific rates - Capitation rates (PMPM, PMPY, per member costs) - Flat rates per service or episode INVALID entries - return NO unless there is VALID language as well: +- 'Empty' Reimbursements - have the same structure as a valid reimbursement but doesn't actually reference any specific rate or fee schedule (e.g. "Covered Services will be paid at the rates stated in the next section") - Service definitions without payment terms - Authorization/coverage requirements only - Processing instructions, funding arrangements, and administrative statements without specific rates @@ -1113,7 +1007,8 @@ INVALID entries - return NO unless there is VALID language as well: HARD INVALID entries - return NO even if there is VALID reimbursement language present as well: - Any reference to Coordination of Benefits or general benefit descriptions -- Any reference to Clean Claims +- Any reference to Clean Claims in SERVICE TERM only +- Audit and Time-based language describing how many claims must be paid in a certain timeframe (e.g. "... pay 85% of claims within a 30-day window") - Risk-sharing arrangements, surplus/deficit distributions, settlement payments - Performance bonuses and incentive payments, or administrative compensation - Non-reimbursement statements ("not covered", "excluded", "not payable") @@ -1127,94 +1022,21 @@ Examples: - "Contact lenses must be prior authorized" → NO (authorization only) - "Not payable per contract" → NO (no reimbursement) - "payment to IPA equal to fifty percent (50%) of the Surplus" → NO (risk-sharing distribution) -- "... shall process Clean Claims based on the then current Medicare or Medicaid Fee Schedule cost index as applicable to services rendered and according to Medicare reimbursement methodology, Including the geographic practice (GPCI), or Medicaid reimbursement methodology as applicable." → NO (definition only) -Here is the Term to analyze: -SERVICE: {service_term} +Here are the SERVICE TERM and REIMBURSEMENT TERM to analyze: +SERVICE TERM: {service_term} REIMBURSEMENT TERM: {reimb_term} - Briefly explain your reasoning, then return YES or NO in |pipes|.""" +Briefly explain your reasoning, then return YES or NO in |pipes|.""" def VALIDATE_REIMBURSEMENTS_INSTRUCTION() -> str: return ( "[OBJECTIVE]\n" - "Validate whether a service-reimbursement pair constitutes a complete reimbursement methodology." + "Validate whether a Reimbursement Term constitutes a complete reimbursement methodology." ) -def SPLIT_REIMBURSEMENTS_PROMPT(service_term: str, reimb_term: str) -> str: - """Prompt for splitting a service-reimbursement pair into its component parts. - Args: - service_term (str): The service term to split. - reimb_term (str): The reimbursement term to split. - Returns: - str: A prompt for the LLM to split the pair into its components. - """ - return f"""Analyze this service-reimbursement pair to determine if it contains multiple distinct payment methodologies that should be separated. - -Look for patterns where: -1. There's a primary payment method AND a secondary/fallback method -2. Different conditions trigger different payment calculations -3. Exception clauses create alternative payment paths -4. "For services where..." or similar language introduces a different methodology - -[PAYMENT LIMITERS]: When splitting, you may find payment limiters such as "lesser of" or "not to exceed". Make sure you distinguish between: -- GLOBAL limiters: Apply to ALL methodologies (e.g., "lesser of Provider's billed charges") -- LOCAL limiters: Apply only to specific methodologies (e.g., "not to exceed $50" for one rate only) - -Preserve global limiters in every split component. Keep local limiters only with their specific methodology. - -If you find multiple distinct methodologies, split them into separate entries. For each methodology, provide: -- The portion of the service description it applies to; for conditional splits, preserve the full original service context plus the condition -- The specific reimbursement methodology text WITH appropriate limiters preserved - -Example of CORRECT splitting with preserved limiters: -INPUT: -SERVICE: "Drugs" -REIMBURSEMENT: "lesser of Provider's billed charges or: (i) Fee schedule rate for standard drugs, (ii) WAC + 5% for specialty drugs, not to exceed $50" -CORRECT OUTPUT: -[ - {{ - "SERVICE_TERM": "standard drugs", - "REIMB_TERM": "lesser of Provider's billed charges or Fee schedule rate" - }}, - {{ - "SERVICE_TERM": "specialty drugs", - "REIMB_TERM": "lesser of Provider's billed charges or WAC + 5%, not to exceed $50" - }} -] - -Example of INCORRECT splitting (missing limiter): -INCORRECT: "Fee schedule rate" and "WAC + 5%" (missing "lesser of billed charges") - -Example of CORRECT service splitting with conditions: -INPUT: -SERVICE: "Covered Services provided under the Medicare-Medicaid Program Product which are covered by or are primary to Medicaid" -REIMBURSEMENT: "Standard rate, except for services where there is no Medicaid Rate, payment is 30% of charges" - -CORRECT OUTPUT: -[ - {{ - "SERVICE_TERM": "Covered Services provided under the Medicare-Medicaid Program Product which are covered by or are primary to Medicaid", - "REIMB_TERM": "Standard rate" - }}, - {{ - "SERVICE_TERM": "Covered Services provided under the Medicare-Medicaid Program Product which are covered by or are primary to Medicaid where there is no Medicaid Rate", - "REIMB_TERM": "30% of charges" - }} -] - -INCORRECT: "Covered Services where there is no Medicaid Rate or Medicare Rate" (missing original context) - -Return your answer as a JSON array. If no splitting is needed, return the original service and reimbursement as a single entry. - -Feel free to justify your answer, but ensure the output is a valid JSON array with each entry containing SERVICE_TERM and REIMB_TERM keys. - -SERVICE: {service_term} -REIMBURSEMENT: {reimb_term}""" - - def DATE_FIX_PROMPT(context: str) -> str: return f"""Given a date string, convert it to YYYY/MM/DD format following these rules: Input: Text potentially containing a date @@ -1289,26 +1111,33 @@ Answer YES only if the exhibit constraint adds NO additional limitation beyond w Briefly explain your reasoning, then return YES or NO in |pipes|.""" -def CARVEOUT_CHECK(service, methodology, base_reimb_answer, VALID_CARVEOUTS): - carveout_definitions = { - carveout: definition - for carveout, definition in VALID_CARVEOUTS.items() - if definition - } # Get only not None - return f"""Examine the combination of Service, Reimbursement Terms, and specific reimbursement method below, and identify which case the specific reimbursement method falls under. +def CARVEOUT_CHECK(service_term: str, reimb_term: str, carveout_definitions, special_case_definitions): + return f""" +[OBJECTIVE] +Examine the Reimbursement Term and identify which case it falls under. -Here are the Cases, with their definitions: -{carveout_definitions} +[CASE DEFINITIONS] +Here are the cases, with their definitions: +Carveout cases: {"\n\n".join([name + " : " + description for name, description in carveout_definitions.items()])} +Special cases: {"\n\n".join([name + " : " + description for name, description in special_case_definitions.items()])} +Carveout cases generally apply only to specific mentioned services, while special cases generally apply more broadly to all services in the exhibit. -Here is the Service, Reimbursement Terms, and specific reimbursement method to analyze: -Service: {service.replace('"', "'")} -Reimbursement Terms: {methodology.replace('"', "'")} -Specific reimbursement method details: {base_reimb_answer.replace('"', "'")} -Determine which case description corresponds to the specific reimbursement method. It will always match one of the listed cases. If multiple cases apply, choose the one that describes the specific reimbursement method, or choose a special case if applicable. Always return a value from the list of cases above; do not return 'N/A' or 'UNKNOWN'. Also explain why it is not a payment_carveout, default_term, or base_covered_service, if that is the case. +[REIMBURSEMENT TERM] +Here is the Reimbursement Term to analyze: +Service: {service_term} +Reimbursement Term: {reimb_term.replace('"', "'")} - {PIPE_FORMAT_INSTRUCTIONS} - """ +[INSTRUCTIONS] +- Determine which case description from the combined list of carveout cases and special cases corresponds to the specific reimbursement method. +- The Service and Reimbursement will always match one of the listed cases. +- If either the Service or Reimbursement is an explicit match with any of the cases, then that is the correct case regardless of other information (for example, if the Service says "Outlier", then choose "OUTLIER_TERM" because it is explicit). +- If multiple cases apply, choose the definition that BEST fits the Reimbursement Term. +- Always return one value from the combined list of cases above; do not return 'N/A' or 'UNKNOWN'. + +[OUTPUT FORMAT] +Briefly explain your answer, then enclose your final answer in |pipes|. +""" def CHECK_REDUNDANT_LESSER_INSTRUCTION() -> str: diff --git a/fieldExtraction/src/utils/crosswalk_utils.py b/fieldExtraction/src/utils/crosswalk_utils.py index f56b112..f62fb2c 100644 --- a/fieldExtraction/src/utils/crosswalk_utils.py +++ b/fieldExtraction/src/utils/crosswalk_utils.py @@ -4,7 +4,7 @@ import logging import src.utils.string_utils as string_utils -def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "") -> str: +def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "N/A") -> str: """Apply a crosswalk to a value Args: @@ -18,7 +18,7 @@ def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "") -> str # Handle None or blank values defensively if val is None: return default - + # Store original for fallback original_val = str(val).strip() @@ -68,5 +68,4 @@ def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "") -> str result = "|".join(final_mapped) return result else: - result = default if default else original_val - return result + return "N/A" diff --git a/fieldExtraction/src/utils/logging_utils.py b/fieldExtraction/src/utils/logging_utils.py index f34b003..63711bf 100644 --- a/fieldExtraction/src/utils/logging_utils.py +++ b/fieldExtraction/src/utils/logging_utils.py @@ -198,6 +198,9 @@ def setup_per_file_logging() -> None: console_formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") console_handler.setFormatter(console_formatter) logger.addHandler(console_handler) + + # Prevent propagation to avoid duplicate console output + logger.propagate = False # Prevent propagation to avoid duplicate console output logger.propagate = False diff --git a/fieldExtraction/tests/test_crosswalk_utils.py b/fieldExtraction/tests/test_crosswalk_utils.py index 330b151..19ab24c 100644 --- a/fieldExtraction/tests/test_crosswalk_utils.py +++ b/fieldExtraction/tests/test_crosswalk_utils.py @@ -131,8 +131,8 @@ def test_to_csv_no_metadata(tmp_path): def test_apply_crosswalk(): mapping = {"A": "1", "B": "2", "C": "3"} assert apply_crosswalk("A", mapping) == "1" - assert apply_crosswalk("D", mapping) == "D" - assert apply_crosswalk("D", mapping, default="0") == "0" + assert apply_crosswalk("D", mapping) == "N/A" + assert apply_crosswalk("D", mapping, default="0") == "N/A" def test_create_reverse_mapping(): @@ -154,11 +154,11 @@ def test_apply_crosswalk_edge_cases(): assert ( apply_crosswalk(" ", mapping, default="default") == "default" ) # Empty after strip, WITH default - assert apply_crosswalk(" ", mapping) == "" # Empty after strip, no default + assert apply_crosswalk(" ", mapping) == "N/A" # Empty after strip, no default # Test non-string inputs (should return original as string) - assert apply_crosswalk(123, mapping) == "123" # No default = return original - assert apply_crosswalk(123, mapping, default="default") == "default" # With default + assert apply_crosswalk(123, mapping) == "N/A" # No default = return original + assert apply_crosswalk(123, mapping, default="default") == "N/A" # With default # Test list-like string input assert apply_crosswalk("['A']", mapping) == "1" # String representation of list @@ -209,7 +209,7 @@ def test_apply_crosswalk_empty_results(): # Should return non-empty results only assert apply_crosswalk("A,B", mapping) == "2" - assert apply_crosswalk("A,C", mapping, default="default") == "default" + assert apply_crosswalk("A,C", mapping, default="default") == "N/A" def test_flatten_to_strings(): diff --git a/fieldExtraction/tests/test_dynamic.py b/fieldExtraction/tests/test_dynamic.py index fa36773..23b9578 100644 --- a/fieldExtraction/tests/test_dynamic.py +++ b/fieldExtraction/tests/test_dynamic.py @@ -130,36 +130,16 @@ class TestDynamicFuncsWithRealConstants(unittest.TestCase): field_to_add = self.test_field # Execute - result = dynamic_funcs.add_full_context_field( - one_to_one_fields, field_to_add, self.answer_dicts + result = dynamic_funcs.add_one_to_one_field( + one_to_one_fields, field_to_add, self.answer_dicts, self.constants ) # Assert self.assertEqual(len(result.fields), 1) self.assertEqual(result.fields[0].field_name, "TEST_FIELD") - self.assertEqual(result.fields[0].field_type, "full_context") + self.assertEqual(result.fields[0].field_type, "smart_chunked") self.assertEqual(result.fields[0].relationship, "one_to_one") - - def test_add_full_context_field_program_with_multiple_lobs(self): - """Test that program fields are not added when multiple LOBs exist.""" - # Setup - one_to_one_fields = FieldSet() - program_field = Field.from_values( - field_name="PROGRAM", - relationship="one-to-one", - field_type="exhibit_level", - prompt="What is the program?", - base_field="PROGRAM", - ) - - # Execute - use our test answer_dicts which has multiple non-empty LOBs - result = dynamic_funcs.add_full_context_field( - one_to_one_fields, program_field, self.answer_dicts - ) - - # Assert - self.assertEqual(len(result.fields), 0) # Field should not be added - + def test_add_full_context_field_program_with_single_lob(self): """Test that program fields are added when only one LOB exists.""" # Setup @@ -195,8 +175,8 @@ class TestDynamicFuncsWithRealConstants(unittest.TestCase): ] # Execute - result = dynamic_funcs.add_full_context_field( - one_to_one_fields, program_field, single_lob_answer_dicts + result = dynamic_funcs.add_one_to_one_field( + one_to_one_fields, program_field, single_lob_answer_dicts, self.constants ) # Assert @@ -241,7 +221,7 @@ class TestDynamicFuncsWithRealConstants(unittest.TestCase): mock_load_field.return_value = escalator_statement_field # Execute - result = dynamic_funcs.get_dynamic_one_to_one_fields(self.answer_dicts) + result = dynamic_funcs.get_dynamic_one_to_one_fields(self.answer_dicts, self.constants) @patch("src.investment.prompt_calls.prompt_dynamic") def test_dynamic_with_real_constants(self, mock_prompt_dynamic): diff --git a/fieldExtraction/tests/test_exhibit_chunk_mapping.py b/fieldExtraction/tests/test_exhibit_chunk_mapping.py deleted file mode 100644 index e478bcd..0000000 --- a/fieldExtraction/tests/test_exhibit_chunk_mapping.py +++ /dev/null @@ -1,43 +0,0 @@ -from src.investment.preprocessing_funcs import chunk_by_exhibit - - -def test1(): - """Test non-empty text_dict with empty exhibit_pages.""" - text_dict = {"1": "This", "2": "is", "3": "an", "4": "exhibit", "5": "test"} - exhibit_pages = [] - assert chunk_by_exhibit(text_dict, exhibit_pages) == { - "1": "1", - "2": "2", - "3": "3", - "4": "4", - "5": "5", - } - - -def test2(): - """Test when exhibit_pages contains some keys that are also in text_dict.""" - text_dict = {"1": "Welcome", "2": "to", "3": "the", "4": "exhibit"} - exhibit_pages = ["1", "2", "4"] - expected_output = {"1": "1", "2": "2", "3": "2", "4": "4"} - assert chunk_by_exhibit(text_dict, exhibit_pages) == expected_output - - -def test3(): - """Test multiple exhibit pages with transitions between them.""" - text_dict = { - "1": "Page 1", - "2": "Page 2", - "3": "Page 3", - "4": "Page 4", - "5": "Page 5", - } - exhibit_pages = ["2", "4"] - expected_output = {"1": "0", "2": "2", "3": "2", "4": "4", "5": "4"} - assert chunk_by_exhibit(text_dict, exhibit_pages) == expected_output - - -test1() - -test2() - -test3() diff --git a/fieldExtraction/tests/test_one_to_n.py b/fieldExtraction/tests/test_one_to_n.py index 1751aae..724a317 100644 --- a/fieldExtraction/tests/test_one_to_n.py +++ b/fieldExtraction/tests/test_one_to_n.py @@ -1,870 +1,698 @@ import unittest -from unittest.mock import MagicMock, call, patch - -import pandas as pd -from constants.constants import Constants +from unittest.mock import Mock, patch, MagicMock from src.investment import one_to_n_funcs -from src.prompts.fieldset import Field, FieldSet +from src.prompts.fieldset import FieldSet +from constants.constants import Constants class TestOneToNFuncs(unittest.TestCase): - """Unit tests for functions in src/investment/one_to_n_funcs.py""" + """Test suite for one_to_n_funcs.py functions.""" def setUp(self): - """Set up test fixtures before each test method.""" - # Create real Constants object + """Set up test fixtures.""" self.constants = Constants() + self.filename = "test_contract.pdf" + self.exhibit_text = "This is a sample exhibit text with reimbursement terms." + self.exhibit_header = "Exhibit A - Fee Schedule" + self.exhibit_page = "1" - # Sample filename - self.filename = "test_file.pdf" + # ==================== exhibit_level Tests ==================== - # Sample text - self.exhibit_text = "This is sample exhibit text with reimbursement details." - self.exhibit_header = "Exhibit A: Reimbursement Terms" - self.exhibit_page = "2.0" - - # Sample exhibit fields - self.exhibit_level_fields = FieldSet() - self.exhibit_level_fields.add_field( - Field.from_values( - field_name="EXHIBIT_FIELD", - relationship="one-to-one", - field_type="exhibit_level", - prompt="What is the exhibit field?", - ) + @patch('src.utils.llm_utils.invoke_claude') # Mock at the invoke_claude level + @patch('src.investment.one_to_n_funcs.dynamic_funcs.dynamic_primary') + def test_exhibit_level_basic(self, mock_dynamic_primary, mock_invoke_claude): + """Test basic exhibit level processing.""" + # Mock LLM response for dynamic_primary call + mock_invoke_claude.return_value = '{"AARETE_DERIVED_LOB": "Medicaid", "AARETE_DERIVED_NETWORK": "In-Network"}' + + # Mock dynamic primary returning updated answers and fields + mock_dynamic_fields = Mock(spec=FieldSet) + mock_dynamic_primary.return_value = ( + {"SERVICE_CATEGORY_CD": "INPATIENT", "PROV_TYPE_CD": "HOSPITAL", "AARETE_DERIVED_LOB": "Medicaid"}, + mock_dynamic_fields ) - # Sample dynamic fields - self.dynamic_primary_fields = FieldSet() - self.dynamic_primary_fields.add_field( - Field.from_values( - field_name="LOB", - relationship="one-to-many", - field_type="dynamic_primary", - prompt="What is the line of business?", - ) + exhibit_level_answers, dynamic_fields = one_to_n_funcs.exhibit_level( + self.exhibit_text, self.exhibit_header, self.exhibit_page, self.constants, self.filename ) + + # Assertions + assert exhibit_level_answers is not None + mock_invoke_claude.assert_called() - self.dynamic_code_fields = FieldSet() - self.dynamic_code_fields.add_field( - Field.from_values( - field_name="CODE_FIELD", - relationship="one-to-many", - field_type="dynamic_code", - prompt="What is the code field?", - ) - ) + # ==================== reimbursement_level Tests ==================== - self.dynamic_reimb_info_fields = FieldSet() - self.dynamic_reimb_info_fields.add_field( - Field.from_values( - field_name="REIMB_INFO_FIELD", - relationship="one-to-many", - field_type="dynamic_reimb_info", - prompt="What is the reimbursement info field?", - ) - ) - - # Sample reimbursement fields - self.reimbursement_level_fields = FieldSet() - self.reimbursement_level_fields.add_field( - Field.from_values( - field_name="REIMB_LEVEL_FIELD", - relationship="one-to-many", - field_type="reimbursement_level", - prompt="What is the reimbursement level field?", - ) - ) - - # Sample special case fields - self.special_case_primary_fields = FieldSet() - self.special_case_primary_fields.add_field( - Field.from_values( - field_name="SPECIAL_CASE_FIELD", - relationship="one-to-many", - field_type="special_case_primary", - prompt="What is the special case field?", - ) - ) - - # Sample reimbursement answer dictionaries - self.reimbursement_primary_answers = [ - { - "SERVICE_TERM": "Inpatient Services", - "REIMB_TERM": "Reimbursed at 90% of billed charges or Medicare rate, whichever is less.", - }, - { - "SERVICE_TERM": "Outpatient Services", - "REIMB_TERM": "Reimbursed at $500 per visit.", - }, + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_reimbursement_primary') + @patch('src.investment.one_to_n_funcs.clean_reimbursement_primary') + def test_reimbursement_level_with_results(self, mock_clean, mock_prompt_reimb): + """Test reimbursement level extraction with valid results.""" + mock_prompt_reimb.return_value = [ + {"SERVICE_TERM": "Lab Services", "REIMB_TERM": "100% of Medicare"}, + {"SERVICE_TERM": "Radiology", "REIMB_TERM": "90% of billed charges"} ] + mock_clean.return_value = mock_prompt_reimb.return_value - # Sample special case answers - self.special_case_primary_answers = { - "SPECIAL_CASE_FIELD": ["Special case value 1", "Special case value 2"] - } + page_text = "Lab Services will be reimbursed at 100% of Medicare." + result = one_to_n_funcs.reimbursement_level(page_text, self.constants, self.filename) - # Sample seen pairs for deduplication - self.seen_pairs = set() + self.assertEqual(len(result), 2) + mock_prompt_reimb.assert_called_once_with(page_text, self.filename) + mock_clean.assert_called_once() - def test_get_exhibit_header(self): - """Test retrieving exhibit headers.""" - # Setup - all_exhibit_headers = { - "1": "Exhibit 1 Header", - "2": "Exhibit 2 Header", - "3.0": "Exhibit 3.0 Header", - } + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_reimbursement_primary') + def test_reimbursement_level_no_results(self, mock_prompt_reimb): + """Test reimbursement level when no reimbursements found.""" + mock_prompt_reimb.return_value = None - # Test direct match - result = one_to_n_funcs.get_exhibit_header(all_exhibit_headers, "2") - self.assertEqual(result, "Exhibit 2 Header") + page_text = "This page has no reimbursement information." + result = one_to_n_funcs.reimbursement_level(page_text, self.constants, self.filename) - # Test parent header match - result = one_to_n_funcs.get_exhibit_header(all_exhibit_headers, "3.0.1") - self.assertEqual(result, "Exhibit 3.0 Header") + self.assertIsNone(result) - # Test no match - result = one_to_n_funcs.get_exhibit_header(all_exhibit_headers, "4") - self.assertEqual(result, "No Header Found") + # ==================== clean_reimbursement_primary Tests ==================== - @patch("src.investment.prompt_calls.prompt_special_case_primary") - @patch("src.investment.prompt_calls.prompt_reimbursement_primary") - @patch("src.investment.one_to_n_funcs.clean_reimbursement_primary") - def test_reimbursement_level( - self, - mock_clean_reimbursement, - mock_prompt_reimbursement, - mock_prompt_special_case, - ): - """Test the reimbursement_level function processing.""" - # Setup mocks - mock_prompt_special_case.return_value = { - "SPECIAL_CASE_FIELD": ["Special case value"] - } - mock_prompt_reimbursement.return_value = [ - {"SERVICE_TERM": "Test Service", "REIMB_TERM": "Test Reimbursement"} - ] - mock_clean_reimbursement.return_value = [ - { - "SERVICE_TERM": "Test Service", - "REIMB_TERM": "Test Reimbursement", - "CLEANED": True, - } + @patch('src.investment.one_to_n_funcs.filter_services_without_reimbursements') + def test_clean_reimbursement_primary_basic(self, mock_filter): + """Test basic cleaning of reimbursement primary answers.""" + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"}, + {"SERVICE_TERM": "X-Ray", "REIMB_TERM": "Fee schedule"} ] + mock_filter.return_value = input_answers - # Execute - reimb_answers, special_case_answers = one_to_n_funcs.reimbursement_level( - self.exhibit_text, - self.reimbursement_level_fields, - self.special_case_primary_fields, - "Lesser of statement", - self.seen_pairs, - "1.0", - self.filename, - self.constants, - ) - - # Assert - mock_prompt_special_case.assert_called_once() - mock_prompt_reimbursement.assert_called_once() - mock_clean_reimbursement.assert_called_once() - - self.assertEqual(reimb_answers[0]["CLEANED"], True) - self.assertEqual( - special_case_answers["SPECIAL_CASE_FIELD"], ["Special case value"] - ) - - @patch("src.investment.lesser_of_funcs.apply_exhibit_lesser_of") - @patch("src.investment.one_to_n_funcs.filter_services_without_reimbursements") - @patch("src.investment.one_to_n_funcs.split_compound_reimbursements") - @patch("src.investment.one_to_n_funcs.deduplicate_with_accumulator") - def test_clean_reimbursement_primary_with_lesser_of( - self, mock_deduplicate, mock_split, mock_filter, mock_apply_lesser_of - ): - """Test clean_reimbursement_primary with lesser of statement.""" - # Setup mocks - mock_split.return_value = [ - {"SERVICE_TERM": "Split Service", "REIMB_TERM": "Split Reimbursement"} - ] - mock_filter.return_value = [ - {"SERVICE_TERM": "Filtered Service", "REIMB_TERM": "Filtered Reimbursement"} - ] - mock_deduplicate.return_value = [ - {"SERVICE_TERM": "Deduped Service", "REIMB_TERM": "Deduped Reimbursement"} - ] - mock_apply_lesser_of.return_value = [ - {"SERVICE_TERM": "Final Service", "REIMB_TERM": "Final Reimbursement"} - ] - - # Execute with lesser of result = one_to_n_funcs.clean_reimbursement_primary( - self.reimbursement_primary_answers, - "Lesser of statement", - self.seen_pairs, - "1.0", - self.constants, - self.filename, + input_answers, self.constants, self.filename ) - # Assert - mock_split.assert_called_once() - mock_filter.assert_called_once() - mock_deduplicate.assert_called_once() - mock_apply_lesser_of.assert_called_once() - self.assertEqual(result[0]["SERVICE_TERM"], "Final Service") + self.assertEqual(len(result), 2) + mock_filter.assert_called_once_with(input_answers, self.filename) - @patch("src.investment.lesser_of_funcs.apply_exhibit_lesser_of") - @patch("src.investment.one_to_n_funcs.filter_services_without_reimbursements") - @patch("src.investment.one_to_n_funcs.split_compound_reimbursements") - @patch("src.investment.one_to_n_funcs.deduplicate_with_accumulator") - def test_clean_reimbursement_primary_without_lesser_of( - self, mock_deduplicate, mock_split, mock_filter, mock_apply_lesser_of - ): - """Test clean_reimbursement_primary without lesser of statement.""" - # Setup mocks - mock_split.return_value = [ - {"SERVICE_TERM": "Split Service", "REIMB_TERM": "Split Reimbursement"} - ] - mock_filter.return_value = [ - {"SERVICE_TERM": "Filtered Service", "REIMB_TERM": "Filtered Reimbursement"} - ] - mock_deduplicate.return_value = [ - {"SERVICE_TERM": "Deduped Service", "REIMB_TERM": "Deduped Reimbursement"} - ] - - # Execute without lesser of + def test_clean_reimbursement_primary_empty_input(self): + """Test cleaning with empty input.""" result = one_to_n_funcs.clean_reimbursement_primary( - self.reimbursement_primary_answers, - "", - self.seen_pairs, - "1.0", - self.constants, - self.filename, + [], self.constants, self.filename ) - - # Assert - mock_split.assert_called_once() - mock_filter.assert_called_once() - mock_deduplicate.assert_called_once() - mock_apply_lesser_of.assert_not_called() # Should not be called - self.assertEqual(result[0]["SERVICE_TERM"], "Deduped Service") - - def test_clean_reimbursement_primary_empty_answers(self): - """Test clean_reimbursement_primary with empty answers.""" - # Execute with empty answers - result = one_to_n_funcs.clean_reimbursement_primary( - [], - "Lesser of statement", - self.seen_pairs, - "1.0", - self.constants, - self.filename, - ) - - # Assert self.assertEqual(result, []) - @patch("src.investment.one_to_n_funcs.carveout_check") - @patch("src.investment.one_to_n_funcs.methodology_breakout") - @patch("src.investment.one_to_n_funcs.special_case_breakout") - def test_breakout( - self, mock_special_breakout, mock_methodology_breakout, mock_carveout_check - ): - """Test the breakout function processing flow.""" - # Setup mocks - mock_methodology_breakout.return_value = [ - { - "SERVICE_TERM": "Service", - "REIMB_TERM": "Reimb", - "METHODOLOGY": "Value", - } - ] - mock_carveout_check.return_value = [ - {"SERVICE_TERM": "Service", "REIMB_TERM": "Reimb", "CARVEOUT_CD": "N/A", "METHODOLOGY": "Value"} - ] - mock_special_breakout.return_value = { - "SPECIAL_CASE_FIELD": [ - { - "SPECIAL_CASE_FIELD": "Special value", - "SPECIAL_FIELD": "Special field value", - } - ] - } - - # Execute - reimb_answers, special_answers = one_to_n_funcs.breakout( - self.reimbursement_primary_answers, - self.special_case_primary_answers, - self.special_case_primary_fields, - self.filename, - self.constants, + def test_clean_reimbursement_primary_none_input(self): + """Test cleaning with None input.""" + result = one_to_n_funcs.clean_reimbursement_primary( + None, self.constants, self.filename ) + self.assertEqual(result, []) - # Assert - mock_carveout_check.assert_called_once() - mock_methodology_breakout.assert_called_once() - mock_special_breakout.assert_called_once() + # ==================== methodology_breakout Tests ==================== - self.assertEqual(reimb_answers[0]["METHODOLOGY"], "Value") - self.assertEqual( - special_answers["SPECIAL_CASE_FIELD"][0]["SPECIAL_FIELD"], - "Special field value", - ) - - @patch("src.investment.prompt_calls.prompt_carveout_check") - def test_carveout_check(self, mock_prompt_carveout): - """Test the carveout_check function.""" - # Setup mocks - mock_prompt_carveout.side_effect = ["N/A", "Carveout Value"] - - # Execute - result = one_to_n_funcs.carveout_check( - self.reimbursement_primary_answers, self.constants, self.filename - ) - - # Assert - self.assertEqual(mock_prompt_carveout.call_count, 2) - - # Check N/A response - self.assertEqual(result[0]["CARVEOUT_CD"], "N/A") - self.assertEqual(result[0]["CARVEOUT_IND"], "N") - self.assertEqual(result[0]["DEFAULT_IND"], "N") - - # Check carveout response - self.assertEqual(result[1]["CARVEOUT_CD"], "Carveout Value") - self.assertEqual(result[1]["CARVEOUT_IND"], "Y") - - @patch("src.investment.one_to_n_funcs.methodology_breakout_single_row") - def test_methodology_breakout(self, mock_breakout_single): - """Test the methodology_breakout function.""" - # Setup mocks - mock_breakout_single.side_effect = [ - [{"SERVICE_TERM": "Service 1", "METHODOLOGY": "Method 1"}], - [ - {"SERVICE_TERM": "Service 2", "METHODOLOGY": "Method 2A"}, - {"SERVICE_TERM": "Service 2", "METHODOLOGY": "Method 2B"}, - ], + @patch('src.investment.one_to_n_funcs.methodology_breakout_single_row') + def test_methodology_breakout_multiple_rows(self, mock_single_row): + """Test methodology breakout with multiple rows.""" + mock_single_row.side_effect = [ + [{"SERVICE_TERM": "Labs", "AARETE_DERIVED_REIMB_METHOD": "Fee Schedule"}], + [{"SERVICE_TERM": "Surgery", "AARETE_DERIVED_REIMB_METHOD": "Grouper"}] + ] + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "Medicare fee schedule"}, + {"SERVICE_TERM": "Surgery", "REIMB_TERM": "MS-DRG"} ] - # Execute result = one_to_n_funcs.methodology_breakout( - self.reimbursement_primary_answers, self.constants, self.filename + input_answers, self.constants, self.filename ) - # Assert - self.assertEqual(mock_breakout_single.call_count, 2) - self.assertEqual(len(result), 3) # One result from first call, two from second - self.assertEqual(result[0]["METHODOLOGY"], "Method 1") - self.assertEqual(result[1]["METHODOLOGY"], "Method 2A") - self.assertEqual(result[2]["METHODOLOGY"], "Method 2B") + self.assertEqual(len(result), 2) + self.assertEqual(mock_single_row.call_count, 2) - @patch("src.investment.prompt_calls.prompt_methodology_breakout") - @patch("src.investment.one_to_n_funcs.methodology_breakout_secondary") - @patch("src.investment.one_to_n_funcs.FieldSet") - def test_methodology_breakout_single_row( - self, mock_fieldset, mock_secondary, mock_prompt_methodology - ): - """Test the methodology_breakout_single_row function.""" - # Setup mocks and test data - answer_dict = { - "SERVICE_TERM": "Test Service", - "REIMB_TERM": "Test Reimbursement", + # ==================== methodology_breakout_single_row Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_methodology_breakout') + @patch('src.investment.one_to_n_funcs.methodology_breakout_secondary') + def test_methodology_breakout_single_row_basic(self, mock_secondary, mock_prompt): + """Test single row methodology breakout.""" + mock_prompt.return_value = { + "AARETE_DERIVED_REIMB_METHOD": "Fee Schedule", + "REIMB_FEE_RATE": "N/A", + "REIMB_PCT_RATE": "100" } + mock_secondary.return_value = {"FEE_SCHEDULE": "Medicare"} - # Mock the FieldSet initialization - mock_fieldset.return_value = MagicMock() + answer_dict = {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"} + + result = one_to_n_funcs.methodology_breakout_single_row( + answer_dict, self.constants, self.filename + ) - # Mock the methodology breakout prompt - mock_prompt_methodology.return_value = [ - {"AARETE_DERIVED_REIMB_METHOD": "Fee Schedule", "REIMB_PCT_RATE": "100%"} + self.assertEqual(len(result), 1) + self.assertIn("AARETE_DERIVED_REIMB_METHOD", result[0]) + self.assertIn("FEE_SCHEDULE", result[0]) + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_methodology_breakout') + def test_methodology_breakout_single_row_no_reimb_term(self, mock_prompt): + """Test single row breakout when REIMB_TERM is missing.""" + answer_dict = {"SERVICE_TERM": "Labs"} + + result = one_to_n_funcs.methodology_breakout_single_row( + answer_dict, self.constants, self.filename + ) + + self.assertEqual(result, [answer_dict]) + mock_prompt.assert_not_called() + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_methodology_breakout') + @patch('src.investment.one_to_n_funcs.methodology_breakout_secondary') + def test_methodology_breakout_single_row_list_output(self, mock_secondary, mock_prompt): + """Test single row breakout when LLM returns list of methodologies.""" + mock_prompt.return_value = [ + {"AARETE_DERIVED_REIMB_METHOD": "Fee Schedule"}, + {"AARETE_DERIVED_REIMB_METHOD": "Per Diem"} + ] + mock_secondary.side_effect = [ + {"FEE_SCHEDULE": "Medicare"}, + {"FEE_SCHEDULE": "N/A"} ] - # Mock the secondary breakout - mock_secondary.return_value = {"FS_FIELD": "FS Value", "GROUPER_FIELD": "N/A"} - - # Execute + answer_dict = {"SERVICE_TERM": "Labs", "REIMB_TERM": "Medicare or $500 per day"} + result = one_to_n_funcs.methodology_breakout_single_row( answer_dict, self.constants, self.filename ) - # Assert - self.assertEqual(len(result), 1) - self.assertEqual(result[0]["SERVICE_TERM"], "Test Service") - self.assertEqual(result[0]["REIMB_TERM"], "Test Reimbursement") - self.assertEqual(result[0]["AARETE_DERIVED_REIMB_METHOD"], "Fee Schedule") - self.assertEqual(result[0]["REIMB_PCT_RATE"], "100%") - self.assertEqual(result[0]["FS_FIELD"], "FS Value") - self.assertEqual(result[0]["GROUPER_FIELD"], "N/A") + self.assertEqual(len(result), 2) + self.assertEqual(mock_secondary.call_count, 2) - @patch("src.investment.prompt_calls.prompt_methodology_breakout") - def test_methodology_breakout_single_row_no_reimbursement( - self, mock_prompt_methodology - ): - """Test methodology_breakout_single_row with no reimbursement method.""" - # Setup test data with no reimbursement method - answer_dict = {"SERVICE_TERM": "Test Service", "REIMB_TERM": ""} + # ==================== methodology_breakout_secondary Tests ==================== - # Execute - result = one_to_n_funcs.methodology_breakout_single_row( - answer_dict, self.constants, self.filename - ) + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_fee_schedule_breakout') + def test_methodology_breakout_secondary_fee_schedule(self, mock_fs_breakout): + """Test secondary breakout for Fee Schedule methodology.""" + mock_fs_breakout.return_value = { + "FEE_SCHEDULE": "Medicare", + "FEE_SCHEDULE_VERSION": "2024" + } - # Assert - mock_prompt_methodology.assert_not_called() - self.assertEqual(len(result), 1) - self.assertEqual( - result[0], answer_dict - ) # Should return original dict unchanged - - def test_methodology_breakout_secondary_fee_schedule(self): - """Test methodology_breakout_secondary for Fee Schedule.""" - # Setup methodology_dict = {"AARETE_DERIVED_REIMB_METHOD": "Fee Schedule"} - service = "Test Service" - reimb_method = "Test Reimbursement" + + with patch('src.investment.one_to_n_funcs.FieldSet') as mock_fieldset: + mock_grouper_fields = Mock() + mock_grouper_fields.list_fields.return_value = ["GROUPER_TYPE", "DRG_WEIGHT_TYPE"] + mock_fieldset.return_value = mock_grouper_fields - # Create FieldSets - fs_fieldset = FieldSet() - fs_fieldset.add_field( - Field.from_values( - field_name="FS_FIELD", - relationship="one-to-many", - field_type="fee_schedule_breakout", - ) - ) - - grouper_fieldset = FieldSet() - grouper_fieldset.add_field( - Field.from_values( - field_name="GROUPER_FIELD", - relationship="one-to-many", - field_type="grouper_breakout", - ) - ) - - # Mock prompt calls - with patch( - "src.investment.prompt_calls.prompt_fee_schedule_breakout" - ) as mock_fs_prompt: - mock_fs_prompt.return_value = {"FS_FIELD": "FS Value"} - - # Execute result = one_to_n_funcs.methodology_breakout_secondary( - methodology_dict, - service, - reimb_method, - fs_fieldset, - grouper_fieldset, - self.constants, - self.filename, + methodology_dict, "Labs", "Medicare fee schedule", + Mock(), mock_grouper_fields, self.constants, self.filename ) - # Assert - mock_fs_prompt.assert_called_once() - self.assertEqual(result["FS_FIELD"], "FS Value") - self.assertEqual(result["GROUPER_FIELD"], "N/A") + self.assertIn("FEE_SCHEDULE", result) + self.assertEqual(result["GROUPER_TYPE"], "N/A") + self.assertEqual(result["DRG_WEIGHT_TYPE"], "N/A") - def test_methodology_breakout_secondary_grouper(self): - """Test methodology_breakout_secondary for Grouper.""" - # Setup - methodology_dict = {"AARETE_DERIVED_REIMB_METHOD": "Grouper"} - service = "Test Service" - reimb_method = "Test Reimbursement" - - # Create FieldSets - fs_fieldset = FieldSet() - fs_fieldset.add_field( - Field.from_values( - field_name="FS_FIELD", - relationship="one-to-many", - field_type="fee_schedule_breakout", - ) - ) - - grouper_fieldset = FieldSet() - grouper_fieldset.add_field( - Field.from_values( - field_name="GROUPER_FIELD", - relationship="one-to-many", - field_type="grouper_breakout", - ) - ) - - # Mock prompt calls - with patch( - "src.investment.prompt_calls.prompt_grouper_breakout" - ) as mock_grouper_prompt: - mock_grouper_prompt.return_value = {"GROUPER_FIELD": "Grouper Value"} - - # Execute - result = one_to_n_funcs.methodology_breakout_secondary( - methodology_dict, - service, - reimb_method, - fs_fieldset, - grouper_fieldset, - self.constants, - self.filename, - ) - - # Assert - mock_grouper_prompt.assert_called_once() - self.assertEqual(result["GROUPER_FIELD"], "Grouper Value") - self.assertEqual(result["FS_FIELD"], "N/A") - - def test_methodology_breakout_secondary_other(self): - """Test methodology_breakout_secondary for other methodologies.""" - # Setup - methodology_dict = {"AARETE_DERIVED_REIMB_METHOD": "Other"} - service = "Test Service" - reimb_method = "Test Reimbursement" - - # Create FieldSets - fs_fieldset = FieldSet() - fs_fieldset.add_field( - Field.from_values( - field_name="FS_FIELD", - relationship="one-to-many", - field_type="fee_schedule_breakout", - ) - ) - - grouper_fieldset = FieldSet() - grouper_fieldset.add_field( - Field.from_values( - field_name="GROUPER_FIELD", - relationship="one-to-many", - field_type="grouper_breakout", - ) - ) - - # Execute - result = one_to_n_funcs.methodology_breakout_secondary( - methodology_dict, - service, - reimb_method, - fs_fieldset, - grouper_fieldset, - self.constants, - self.filename, - ) - - # Assert - both should be N/A - self.assertEqual(result["FS_FIELD"], "N/A") - self.assertEqual(result["GROUPER_FIELD"], "N/A") - - @patch("src.investment.prompt_calls.prompt_special_case_breakout") - def test_special_case_breakout(self, mock_prompt_special): - """Test special_case_breakout function.""" - # Setup - term_field = Field.from_values( - field_name="SPECIAL_CASE_FIELD", - relationship="one-to-many", - field_type="special_case_primary", - ) - - # Mock the term field to return a breakout template - with patch.object(Field, "get_breakout_template") as mock_get_template: - mock_get_template.return_value = "METHODOLOGY_BREAKOUT" - - # Mock the prompt call - mock_prompt_special.side_effect = [ - {"BREAKOUT_FIELD": "Breakout 1"}, - {"BREAKOUT_FIELD": "Breakout 2"}, - ] - - # Add the field to the fieldset - self.special_case_primary_fields = FieldSet() - self.special_case_primary_fields.add_field(term_field) - - # Execute - result = one_to_n_funcs.special_case_breakout( - self.special_case_primary_answers, - self.special_case_primary_fields, - self.filename, - ) - - # Assert - self.assertEqual(mock_prompt_special.call_count, 2) - self.assertEqual(len(result["SPECIAL_CASE_FIELD"]), 2) - self.assertEqual( - result["SPECIAL_CASE_FIELD"][0]["BREAKOUT_FIELD"], "Breakout 1" - ) - self.assertEqual( - result["SPECIAL_CASE_FIELD"][0]["SPECIAL_CASE_FIELD"], - "Special case value 1", - ) - self.assertEqual( - result["SPECIAL_CASE_FIELD"][1]["BREAKOUT_FIELD"], "Breakout 2" - ) - self.assertEqual( - result["SPECIAL_CASE_FIELD"][1]["SPECIAL_CASE_FIELD"], - "Special case value 2", - ) - - def test_special_case_breakout_empty(self): - """Test special_case_breakout with empty answers.""" - # Execute with empty answers - result = one_to_n_funcs.special_case_breakout( - {}, self.special_case_primary_fields, self.filename - ) - - # Assert - self.assertEqual(result, {}) - - def test_deduplicate_with_accumulator(self): - """Test deduplicate_with_accumulator function.""" - # Setup - answers = [ - {"SERVICE_TERM": "Service A", "REIMB_TERM": "Reimb A"}, - {"SERVICE_TERM": "Service B", "REIMB_TERM": "Reimb B"}, - {"SERVICE_TERM": "Service A", "REIMB_TERM": "Reimb A"}, # Duplicate - ] - seen_pairs = set() - - # Execute - result = one_to_n_funcs.deduplicate_with_accumulator( - answers, seen_pairs, "1.0", self.filename - ) - - # Assert - self.assertEqual(len(result), 2) # Should have 2 unique pairs - self.assertEqual(len(seen_pairs), 2) # Should have 2 pairs in accumulator - self.assertTrue(("1", "Service A", "Reimb A") in seen_pairs) - self.assertTrue(("1", "Service B", "Reimb B") in seen_pairs) - - def test_deduplicate_with_accumulator_with_existing_pairs(self): - """Test deduplicate_with_accumulator with existing pairs in accumulator.""" - # Setup - answers = [ - {"SERVICE_TERM": "Service A", "REIMB_TERM": "Reimb A"}, - {"SERVICE_TERM": "Service B", "REIMB_TERM": "Reimb B"}, - {"SERVICE_TERM": "Service C", "REIMB_TERM": "Reimb C"}, - ] - seen_pairs = {("1", "Service A", "Reimb A")} # Already seen - - # Execute - result = one_to_n_funcs.deduplicate_with_accumulator( - answers, seen_pairs, "1.0", self.filename - ) - - # Assert - self.assertEqual(len(result), 2) # Should have 2 new unique pairs - self.assertEqual(len(seen_pairs), 3) # Should have 3 pairs in accumulator - self.assertEqual(result[0]["SERVICE_TERM"], "Service B") - self.assertEqual(result[1]["SERVICE_TERM"], "Service C") - - @patch("src.investment.prompt_calls.validate_reimbursements_for_llm") - def test_filter_services_without_reimbursements(self, mock_validate): - """Test filter_services_without_reimbursements function.""" - # Setup - answers = [ - {"SERVICE_TERM": "Valid Service", "REIMB_TERM": "Valid Reimbursement"}, - {"SERVICE_TERM": "Invalid Service", "REIMB_TERM": "Not a reimbursement"}, - ] - - # Configure mock - mock_validate.side_effect = [True, False] - - # Execute - result = one_to_n_funcs.filter_services_without_reimbursements( - answers, self.filename - ) - - # Assert - self.assertEqual(len(result), 1) - self.assertEqual(result[0]["SERVICE_TERM"], "Valid Service") - - def test_split_compound_reimbursements(self): - """Test split_compound_reimbursements function that identifies and splits compound reimbursement terms.""" - # Setup - answers = [ - # Case 1: A compound reimbursement with "lesser of" that should be split - { - "SERVICE_TERM": "Inpatient Services", - "REIMB_TERM": "Reimbursed at the lesser of 90% of billed charges or Medicare rate", - }, - # Case 2: A simple reimbursement that should not be split - {"SERVICE_TERM": "Outpatient Services", "REIMB_TERM": "100% of charges"}, - # Case 3: Another compound with "not to exceed" that should be split - { - "SERVICE_TERM": "Emergency Services", - "REIMB_TERM": "$500 per day not to exceed $2000 per admission", - }, - ] - - # Mock the compound indicators from constants - compound_indicators = [ - "lesser of", - "not to exceed", - "greater of", - "whichever is", - ] - - # Mock the LLM splitting function - with patch( - "src.investment.prompt_calls.split_compound_reimbursement_llm" - ) as mock_split: - # Define the return values for each call to the LLM - mock_split.side_effect = [ - # First call: split the "lesser of" compound - [ - { - "SERVICE_TERM": "Inpatient Services", - "REIMB_TERM": "90% of billed charges", - }, - { - "SERVICE_TERM": "Inpatient Services", - "REIMB_TERM": "Medicare rate", - }, - ], - # Third call: split the "not to exceed" compound - [ - { - "SERVICE_TERM": "Emergency Services", - "REIMB_TERM": "$500 per day", - }, - { - "SERVICE_TERM": "Emergency Services", - "REIMB_TERM": "$2000 per admission", - }, - ], - ] - - # Execute - result = one_to_n_funcs.split_compound_reimbursements( - answers, compound_indicators, self.filename - ) - - # Assert - # Verify the LLM was called twice - only for the compound reimbursements - self.assertEqual(mock_split.call_count, 2) - - # Verify the result has 5 entries: 2 from the first split, 1 unchanged, 2 from the second split - self.assertEqual(len(result), 5) - - # Check each of the split entries - self.assertEqual(result[0]["REIMB_TERM"], "90% of billed charges") - self.assertEqual(result[1]["REIMB_TERM"], "Medicare rate") - self.assertEqual(result[2]["REIMB_TERM"], "100% of charges") # Unchanged - self.assertEqual(result[3]["REIMB_TERM"], "$500 per day") - self.assertEqual(result[4]["REIMB_TERM"], "$2000 per admission") - - # Verify the SERVICE_TERM is preserved in the splits - self.assertEqual(result[0]["SERVICE_TERM"], "Inpatient Services") - self.assertEqual(result[1]["SERVICE_TERM"], "Inpatient Services") - - @patch("src.utils.llm_utils.invoke_claude") - @patch("src.utils.string_utils.universal_json_load") - def test_split_reimb_dates(self, mock_json_load, mock_invoke_claude): - """Test split_reimb_dates function.""" - # Setup - one_to_n_results = [ - {"SERVICE_TERM": "Service A", "REIMB_DATES": "01/01/2023 - 12/31/2023"}, - {"SERVICE_TERM": "Service B", "REIMB_DATES": ""}, - {"SERVICE_TERM": "Service C", "REIMB_DATES": None}, - ] - - # Mock LLM responses - mock_invoke_claude.return_value = ( - '{"start_date": "01/01/2023", "end_date": "12/31/2023"}' - ) - mock_json_load.return_value = { - "start_date": "01/01/2023", - "end_date": "12/31/2023", + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_grouper_breakout') + def test_methodology_breakout_secondary_grouper(self, mock_grouper_breakout): + """Test secondary breakout for Grouper methodology.""" + mock_grouper_breakout.return_value = { + "GROUPER_TYPE": "MS-DRG", + "DRG_WEIGHT_TYPE": "CMS" } - # Execute - result = one_to_n_funcs.split_reimb_dates(one_to_n_results, self.filename) + methodology_dict = {"AARETE_DERIVED_REIMB_METHOD": "Grouper"} + + with patch('src.investment.one_to_n_funcs.FieldSet') as mock_fieldset: + mock_fs_fields = Mock() + mock_fs_fields.list_fields.return_value = ["FEE_SCHEDULE", "FEE_SCHEDULE_VERSION"] + mock_fieldset.return_value = mock_fs_fields - # Assert - self.assertEqual(len(result), 3) + result = one_to_n_funcs.methodology_breakout_secondary( + methodology_dict, "Surgery", "MS-DRG", + mock_fs_fields, Mock(), self.constants, self.filename + ) - # Check that dates were split properly - self.assertEqual(result[0]["REIMB_EFFECTIVE_DT"], "01/01/2023") - self.assertEqual(result[0]["REIMB_TERMINATION_DT"], "12/31/2023") - self.assertFalse("REIMB_DATES" in result[0]) + self.assertIn("GROUPER_TYPE", result) + self.assertEqual(result["FEE_SCHEDULE"], "N/A") + self.assertEqual(result["FEE_SCHEDULE_VERSION"], "N/A") - # Check that empty dates were handled properly - self.assertIsNone(result[1]["REIMB_EFFECTIVE_DT"]) - self.assertIsNone(result[1]["REIMB_TERMINATION_DT"]) - self.assertFalse("REIMB_DATES" in result[1]) + def test_methodology_breakout_secondary_other_methodology(self): + """Test secondary breakout for non-FS, non-Grouper methodology.""" + methodology_dict = {"AARETE_DERIVED_REIMB_METHOD": "Per Diem"} + + with patch('src.investment.one_to_n_funcs.FieldSet') as mock_fieldset: + mock_fields = Mock() + mock_fields.list_fields.return_value = ["FEE_SCHEDULE", "GROUPER_TYPE"] + mock_fieldset.return_value = mock_fields - # Check that None dates were handled properly - self.assertIsNone(result[2]["REIMB_EFFECTIVE_DT"]) - self.assertIsNone(result[2]["REIMB_TERMINATION_DT"]) - self.assertFalse("REIMB_DATES" in result[2]) + result = one_to_n_funcs.methodology_breakout_secondary( + methodology_dict, "Inpatient", "$500 per day", + mock_fields, mock_fields, self.constants, self.filename + ) - @patch("src.utils.llm_utils.invoke_claude") - @patch("src.utils.string_utils.universal_json_load") - def test_split_reimb_dates_error_handling(self, mock_json_load, mock_invoke_claude): - """Test error handling in split_reimb_dates.""" - # Setup - one_to_n_results = [ - {"SERVICE_TERM": "Service A", "REIMB_DATES": "01/01/2023 - 12/31/2023"} + self.assertEqual(result["FEE_SCHEDULE"], "N/A") + self.assertEqual(result["GROUPER_TYPE"], "N/A") + + # ==================== carveout_and_special_case Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_carveout_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_special_case_breakout') + def test_carveout_and_special_case_base_covered(self, mock_special_breakout, mock_carveout_check): + """Test carveout check for BASE_COVERED_SERVICES.""" + mock_carveout_check.return_value = "BASE_COVERED_SERVICES" + + input_answers = [{"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"}] + + reimb_answers, special_answers = one_to_n_funcs.carveout_and_special_case( + input_answers, self.constants, self.filename + ) + + self.assertEqual(len(reimb_answers), 1) + self.assertEqual(len(special_answers), 0) + self.assertEqual(reimb_answers[0]["CARVEOUT_CD"], "BASE_COVERED_SERVICES") + self.assertEqual(reimb_answers[0]["CARVEOUT_IND"], "N") + self.assertEqual(reimb_answers[0]["DEFAULT_IND"], "N") + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_carveout_check') + def test_carveout_and_special_case_default_term(self, mock_carveout_check): + """Test carveout check for DEFAULT_TERM.""" + mock_carveout_check.return_value = "DEFAULT_TERM" + + input_answers = [{"SERVICE_TERM": "All other services", "REIMB_TERM": "As negotiated"}] + + reimb_answers, special_answers = one_to_n_funcs.carveout_and_special_case( + input_answers, self.constants, self.filename + ) + + self.assertEqual(reimb_answers[0]["CARVEOUT_CD"], "DEFAULT_TERM") + self.assertEqual(reimb_answers[0]["CARVEOUT_IND"], "N") + self.assertEqual(reimb_answers[0]["DEFAULT_IND"], "Y") + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_carveout_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_special_case_breakout') + def test_carveout_and_special_case_trigger_cap(self, mock_special_breakout, mock_carveout_check): + """Test carveout check for TRIGGER_CAP with breakout.""" + mock_carveout_check.return_value = "TRIGGER_CAP" + mock_special_breakout.return_value = {"TRIGGER_AMOUNT": "$10000", "CAP_AMOUNT": "$50000"} + + input_answers = [{"SERVICE_TERM": "Surgery", "REIMB_TERM": "Trigger at $10k, cap at $50k"}] + + reimb_answers, special_answers = one_to_n_funcs.carveout_and_special_case( + input_answers, self.constants, self.filename + ) + + self.assertEqual(reimb_answers[0]["CARVEOUT_CD"], "TRIGGER_CAP") + self.assertEqual(reimb_answers[0]["CARVEOUT_IND"], "Y") + self.assertEqual(reimb_answers[0]["DEFAULT_IND"], "N") + self.assertIn("TRIGGER_AMOUNT", reimb_answers[0]) + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_carveout_check') + def test_carveout_and_special_case_special_case_field(self, mock_carveout_check): + """Test when result is a special case field, not a carveout.""" + mock_carveout_check.return_value = "OUTLIER_TERM" + + input_answers = [{"SERVICE_TERM": "High-cost cases", "REIMB_TERM": "Outlier payment above $25,000"}] + + reimb_answers, special_answers = one_to_n_funcs.carveout_and_special_case( + input_answers, self.constants, self.filename + ) + + self.assertEqual(len(reimb_answers), 0) + self.assertEqual(len(special_answers), 1) + self.assertIn("OUTLIER_TERM", special_answers[0]) + self.assertNotIn("SERVICE_TERM", special_answers[0]) + self.assertNotIn("REIMB_TERM", special_answers[0]) + + # ==================== special_case_breakout Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_special_case_breakout') + def test_special_case_breakout_outlier(self, mock_breakout): + """Test special case breakout for outlier terms.""" + mock_breakout.return_value = { + "OUTLIER_THRESHOLD": "$25000", + "OUTLIER_PAYMENT_METHOD": "Cost-based" + } + + input_answers = [{"OUTLIER_TERM": "Outlier payment for cases exceeding $25,000"}] + + with patch('src.investment.one_to_n_funcs.FieldSet') as mock_fieldset: + mock_field = Mock() + mock_field.get_breakout_template.return_value = "OUTLIER_BREAKOUT template" + + mock_fields = Mock() + mock_fields.get_field.return_value = mock_field + mock_fieldset.return_value = mock_fields + + result = one_to_n_funcs.special_case_breakout(input_answers, self.filename) + + self.assertEqual(len(result), 1) + self.assertIn("OUTLIER_THRESHOLD", result[0]) + self.assertIn("OUTLIER_PAYMENT_METHOD", result[0]) + + # ==================== filter_services_without_reimbursements Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.validate_reimbursements_for_llm') + def test_filter_services_all_valid(self, mock_validate): + """Test filtering when all services have valid reimbursements.""" + mock_validate.return_value = True + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"}, + {"SERVICE_TERM": "X-Ray", "REIMB_TERM": "Fee schedule"} ] - # Mock error in JSON parsing - mock_invoke_claude.return_value = ( - '{"start_date": "01/01/2023", "end_date": "12/31/2023"}' + result = one_to_n_funcs.filter_services_without_reimbursements( + input_answers, self.filename ) - mock_json_load.side_effect = Exception("Test error") - # Execute - result = one_to_n_funcs.split_reimb_dates(one_to_n_results, self.filename) + self.assertEqual(len(result), 2) + self.assertEqual(mock_validate.call_count, 2) + + @patch('src.investment.one_to_n_funcs.prompt_calls.validate_reimbursements_for_llm') + def test_filter_services_some_invalid(self, mock_validate): + """Test filtering when some services lack reimbursements.""" + mock_validate.side_effect = [True, False, True] + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"}, + {"SERVICE_TERM": "Definitions", "REIMB_TERM": "Standard eyeglass lenses as defined by DMAP"}, + {"SERVICE_TERM": "Surgery", "REIMB_TERM": "MS-DRG"} + ] + + result = one_to_n_funcs.filter_services_without_reimbursements( + input_answers, self.filename + ) + + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["SERVICE_TERM"], "Labs") + self.assertEqual(result[1]["SERVICE_TERM"], "Surgery") + + @patch('src.investment.one_to_n_funcs.prompt_calls.validate_reimbursements_for_llm') + def test_filter_services_all_invalid(self, mock_validate): + """Test filtering when all services are invalid.""" + mock_validate.return_value = False + + input_answers = [ + {"SERVICE_TERM": "Admin", "REIMB_TERM": "Administrative terms apply"}, + {"SERVICE_TERM": "Definitions", "REIMB_TERM": "See glossary"} + ] + + result = one_to_n_funcs.filter_services_without_reimbursements( + input_answers, self.filename + ) + + self.assertEqual(len(result), 0) + + # ==================== split_reimb_dates Tests ==================== + + @patch('src.investment.one_to_n_funcs.llm_utils.invoke_claude') + @patch('src.investment.one_to_n_funcs.string_utils.universal_json_load') + def test_split_reimb_dates_valid_range(self, mock_json_load, mock_invoke): + """Test splitting valid date range.""" + mock_invoke.return_value = '{"start_date": "2024-01-01", "end_date": "2024-12-31"}' + mock_json_load.return_value = {"start_date": "2024-01-01", "end_date": "2024-12-31"} + + input_data = [ + {"SERVICE_TERM": "Labs", "REIMB_DATES": "January 1, 2024 through December 31, 2024"} + ] + + result = one_to_n_funcs.split_reimb_dates(input_data, self.filename) + + self.assertEqual(result[0]["REIMB_EFFECTIVE_DT"], "2024-01-01") + self.assertEqual(result[0]["REIMB_TERMINATION_DT"], "2024-12-31") + self.assertNotIn("REIMB_DATES", result[0]) + + def test_split_reimb_dates_no_dates(self): + """Test splitting when no REIMB_DATES field exists.""" + input_data = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"} + ] + + result = one_to_n_funcs.split_reimb_dates(input_data, self.filename) - # Assert - should not crash and should still remove REIMB_DATES - self.assertEqual(len(result), 1) - self.assertFalse("REIMB_DATES" in result[0]) self.assertIsNone(result[0]["REIMB_EFFECTIVE_DT"]) self.assertIsNone(result[0]["REIMB_TERMINATION_DT"]) - @patch("src.investment.prompt_calls.prompt_lob_relationship") - def test_get_lob_relationship(self, mock_prompt_lob): - """Test get_lob_relationship function.""" - # Setup - answer_dicts = [ - { - "SERVICE_TERM": "Service A", - "AARETE_DERIVED_LOB": "Commercial", - "AARETE_DERIVED_PROGRAM": "Program X", - "AARETE_DERIVED_PRODUCT": "Product Y", - }, - { - "SERVICE_TERM": "Service B", - "AARETE_DERIVED_LOB": "Medicare", - "AARETE_DERIVED_PROGRAM": "", - "AARETE_DERIVED_PRODUCT": "Product Z", - }, - { - "SERVICE_TERM": "Service C", - "AARETE_DERIVED_LOB": "", - "AARETE_DERIVED_PROGRAM": "Program W", - "AARETE_DERIVED_PRODUCT": "Product V", - }, + def test_split_reimb_dates_empty_dates(self): + """Test splitting when REIMB_DATES is empty string.""" + input_data = [ + {"SERVICE_TERM": "Labs", "REIMB_DATES": ""} ] - # Mock prompt responses - mock_prompt_lob.side_effect = ["AND", "OR", "N/A"] + result = one_to_n_funcs.split_reimb_dates(input_data, self.filename) + + self.assertIsNone(result[0]["REIMB_EFFECTIVE_DT"]) + self.assertIsNone(result[0]["REIMB_TERMINATION_DT"]) + self.assertNotIn("REIMB_DATES", result[0]) + + def test_split_reimb_dates_empty_input(self): + """Test splitting with empty input list.""" + result = one_to_n_funcs.split_reimb_dates([], self.filename) + self.assertEqual(result, []) + + def test_split_reimb_dates_none_input(self): + """Test splitting with None input.""" + result = one_to_n_funcs.split_reimb_dates(None, self.filename) + self.assertIsNone(result) + + @patch('src.investment.one_to_n_funcs.llm_utils.invoke_claude') + def test_split_reimb_dates_llm_error(self, mock_invoke): + """Test splitting when LLM call fails.""" + mock_invoke.side_effect = Exception("LLM API error") + + input_data = [ + {"SERVICE_TERM": "Labs", "REIMB_DATES": "January 1, 2024 through December 31, 2024"} + ] + + result = one_to_n_funcs.split_reimb_dates(input_data, self.filename) + + # Should still remove REIMB_DATES and initialize new fields + self.assertIsNone(result[0]["REIMB_EFFECTIVE_DT"]) + self.assertIsNone(result[0]["REIMB_TERMINATION_DT"]) + self.assertNotIn("REIMB_DATES", result[0]) + + # ==================== get_lob_relationship Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lob_relationship') + @patch('src.investment.one_to_n_funcs.string_utils.is_empty') + def test_get_lob_relationship_with_program(self, mock_is_empty, mock_prompt_lob): + """Test LOB-Program relationship extraction.""" + mock_is_empty.side_effect = [False, False, True] # LOB not empty, PROGRAM not empty, PRODUCT empty + mock_prompt_lob.return_value = "PROGRAM_SPECIFIC" + + input_data = [ + {"AARETE_DERIVED_LOB": "Medicaid", "AARETE_DERIVED_PROGRAM": "CHIP"} + ] - # Execute result = one_to_n_funcs.get_lob_relationship( - answer_dicts, self.exhibit_text, self.filename + input_data, self.exhibit_text, self.filename ) - # First dict should have both relationships - self.assertEqual(result[0]["LOB_PROGRAM_RELATIONSHIP"], "AND") - self.assertEqual(result[0]["LOB_PRODUCT_RELATIONSHIP"], "OR") + self.assertEqual(result[0]["LOB_PROGRAM_RELATIONSHIP"], "PROGRAM_SPECIFIC") + mock_prompt_lob.assert_called_once() - # Second dict should have only product relationship - self.assertFalse("LOB_PROGRAM_RELATIONSHIP" in result[1]) - self.assertEqual(result[1]["LOB_PRODUCT_RELATIONSHIP"], "N/A") + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lob_relationship') + @patch('src.investment.one_to_n_funcs.string_utils.is_empty') + def test_get_lob_relationship_with_product(self, mock_is_empty, mock_prompt_lob): + """Test LOB-Product relationship extraction.""" + mock_is_empty.side_effect = [False, True, False] # LOB not empty, PROGRAM empty, PRODUCT not empty + mock_prompt_lob.return_value = "PRODUCT_SPECIFIC" - # Third dict should have neither (empty LOB) - self.assertFalse("LOB_PROGRAM_RELATIONSHIP" in result[2]) - self.assertFalse("LOB_PRODUCT_RELATIONSHIP" in result[2]) + input_data = [ + {"AARETE_DERIVED_LOB": "Commercial", "AARETE_DERIVED_PRODUCT": "HMO"} + ] + + result = one_to_n_funcs.get_lob_relationship( + input_data, self.exhibit_text, self.filename + ) + + self.assertEqual(result[0]["LOB_PRODUCT_RELATIONSHIP"], "PRODUCT_SPECIFIC") + mock_prompt_lob.assert_called_once() + + @patch('src.investment.one_to_n_funcs.string_utils.is_empty') + def test_get_lob_relationship_no_lob(self, mock_is_empty): + """Test relationship extraction when LOB is empty.""" + mock_is_empty.return_value = True + + input_data = [ + {"AARETE_DERIVED_PROGRAM": "CHIP"} + ] + + result = one_to_n_funcs.get_lob_relationship( + input_data, self.exhibit_text, self.filename + ) + + # Should skip processing and return unchanged + self.assertEqual(result, input_data) + + # ==================== one_to_n_cleaning Tests ==================== + + @patch('src.investment.one_to_n_funcs.aarete_derived.get_crosswalk_fields') + @patch('src.investment.one_to_n_funcs.get_lob_relationship') + @patch('src.investment.one_to_n_funcs.aarete_derived.fill_na_mapping') + @patch('src.investment.one_to_n_funcs.postprocessing_funcs.update_lob_for_duals') + @patch('src.investment.one_to_n_funcs.split_reimb_dates') + def test_one_to_n_cleaning_full_pipeline(self, mock_split_dates, mock_update_duals, + mock_fill_na, mock_get_lob, mock_crosswalk): + """Test full one-to-n cleaning pipeline.""" + input_data = [{"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"}] + + mock_crosswalk.return_value = input_data + mock_get_lob.return_value = input_data + mock_fill_na.return_value = input_data + mock_update_duals.return_value = input_data + mock_split_dates.return_value = input_data + + result = one_to_n_funcs.one_to_n_cleaning( + input_data, self.exhibit_text, self.constants, self.filename + ) + + # Verify all cleaning steps were called + mock_crosswalk.assert_called_once() + mock_get_lob.assert_called_once() + mock_fill_na.assert_called_once() + mock_update_duals.assert_called_once() + mock_split_dates.assert_called_once() + + self.assertEqual(result, input_data) + + # ==================== lesser_of_distribution Tests ==================== + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_with_lesser_term_not_overarching(self, mock_lesser_dist, mock_lesser_check): + """Test lesser_of_distribution when 'less' is in term but not overarching.""" + mock_lesser_check.return_value = False # Not overarching + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "lesser of Medicare or billed charges"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["REIMB_TERM"], "lesser of Medicare or billed charges") + mock_lesser_check.assert_called_once() + mock_lesser_dist.assert_not_called() + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_with_lesser_term_overarching(self, mock_lesser_dist, mock_lesser_check): + """Test lesser_of_distribution when 'less' is in term and is overarching.""" + mock_lesser_check.return_value = True # Is overarching + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "lesser of Medicare or billed charges"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + # When overarching, the answer should not be included (filtered out) + self.assertEqual(len(result), 0) + mock_lesser_check.assert_called_once() + mock_lesser_dist.assert_not_called() + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_without_lesser_term(self, mock_lesser_dist, mock_lesser_check): + """Test lesser_of_distribution when 'less' is not in term.""" + mock_lesser_dist.return_value = "100% of Medicare fee schedule" + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["REIMB_TERM"], "100% of Medicare fee schedule") + mock_lesser_check.assert_not_called() + mock_lesser_dist.assert_called_once_with( + "Labs", "100% of Medicare", "exhibit text", "1", self.filename + ) + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_multiple_answers(self, mock_lesser_dist, mock_lesser_check): + """Test lesser_of_distribution with multiple reimbursement answers.""" + mock_lesser_check.return_value = False + mock_lesser_dist.side_effect = [ + "Updated term 1", + "Updated term 2" + ] + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "lesser of A or B"}, + {"SERVICE_TERM": "X-Ray", "REIMB_TERM": "Fee schedule"}, + {"SERVICE_TERM": "Surgery", "REIMB_TERM": "Per diem"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + self.assertEqual(len(result), 3) + # First answer has "less" but not overarching, so kept as-is + self.assertEqual(result[0]["REIMB_TERM"], "lesser of A or B") + # Second and third answers get updated via prompt_lesser_of_distribution + self.assertEqual(result[1]["REIMB_TERM"], "Updated term 1") + self.assertEqual(result[2]["REIMB_TERM"], "Updated term 2") + self.assertEqual(mock_lesser_check.call_count, 1) + self.assertEqual(mock_lesser_dist.call_count, 2) + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_case_insensitive_less(self, mock_lesser_dist, mock_lesser_check): + """Test that 'less' detection is case-insensitive.""" + mock_lesser_check.return_value = False + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "LESSER of Medicare or billed"}, + {"SERVICE_TERM": "X-Ray", "REIMB_TERM": "Less of A or B"}, + {"SERVICE_TERM": "Surgery", "REIMB_TERM": "LESS of X or Y"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + # All three should trigger lesser_of_check (case-insensitive) + self.assertEqual(mock_lesser_check.call_count, 3) + self.assertEqual(len(result), 3) + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_empty_input(self, mock_lesser_dist, mock_lesser_check): + """Test lesser_of_distribution with empty input list.""" + result = one_to_n_funcs.lesser_of_distribution( + [], "exhibit text", "1", self.constants, self.filename + ) + + self.assertEqual(result, []) + mock_lesser_check.assert_not_called() + mock_lesser_dist.assert_not_called() + + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_check') + @patch('src.investment.one_to_n_funcs.prompt_calls.prompt_lesser_of_distribution') + def test_lesser_of_distribution_return_type(self, mock_lesser_dist, mock_lesser_check): + """Test that lesser_of_distribution returns correct type structure.""" + mock_lesser_dist.return_value = "Updated term" + + input_answers = [ + {"SERVICE_TERM": "Labs", "REIMB_TERM": "100% of Medicare"} + ] + + result = one_to_n_funcs.lesser_of_distribution( + input_answers, "exhibit text", "1", self.constants, self.filename + ) + + # Verify return type is list[dict[str, str]] + self.assertIsInstance(result, list) + self.assertGreater(len(result), 0) + self.assertIsInstance(result[0], dict) + self.assertIn("SERVICE_TERM", result[0]) + self.assertIn("REIMB_TERM", result[0]) if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/fieldExtraction/tests/test_preprocessing_funcs.py b/fieldExtraction/tests/test_preprocessing_funcs.py index 241ffbc..3708752 100644 --- a/fieldExtraction/tests/test_preprocessing_funcs.py +++ b/fieldExtraction/tests/test_preprocessing_funcs.py @@ -1,7 +1,9 @@ import pytest -from src.investment.preprocessing_funcs import (clean_law_symbols, +from src.investment.preprocessing_funcs import (chunk_by_exhibit, + clean_law_symbols, filter_quick_review, - remove_page_indicators) + remove_page_indicators, + simplify_exhibit) class TestPreprocessingFuncs: @@ -80,3 +82,143 @@ class TestPreprocessingFuncs: contract, quick_review = filter_quick_review(input_dict) assert contract == expected_contract assert quick_review == expected_quick_review + + # Test cases for chunk_by_exhibit + def test_chunk_by_exhibit_basic(self): + """Test basic exhibit chunking functionality.""" + text_dict = {"1": "Page 1", "2": "Page 2", "3": "Page 3", "4": "Page 4"} + exhibit_pages = ["1", "3"] + + result = chunk_by_exhibit(text_dict, exhibit_pages) + + assert result == {"1": ["1", "2"], "3": ["3", "4"]} + + def test_chunk_by_exhibit_single_exhibit(self): + """Test chunking when all pages belong to one exhibit.""" + text_dict = {"1": "Page 1", "2": "Page 2", "3": "Page 3"} + exhibit_pages = ["1"] + + result = chunk_by_exhibit(text_dict, exhibit_pages) + + assert result == {"1": ["1", "2", "3"]} + + def test_chunk_by_exhibit_empty_exhibit_pages(self): + """Test chunking when exhibit_pages is empty - each page becomes its own exhibit.""" + text_dict = {"1": "Page 1", "2": "Page 2", "3": "Page 3"} + exhibit_pages = [] + + result = chunk_by_exhibit(text_dict, exhibit_pages) + + assert result == {"1": ["1"], "2": ["2"], "3": ["3"]} + + def test_chunk_by_exhibit_exhibit_at_start(self): + """Test chunking when exhibit starts at first page.""" + text_dict = {"1": "Page 1", "2": "Page 2", "3": "Page 3"} + exhibit_pages = ["1", "2"] + + result = chunk_by_exhibit(text_dict, exhibit_pages) + + assert result == {"1": ["1"], "2": ["2", "3"]} + + def test_chunk_by_exhibit_multiple_exhibits(self): + """Test chunking with multiple exhibits.""" + text_dict = {"1": "Page 1", "2": "Page 2", "3": "Page 3", "4": "Page 4", "5": "Page 5"} + exhibit_pages = ["1", "3", "5"] + + result = chunk_by_exhibit(text_dict, exhibit_pages) + + assert result == {"1": ["1", "2"], "3": ["3", "4"], "5": ["5"]} + + # Test cases for simplify_exhibit + def test_simplify_exhibit_no_tables(self): + """Test simplify_exhibit when there are no tables.""" + text_dict = {"1": "Page 1 content", "2": "Page 2 content"} + exhibit_page_nums = ["1", "2"] + current_page_num = "1" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + + assert result == "Page 1 content\nPage 2 content" + + def test_simplify_exhibit_with_tables_current_page_preserved(self): + """Test that current page tables are preserved while others are simplified.""" + text_dict = { + "1": "Before table-------Table Start--------Table content-------Table End--------After", + "2": "Before table-------Table Start--------Table content-------Table End--------After" + } + exhibit_page_nums = ["1", "2"] + current_page_num = "1" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + + # Current page (1) should have full table content + assert "-------Table Start--------" in result.split("\n")[0] + # Other page (2) should have table replaced with placeholder + # Note: Table counter is incremented when counting tables on current page, so page 2 gets "Table 2" + assert "Table 2" in result.split("\n")[1] + assert "-------Table Start--------" not in result.split("\n")[1] + + def test_simplify_exhibit_table_numbering_consistency(self): + """Test that table numbering is consistent across pages.""" + text_dict = { + "1": "Text-------Table Start--------Table 1-------Table End--------Text", + "2": "Text-------Table Start--------Table 2-------Table End--------Text", + "3": "Text-------Table Start--------Table 3-------Table End--------Text" + } + exhibit_page_nums = ["1", "2", "3"] + current_page_num = "2" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + result_pages = result.split("\n") + + # Page 1 (simplified) should have "Table 1" + assert "Table 1" in result_pages[0] + # Page 2 (current, preserved) should have full table content + assert "-------Table Start--------" in result_pages[1] + # Page 3 (simplified) should have "Table 3" + # (counter is 2 after page 1, then incremented to 3 when counting page 2's table) + assert "Table 3" in result_pages[2] + + def test_simplify_exhibit_multiple_tables_per_page(self): + """Test simplify_exhibit with multiple tables on a page.""" + text_dict = { + "1": "Text-------Table Start--------T1-------Table End--------Text-------Table Start--------T2-------Table End--------", + "2": "Text-------Table Start--------T3-------Table End--------" + } + exhibit_page_nums = ["1", "2"] + current_page_num = "2" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + result_pages = result.split("\n") + + # Page 1 should have both tables replaced: "Table 1" and "Table 2" + assert "Table 1" in result_pages[0] + assert "Table 2" in result_pages[0] + # Page 2 (current) should preserve table content + assert "-------Table Start--------" in result_pages[1] + + def test_simplify_exhibit_missing_page_in_text_dict(self): + """Test simplify_exhibit when a page number is missing from text_dict.""" + text_dict = {"1": "Page 1 content"} + exhibit_page_nums = ["1", "2"] + current_page_num = "1" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + + # Should handle missing page gracefully (empty string) + assert "Page 1 content" in result + assert result.count("\n") == 1 # Two pages separated by newline + + def test_simplify_exhibit_table_without_end_marker(self): + """Test simplify_exhibit when table has no end marker.""" + text_dict = { + "1": "Text-------Table Start--------Table content without end", + "2": "Page 2 content" + } + exhibit_page_nums = ["1", "2"] + current_page_num = "2" + + result = simplify_exhibit(text_dict, exhibit_page_nums, current_page_num) + + # Should still process and replace table, stopping at end of text + assert "Table 1" in result or "-------Table Start--------" in result diff --git a/fieldExtraction/tests/test_preprocessing_integration.py b/fieldExtraction/tests/test_preprocessing_integration.py deleted file mode 100644 index b845a96..0000000 --- a/fieldExtraction/tests/test_preprocessing_integration.py +++ /dev/null @@ -1,135 +0,0 @@ -import unittest -from unittest.mock import patch - -import pytest -from constants.constants import Constants -from src.investment.preprocess import clean_tables - - -class TestPreprocessingIntegration(unittest.TestCase): - - def setUp(self): - """Set up test fixtures before each test method.""" - # Define common exhibit header markers used in tests - self.EXHIBIT_HEADER_MARKERS = Constants().EXHIBIT_HEADER_MARKERS - - def test_clean_tables_simple_end_to_end(self): - """Test the complete simplified table cleaning pipeline.""" - text_dict = { - "1": "Metadata\n-------Table Start--------\nMain Table\n[['A', 'B']] + [[f'Row{i}%', str(i)] for i in range(1, 8)]\n-------Table End--------", - "2": "-------Table Start--------\n\n[['Row8', '8%'], ['Row9', '9%']]\n-------Table End--------", # Continuation - "3": "Text without tables", - } - - # This should: 1) Combine continuations, 2) Split by row count - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.txt") - - # Should have combined page 2 into page 1, then split the long table - self.assertNotIn("2", result) # Continuation removed - self.assertIn("3", result) # Non-table page preserved - # Should have split pages like "1.0", "1.1", etc. - - def test_clean_tables_small_table_gets_dot_zero_suffix(self): - """Test that pages with small tables get .0 suffix but aren't split.""" - text_dict = { - "15": "Rate Table\n-------Table Start--------\nRate Table\n[['HCPC', 'Rate'], ['T1019', '$5.00']]\n-------Table End--------" - } - - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.pdf") - - # Should get .0 suffix because it contains tables - self.assertIn("15.0", result) - self.assertNotIn("15", result) - - # Content should be preserved - self.assertIn("T1019", result["15.0"]) - - # Should not be split (small table) - chunk_keys = [k for k in result.keys() if k.startswith("15.")] - self.assertEqual(len(chunk_keys), 1) - - def test_clean_tables_continuation_gets_combined_and_dot_zero(self): - """Test that continuations get combined and result gets .0 suffix.""" - text_dict = { - "1": "Rate Table\n-------Table Start--------\nRate Table Header\n[['HCPC', 'Rate'], ['T1019', '$5.00']]\n-------Table End--------", - "2": "-------Table Start--------\n\n[['T1021', '$4.00']]\n-------Table End--------", # Continuation - } - - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.pdf") - - # Should combine and get .0 suffix since contains tables - self.assertIn("1.0", result) - self.assertNotIn("1", result) - self.assertNotIn("2", result) # Absorbed into Page 1 - - # Verify combined content - self.assertIn("T1019", result["1.0"]) - self.assertIn("T1021", result["1.0"]) - - def test_clean_tables_large_table_row_based_splitting(self): - """Test that large tables get split based on row count.""" - # Create a table with many rows that will exceed TABLE_ROW_LIMIT - large_table_rows = [["HCPC", "Rate"]] + [ - [f"T{i:04d}", f"${i}.00"] for i in range(60) - ] # 60 data rows - text_dict = { - "31": f"EXHIBIT F\n-------Table Start--------\nPCAP Rates\n{large_table_rows}\n-------Table End--------\nPost-table text" - } - - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.pdf") - - # Should create multiple chunks due to row limit (default 50) - chunk_keys = [k for k in result.keys() if k.startswith("31.")] - self.assertGreater( - len(chunk_keys), - 1, - f"Expected multiple chunks for 60 rows, got: {chunk_keys}", - ) - - # Original page should be gone - self.assertNotIn("31", result) - - # All table data should be preserved somewhere - all_content = "".join(result.values()) - self.assertIn("T0000", all_content) - self.assertIn("T0059", all_content) - self.assertIn("EXHIBIT F", all_content) - - # Post-table text should be preserved on last chunk - last_chunk = max(chunk_keys) - self.assertIn("Post-table text", result[last_chunk]) - - def test_clean_tables_no_tables_unchanged(self): - """Test that pages without tables remain unchanged.""" - text_dict = { - "5": "This is just text content with no tables.", - "6": "More text content without any table markers.", - } - - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.pdf") - - # Should remain unchanged (no .0 suffix) - assert "5" in result - assert "6" in result - assert "5.0" not in result - assert "6.0" not in result - - # Content should be identical - assert result["5"] == text_dict["5"] - assert result["6"] == text_dict["6"] - - def test_clean_tables_multiple_tables_same_page(self): - """Test that multiple tables on same page get .0 suffix but stay together.""" - text_dict = { - "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B%']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1%', '2%']]\n-------Table End--------" - } - - result = clean_tables(text_dict, self.EXHIBIT_HEADER_MARKERS, "test_file.pdf") - - # Should get .0 suffix because it contains tables - assert "1.0" in result - assert "1" not in result - - # Both tables should be present in single chunk - assert "First Table" in result["1.0"] - assert "Second Table" in result["1.0"] diff --git a/fieldExtraction/tests/test_table_funcs.py b/fieldExtraction/tests/test_table_funcs.py deleted file mode 100644 index a4742f6..0000000 --- a/fieldExtraction/tests/test_table_funcs.py +++ /dev/null @@ -1,1080 +0,0 @@ -import logging -import unittest -from unittest.mock import patch - -import pandas as pd -from constants.constants import Constants -from src.investment.table_funcs import (END_MARKER, START_MARKER, Table, - combine_continuous_tables, - convert_str_to_dataframe, - get_table_info, recreate_page, - split_large_table_with_headers, - split_tables_by_rows) - -logging.basicConfig(level=logging.INFO) - - -class TestTableFunctions(unittest.TestCase): - def setUp(self): - # New list-of-rows format - self.sample_text_dict_new = { - "1": "Some metadata\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '3%'], ['2', '4']]\n-------Table End--------", - "2": "More metadata\n-------Table Start--------\n\n[['100', '200'], ['5$', '7$'], ['6', '8']]\n-------Table End--------", - } - # Old dict format for backward compatibility testing - self.sample_text_dict_old = { - "1": "Some metadata\n-------Table Start--------\nTable 1\n{'A': ['1', '2$'], 'B': ['3$', '4']}\n-------Table End--------", - "2": "More metadata\n-------Table Start--------\n\n{'A': ['5', '6%'], 'B': ['7%', '8']}\n-------Table End--------", - } - # Multiple tables on one page - self.multi_table_text = { - "1": "Page metadata\n-------Table Start--------\nFirst Table\n[['Col1', 'Col2'], ['A%', 'B'], ['C', 'D']]\n-------Table End--------\n\nSome text\n-------Table Start--------\nSecond Table\n[['X', 'Y'], ['1', '2']]\n-------Table End--------" - } - # Long table for splitting tests - self.long_table_text = { - "1": "Long table metadata\n-------Table Start--------\nLong Table\n[['Name%', 'Value'], ['Row1', '1'], ['Row2', '2'], ['Row3', '3'], ['Row4', '4'], ['Row5', '5']]\n-------Table End--------" - } - - self.EXHIBIT_HEADER_MARKERS = Constants().EXHIBIT_HEADER_MARKERS - - def test_convert_str_to_dataframe_new_format(self): - """Test conversion from new list-of-rows format.""" - table_str = "[['A', 'B'], ['1', '3'], ['2', '4']]" - df = convert_str_to_dataframe(table_str) - - self.assertIsInstance(df, pd.DataFrame) - self.assertEqual(df.shape, (2, 2)) # 2 rows, 2 columns - self.assertEqual(df.columns[0], "A") - self.assertEqual(df.iloc[0, 0], "1") - self.assertEqual(df.iloc[1, 0], "2") - self.assertEqual(df.iloc[0, 1], "3") - self.assertTrue( - df.dtypes.eq("object").all() - ) # All columns should be of type str (object) - - def test_convert_str_to_dataframe_old_format(self): - """Test conversion from old dict format with deprecation warning.""" - table_str = "{'A': ['1', '2'], 'B': ['3', '4']}" - with self.assertLogs(level="INFO") as log: - df = convert_str_to_dataframe(table_str) - - # Should warn about old format - self.assertTrue( - any("old table (dict) format detected" in message for message in log.output) - ) - - self.assertIsInstance(df, pd.DataFrame) - self.assertEqual(df.shape, (2, 2)) # 2 rows, 2 columns - self.assertTrue( - df.dtypes.eq("object").all() - ) # All columns should be of type str (object) - self.assertEqual(df.iloc[0, 0], "1") - self.assertEqual(df.iloc[0, 1], "3") - self.assertEqual(df.iloc[1, 0], "2") - self.assertEqual(df.iloc[1, 1], "4") - - def test_convert_str_to_dataframe_invalid_format(self): - """Test handling of invalid formats.""" - invalid_str = "not a valid table format" - df = convert_str_to_dataframe(invalid_str) - - self.assertIsInstance(df, pd.DataFrame) - self.assertTrue(df.empty) - - def test_table_init_new_format(self): - """Test Table initialization with new list-of-rows format.""" - table_data = [["A", "B"], ["1", "3"], ["2", "4"]] - table = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=table_data, - start_index=0, - end_index=100, - ) - - self.assertEqual(table.page_num, "1") - self.assertEqual(table.metadata, "metadata") - self.assertEqual(table.header, "Table 1") - self.assertEqual(table.table.shape, (2, 2)) # 2 rows, 2 columns - self.assertEqual(list(table.table.columns), ["A", "B"]) - self.assertEqual(table.table.iloc[0, 0], "1") - - def test_table_init_dataframe(self): - """Test Table initialization with a DataFrame.""" - table_data = pd.DataFrame([["A", "B"], ["1", "3"]], dtype=str) - table = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=table_data, - start_index=0, - end_index=100, - ) - self.assertEqual(table.table.shape, (2, 2)) # 2 rows, 2 columns - self.assertTrue( - table.table.dtypes.eq("object").all() - ) # All columns should be of type str (object) - - def test_table_init_column_mismatch_padding(self): - """Test table initialization handles column count mismatches by padding.""" - # Headers have 3 columns, but data rows have different counts - table_data = [ - ["A", "B", "C"], # 3 columns (header) - ["1", "2"], # 2 columns - should be padded - ["3", "4", "5", "6"], # 4 columns - determines max_cols = 4 - ["7"], # 1 column - should be padded - ] - table = Table( - page_num="1", - metadata="metadata", - header="Test Table", - table_data=table_data, - start_index=0, - end_index=100, - ) - - # Should have 4 columns (max from all rows) and 3 data rows - self.assertEqual(table.table.shape, (3, 4)) - self.assertEqual( - list(table.table.columns), ["A", "B", "C", ""] - ) # Header padded too - - # Check first row (padded to 4 columns) - self.assertEqual(table.table.iloc[0, 0], "1") - self.assertEqual(table.table.iloc[0, 1], "2") - self.assertEqual(table.table.iloc[0, 2], "") # Padded - self.assertEqual(table.table.iloc[0, 3], "") # Padded - - # Check second row (kept at 4 columns) - self.assertEqual(table.table.iloc[1, 0], "3") - self.assertEqual(table.table.iloc[1, 1], "4") - self.assertEqual(table.table.iloc[1, 2], "5") - self.assertEqual(table.table.iloc[1, 3], "6") # 4th column preserved - - # Check third row (heavily padded) - self.assertEqual(table.table.iloc[2, 0], "7") - self.assertEqual(table.table.iloc[2, 1], "") # Padded - self.assertEqual(table.table.iloc[2, 2], "") # Padded - self.assertEqual(table.table.iloc[2, 3], "") # Padded - - def test_table_init_continuation_column_handling(self): - """Test continuation table handles column mismatches with generic column names.""" - # Continuation table - no header row, just data with varying lengths - table_data = [ - ["1", "2"], # 2 columns - ["3", "4", "5"], # 3 columns - ["6"], # 1 column - ] - table = Table( - page_num="2", - metadata="", - header="", - table_data=table_data, - start_index=0, - end_index=100, - is_continuation=True, - ) - - # Should use max column count (3) and generate generic column names - self.assertEqual(table.table.shape, (3, 3)) - self.assertEqual(list(table.table.columns), ["col_0", "col_1", "col_2"]) - - # Check padding/truncation - self.assertEqual(table.table.iloc[0, 0], "1") - self.assertEqual(table.table.iloc[0, 1], "2") - self.assertEqual(table.table.iloc[0, 2], "") # Padded - - self.assertEqual(table.table.iloc[1, 0], "3") - self.assertEqual(table.table.iloc[1, 1], "4") - self.assertEqual(table.table.iloc[1, 2], "5") - - self.assertEqual(table.table.iloc[2, 0], "6") - self.assertEqual(table.table.iloc[2, 1], "") # Padded - self.assertEqual(table.table.iloc[2, 2], "") # Padded - - def test_combine_continuous_tables_simple(self): - """Test the simplified continuous table combination logic.""" - # Create test data with continuation table - text_dict = { - "1": "Metadata 1\n-------Table Start--------\nMain Table\n[['Name', 'Value'], ['A', '1%'], ['B', '2']]\n-------Table End--------", - "2": "-------Table Start--------\n\n[['C', '3$'], ['D', '4']]\n-------Table End--------", # Continuation (empty header) - "3": "Metadata 3\n-------Table Start--------\nAnother Table\n[['X', 'Y'], ['Z', '5%']]\n-------Table End--------", - } - - # Debug: Let's see what get_table_info thinks about these tables - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - print(f"Table dict keys: {list(table_dict.keys())}") - for page_num, tables in table_dict.items(): - for i, table in enumerate(tables): - print( - f"Page {page_num}, Table {i}: is_continuation={table.is_continuation}, header='{table.header}'" - ) - - result = combine_continuous_tables(text_dict, table_dict) - print(f"Result keys: {list(result.keys())}") - - # Page 2 should be removed (combined into page 1) - self.assertNotIn("2", result) - self.assertIn("1", result) - self.assertIn("3", result) - - # Page 1 should now contain the combined table - combined_table_dict = get_table_info( - {"1": result["1"]}, self.EXHIBIT_HEADER_MARKERS - ) - combined_table = combined_table_dict["1"][0] - - # Should have 4 rows total (2 from main + 2 from continuation) - self.assertEqual(combined_table.table.shape, (4, 2)) - self.assertEqual(combined_table.table.iloc[0, 0], "A") # First table data - self.assertEqual(combined_table.table.iloc[2, 0], "C") # Continuation data - self.assertEqual(combined_table.table.iloc[3, 0], "D") # Continuation data - - def test_combine_continuous_tables_multiple_tables_per_page(self): - """Test that continuation tables combine with the LAST table on previous page, not the first.""" - # Page 1 has TWO tables: Table A (complete) and Table B (continues on page 2) - text_dict = { - "1": ( - "Metadata 1\n" - "-------Table Start--------\nTable A (Complete)\n[['Col1', 'Col2'], ['A1%', 'A2'], ['A3', 'A4']]\n-------Table End--------\n" - "Some text between tables\n" - "-------Table Start--------\nTable B (Continues)\n[['Name', 'Value'], ['B1%', '1'], ['B2', '2']]\n-------Table End--------" - ), - "2": ( - "-------Table Start--------\n\n[['B3', '3'], ['B4', '4%']]\n-------Table End--------" # Continuation of Table B - ), - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - - # Verify setup: Page 1 should have 2 tables, Page 2 should have 1 continuation table - self.assertEqual(len(table_dict["1"]), 2) - self.assertEqual(len(table_dict["2"]), 1) - self.assertFalse( - table_dict["1"][0].is_continuation - ) # Table A - not continuation - self.assertFalse( - table_dict["1"][1].is_continuation - ) # Table B - not continuation - self.assertTrue( - table_dict["2"][0].is_continuation - ) # Table B continuation - is continuation - - # Get original row counts before combining - table_a_original_rows = table_dict["1"][0].table.shape[0] # Should be 2 rows - table_b_original_rows = table_dict["1"][1].table.shape[0] # Should be 2 rows - continuation_rows = table_dict["2"][0].table.shape[0] # Should be 2 rows - - # Run the combination - result = combine_continuous_tables(text_dict, table_dict) - - # Page 2 should be removed (combined into page 1) - self.assertNotIn("2", result) - self.assertIn("1", result) - - # Parse the combined result to verify tables - combined_table_dict = get_table_info( - {"1": result["1"]}, self.EXHIBIT_HEADER_MARKERS - ) - combined_tables = combined_table_dict["1"] - - # Should still have 2 tables on page 1 - self.assertEqual(len(combined_tables), 2) - - # Table A (first table) should be unchanged - table_a_after = combined_tables[0] - self.assertEqual( - table_a_after.table.shape[0], table_a_original_rows - ) # Same row count - self.assertEqual(table_a_after.header, "Table A (Complete)") - self.assertEqual( - table_a_after.table.iloc[0, 0], "A1%" - ) # Original data preserved - - # Table B (second table) should now include the continuation data - table_b_after = combined_tables[1] - expected_combined_rows = table_b_original_rows + continuation_rows # 2 + 2 = 4 - self.assertEqual(table_b_after.table.shape[0], expected_combined_rows) - self.assertEqual(table_b_after.header, "Table B (Continues)") - - # Verify Table B has both original and continuation data - self.assertEqual(table_b_after.table.iloc[0, 0], "B1%") # Original data - self.assertEqual(table_b_after.table.iloc[1, 0], "B2") # Original data - self.assertEqual(table_b_after.table.iloc[2, 0], "B3") # Continuation data - self.assertEqual(table_b_after.table.iloc[3, 0], "B4") # Continuation data - - def test_combine_continuous_tables_wrong_combination_scenario(self): - """Test edge case: what if we accidentally used [0] instead of [-1]?""" - # This test documents what WOULD happen with the wrong indexing - # Page 1: Table A + Table B (continues) - # Page 2: Table B continuation - # If we used [0], continuation would wrongly combine with Table A - - text_dict = { - "1": ( - "-------Table Start--------\nTable A\n[['X', 'Y'], ['X1%', 'Y1']]\n-------Table End--------\n" - "-------Table Start--------\nTable B\n[['Name', 'Value'], ['B1%', '1']]\n-------Table End--------" - ), - "2": "-------Table Start--------\n\n[['B2', '2%']]\n-------Table End--------", # Should combine with Table B, not Table A - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - result = combine_continuous_tables(text_dict, table_dict) - - # Get the result and verify correct combination - combined_table_dict = get_table_info( - {"1": result["1"]}, self.EXHIBIT_HEADER_MARKERS - ) - - # Table A (index 0) should have 1 row still - NOT combined with continuation - table_a = combined_table_dict["1"][0] - self.assertEqual(table_a.table.shape[0], 1) # Still just 1 row - self.assertEqual(table_a.table.iloc[0, 0], "X1%") # Original data only - - # Table B (index 1, which is -1) should have 2 rows - correctly combined - table_b = combined_table_dict["1"][1] - self.assertEqual(table_b.table.shape[0], 2) # 1 original + 1 continuation = 2 - self.assertEqual(table_b.table.iloc[0, 0], "B1%") # Original - self.assertEqual(table_b.table.iloc[1, 0], "B2") # Continuation - - def test_table_combine_same_header(self): - """Test combining two tables with the same header.""" - table1 = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=[["Column 1", "Column 2"], ["A", "B"], ["1", "2"]], - start_index=0, - end_index=100, - ) - table2 = Table( - page_num="1", - metadata="metadata", - header="Table 2", - table_data=[["Column 1", "Column 2"], ["C", "D"], ["3", "4"]], - start_index=0, - end_index=100, - ) - - original_shape = table1.table.shape - table1.combine(table2) - self.assertEqual( - table1.table.shape[0], original_shape[0] + table2.table.shape[0] - ) # Rows should be combined - self.assertEqual(table1.table.shape, (4, 2)) # 4 rows (2 + 2 rows), 2 columns - self.assertEqual(table1.table.iloc[0, 0], "A") - self.assertEqual( - table1.table.iloc[2, 0], "C" - ) # Check if second table's data is appended correctly - self.assertEqual(table1.table.iloc[3, 1], "4") # Last row of second table - - def test_table_combine_different_headers(self): - """Test combining two tables with different headers (the second table is assumed to be a continuation in this case). - We will pass in table_data as STRING for both we use convert_str_to_dataframe and - check that `is_continuation` is getting instantiated correctly. - """ - - table1 = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data="[['Column 1', 'Column 2'], ['A', 'B'], ['1', '2']]", - start_index=0, - end_index=100, - ) - - # Continuation table with empty header - table2 = Table( - page_num="1", - metadata="metadata", - header="", - table_data="[['C', 'D'], ['3', '4'], ['E', 'F']]", # Pass as string! - start_index=0, - end_index=100, - is_continuation=True, # Mark as continuation - ) - - original_shape = table1.table.shape - table1.combine(table2) - - print(table1.table) - self.assertEqual( - table1.table.shape[0], original_shape[0] + table2.table.shape[0] - ) # Rows should be combined - self.assertEqual(table1.table.shape, (5, 2)) # 5 rows (2 + 3 rows), 2 columns - self.assertEqual(table1.table.iloc[0, 0], "A") - self.assertEqual( - table1.table.iloc[2, 0], "C" - ) # Check if second table's data is appended correctly - self.assertEqual(table1.table.iloc[3, 1], "4") # Last row of second table - self.assertEqual( - list(table1.table.columns), ["Column 1", "Column 2"] - ) # Should keep the first table's header - - def test_table_combine_different_columns(self): - """Test that combining tables with different column counts handles mismatches gracefully.""" - table1 = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=[["A", "B"], ["1", "2"]], - start_index=0, - end_index=100, - ) - table2 = Table( - page_num="1", - metadata="metadata", - header="Table 2", - table_data=[["C", "D", "E"], ["3", "4", "5"]], # 3 columns vs 2 columns - start_index=0, - end_index=100, - ) - - # Should not raise an error anymore - should handle column mismatch - table1.combine(table2) - - # Verify the result - # Should have 2 rows total (1 from each table) - self.assertEqual(table1.table.shape[0], 2) - - # Should have 2 columns (from the main table) - self.assertEqual(table1.table.shape[1], 2) - - # Column names should match the main table - self.assertEqual(list(table1.table.columns), ["A", "B"]) - - # First row should be from table1 - self.assertEqual(table1.table.iloc[0, 0], "1") - self.assertEqual(table1.table.iloc[0, 1], "2") - - # Second row should be from table2, but truncated to 2 columns - self.assertEqual(table1.table.iloc[1, 0], "3") - self.assertEqual(table1.table.iloc[1, 1], "4") - # The "5" from column "E" should be truncated/lost - - def test_table_combine_column_padding(self): - """Test that combining tables pads shorter tables with empty columns.""" - table1 = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=[["A", "B", "C"], ["1", "2", "3"]], # 3 columns - start_index=0, - end_index=100, - ) - table2 = Table( - page_num="1", - metadata="metadata", - header="Table 2", - table_data=[["X"], ["4"]], # Only 1 column - start_index=0, - end_index=100, - ) - - table1.combine(table2) - - # Should have 2 rows total - self.assertEqual(table1.table.shape[0], 2) - - # Should have 3 columns (from the main table) - self.assertEqual(table1.table.shape[1], 3) - - # Column names should match the main table - self.assertEqual(list(table1.table.columns), ["A", "B", "C"]) - - # First row should be unchanged - self.assertEqual(table1.table.iloc[0, 0], "1") - self.assertEqual(table1.table.iloc[0, 1], "2") - self.assertEqual(table1.table.iloc[0, 2], "3") - - # Second row should be from table2, padded with empty strings - self.assertEqual(table1.table.iloc[1, 0], "4") - self.assertEqual(table1.table.iloc[1, 1], "") # Padded - self.assertEqual(table1.table.iloc[1, 2], "") # Padded - - def test_table_combine_empty_tables(self): - """Test combining with empty tables.""" - table1 = Table( - page_num="1", - metadata="meta", - header="Table 1", - table_data=[["A", "B"]], - start_index=0, - end_index=100, - ) - empty_table = Table( - page_num="2", - metadata="meta", - header="", - table_data=[], - start_index=0, - end_index=100, - is_continuation=True, - ) - - original_shape = table1.table.shape - table1.combine(empty_table) - self.assertEqual(table1.table.shape, original_shape) # Should remain unchanged - - def test_convert_str_to_dataframe_malformed_data(self): - """Test handling of malformed table data.""" - malformed_inputs = [ - "[['A', 'B'], ['1']]", # Inconsistent row lengths - "[['A'], [], ['B']]", # Empty rows - "[[]]", # Empty nested list - "['not', 'nested']", # Not nested list - ] - - for malformed_input in malformed_inputs: - with self.subTest(input=malformed_input): - df = convert_str_to_dataframe(malformed_input) - self.assertIsInstance(df, pd.DataFrame) - - def test_table_post_table_text_preservation(self): - """Test that post-table text is properly preserved through operations.""" - text_dict = { - "1": "Metadata\n-------Table Start--------\nTable\n[['A', 'B'], ['1', '2%']]\n-------Table End--------\nImportant footnote\nSignature line" - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - table = table_dict["1"][0] - - # Should capture post-table text - self.assertIn("Important footnote", table.post_table_text) - self.assertIn("Signature line", table.post_table_text) - - def test_split_large_table_preserves_metadata_and_post_text(self): - """Test that table splitting preserves all important information.""" - table = Table( - page_num="31", - metadata="EXHIBIT F", - header="Rate Table", - table_data=[["HCPC", "Rate"]] - + [[f"T{i:04d}", f"${i}.00"] for i in range(15)], - start_index=100, - end_index=2000, - post_table_text="*Rates effective 2024", - ) - - with patch("src.config.TABLE_ROW_LIMIT", 5): - chunks = split_large_table_with_headers(table) - - # All chunks should have metadata and header - for chunk in chunks: - self.assertIn("EXHIBIT F", chunk) - self.assertIn("Rate Table", chunk) - - # Only last chunk should have post-table text - self.assertIn("*Rates effective 2024", chunks[-1]) - for chunk in chunks[:-1]: - self.assertNotIn("*Rates effective 2024", chunk) - - def test_table_split(self): - """Test splitting a table.""" - long_data = [["Name", "Value"]] + [ - [f"Row{i}", str(i)] for i in range(1, 6) - ] # This table will have 5 rows - long_table = Table( - page_num="1", - metadata="metadata", - header="Long Table", - table_data=long_data, - start_index=0, - end_index=100, - ) - split_tables = long_table.split(row_limit=2) - - self.assertEqual( - len(split_tables), 3 - ) # Should split into 3 tables (5 rows split by 2 = 3) - self.assertEqual( - split_tables[0].table.shape, (2, 2) - ) # First split should have 2 rows - self.assertEqual( - split_tables[1].table.shape, (2, 2) - ) # Second split should have 2 rows - self.assertEqual( - split_tables[2].table.shape, (1, 2) - ) # Last split should have 1 row - - # Check that original metadata is preserved - for split_table in split_tables: - self.assertEqual(split_table.metadata, "metadata") - self.assertEqual(split_table.page_num, "1") - self.assertEqual(split_table.header, "Long Table") - - def test_table_split_no_split_needed(self): - """Test splitting a table that does not need splitting. It should return the original table.""" - table = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=[["A", "B"], ["1", "2"], ["3", "4"]], - start_index=0, - end_index=100, - ) - split_tables = table.split(row_limit=10) # Row limit larger than table size - self.assertEqual(len(split_tables), 1) # Should return the original table - self.assertEqual( - split_tables[0].table.shape, (2, 2) - ) # Should be the same shape as original - self.assertEqual(split_tables[0], table) # Should be the same object - - def test_table_to_list_format(self): - """Test conversion back to list format.""" - table_data = [["A", "B"], ["1", "3"], ["2", "4"]] - table = Table( - page_num="1", - metadata="metadata", - header="Table 1", - table_data=table_data, - start_index=0, - end_index=100, - ) - list_format = table.to_list_format() - - self.assertEqual(list_format, table_data) - self.assertIsInstance(list_format, list) - self.assertIsInstance(list_format[0], list) # Should be a list of lists - - def test_table_info_new_format(self): - """Test getting table info from new list-of-rows format.""" - table_dict = get_table_info( - self.sample_text_dict_new, self.EXHIBIT_HEADER_MARKERS - ) - - self.assertIn("1", table_dict) - self.assertEqual(len(table_dict["1"]), 1) - self.assertEqual(table_dict["1"][0].header, "Table 1") - self.assertEqual(table_dict["1"][0].metadata, "Some metadata") - self.assertEqual( - table_dict["1"][0].table.shape, (2, 2) - ) # 2 rows, 2 columns in first table - self.assertEqual(table_dict["2"][0].header, "") # Second table has no header - self.assertEqual( - table_dict["2"][0].table.shape, (3, 2) - ) # 3 rows, 2 columns in second table (empty header means all data rows) - - def test_table_info_old_format(self): - """Test getting table info from old dict format.""" - with self.assertLogs(level="INFO") as log: # Expect a deprecation warning - table_dict = get_table_info( - self.sample_text_dict_old, self.EXHIBIT_HEADER_MARKERS - ) - - self.assertIn("1", table_dict) - self.assertEqual(len(table_dict["1"]), 1) - self.assertEqual(table_dict["1"][0].header, "Table 1") - self.assertEqual(table_dict["1"][0].metadata, "Some metadata") - self.assertEqual(table_dict["1"][0].table.shape, (2, 2)) - - def test_get_table_info_multiple_tables(self): - """Test getting table info from a page with multiple tables.""" - table_dict = get_table_info(self.multi_table_text, self.EXHIBIT_HEADER_MARKERS) - - self.assertIn("1", table_dict) - self.assertEqual(len(table_dict["1"]), 1) - self.assertEqual(table_dict["1"][0].header, "First Table") - - def test_recreate_page(self): - """Test recreating page text from tables.""" - table_dict = get_table_info( - self.sample_text_dict_new, self.EXHIBIT_HEADER_MARKERS - ) - page_tables = table_dict["1"] - recreated_text = recreate_page(page_tables) - - self.assertIn("Table 1", recreated_text) - self.assertIn("-------Table Start--------", recreated_text) - self.assertIn("-------Table End--------", recreated_text) - self.assertIn("Some metadata", recreated_text) - - def test_split_tables_by_row_count(self): - """Test splitting tables that exceed row limit and adding .0 suffix.""" - # Create a long table that will need splitting - long_table_data = [["Name", "Value"]] + [ - [f"Row{i}%", str(i)] for i in range(1, 8) - ] # 7 data rows - text_dict = { - "5": f"Table metadata\n-------Table Start--------\nLong Table\n{long_table_data}\n-------Table End--------\nPost-table text" - } - - result = split_tables_by_rows( - text_dict, self.EXHIBIT_HEADER_MARKERS, row_limit=3 - ) - - # Should create multiple pages with .0, .1, .2 suffixes - expected_pages = ["5.0", "5.1", "5.2"] - for page in expected_pages: - self.assertIn(page, result) - - # Original page should not exist - self.assertNotIn("5", result) - - # Last page should have post-table text - self.assertIn("Post-table text", result["5.2"]) - - def test_split_tables_by_row_count_no_split_needed(self): - """Test that small tables get .0 suffix but aren't split.""" - text_dict = { - "3": "Metadata\n-------Table Start--------\nSmall Table\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" - } - - result = split_tables_by_rows( - text_dict, self.EXHIBIT_HEADER_MARKERS, row_limit=10 - ) - - # Should have .0 suffix but no additional splits - self.assertIn("3.0", result) - self.assertNotIn("3", result) - self.assertNotIn("3.1", result) - - def test_split_tables_by_row_count_no_tables(self): - """Test that pages without tables keep original page numbers.""" - text_dict = { - "1": "Just some regular text without any tables.", - "2": "More text, no tables here either.", - } - - result = split_tables_by_rows( - text_dict, self.EXHIBIT_HEADER_MARKERS, row_limit=5 - ) - - # Pages without tables should keep original numbers - self.assertIn("1", result) - self.assertIn("2", result) - self.assertNotIn("1.0", result) - self.assertNotIn("2.0", result) - - def test_combine_continuous_tables_no_continuation(self): - """Test that non-continuation tables are not combined.""" - text_dict = { - "1": "Metadata 1\n-------Table Start--------\nTable 1\n[['A', 'B'], ['1%', '2']]\n-------Table End--------", - "2": "Metadata 2\n-------Table Start--------\nTable 2\n[['C', 'D'], ['3%', '4']]\n-------Table End--------", - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - result = combine_continuous_tables(text_dict, table_dict) - - # Both pages should remain (no continuation tables to combine) - self.assertIn("1", result) - self.assertIn("2", result) - self.assertEqual(len(result), 2) - - def test_table_combine_post_table_text_accumulation(self): - """Test that post-table text is properly accumulated when combining tables.""" - table1 = Table( - page_num="1", - metadata="meta", - header="Main Table", - table_data=[["A", "B"], ["1", "2"]], - start_index=0, - end_index=100, - post_table_text="First footnote", - ) - - table2 = Table( - page_num="2", - metadata="", - header="", - table_data=[["3", "4"]], - start_index=0, - end_index=100, - is_continuation=True, - post_table_text="Second footnote", - ) - - table1.combine(table2) - - # Both footnotes should be preserved - self.assertIn("First footnote", table1.post_table_text) - self.assertIn("Second footnote", table1.post_table_text) - - def test_table_combine_column_mismatch_preserves_post_table_text(self): - """Test that post-table text is preserved when combining tables with column mismatches.""" - table1 = Table( - page_num="1", - metadata="metadata", - header="Main Table", - table_data=[["A", "B"], ["1", "2"], ["x", "y"]], # Header + 2 data rows - start_index=0, - end_index=100, - post_table_text="Main table footnote", - ) - - table2 = Table( - page_num="2", - metadata="", - header="", - table_data=[["3", "4", "5"]], # Just 1 data row (3 columns - mismatch!) - start_index=0, - end_index=100, - is_continuation=True, - post_table_text="Continuation footnote", - ) - - table1.combine(table2) - - # Verify both post-table texts are preserved despite column mismatch - self.assertIn("Main table footnote", table1.post_table_text) - self.assertIn("Continuation footnote", table1.post_table_text) - - # Verify the table combination worked correctly - self.assertEqual( - table1.table.shape[0], 3 - ) # 2 original + 1 continuation = 3 total rows - self.assertEqual(table1.table.shape[1], 2) # 2 columns (from main table) - - # Verify data integrity - self.assertEqual(table1.table.iloc[0, 0], "1") # Original data row 1 - self.assertEqual(table1.table.iloc[1, 0], "x") # Original data row 2 - self.assertEqual(table1.table.iloc[2, 0], "3") # Continuation data (truncated) - - def test_table_combine_column_mismatch_padding_preserves_post_table_text(self): - """Test post-table text preservation when padding columns.""" - table1 = Table( - page_num="1", - metadata="metadata", - header="Main Table", - table_data=[ - ["A", "B", "C"], - ["1", "2", "3"], - ["x", "y", "z"], - ], # Header + 2 data rows - start_index=0, - end_index=100, - post_table_text="Main footnote", - ) - - table2 = Table( - page_num="2", - metadata="", - header="", - table_data=[["4"]], # Just 1 data row (1 column - needs padding) - start_index=0, - end_index=100, - is_continuation=True, - post_table_text="Padded footnote", - ) - - table1.combine(table2) - - # Both footnotes should be preserved even with column padding - self.assertIn("Main footnote", table1.post_table_text) - self.assertIn("Padded footnote", table1.post_table_text) - - # Verify padding worked - self.assertEqual( - table1.table.shape, (3, 3) - ) # 2 original + 1 continuation = 3 rows, 3 columns - self.assertEqual(table1.table.iloc[0, 0], "1") # Original data row 1 - self.assertEqual(table1.table.iloc[1, 0], "x") # Original data row 2 - self.assertEqual(table1.table.iloc[2, 0], "4") # Continuation data - self.assertEqual(table1.table.iloc[2, 1], "") # Padded - self.assertEqual(table1.table.iloc[2, 2], "") # Padded - - def test_exhibit_boundary_detection_prevents_combination(self): - """Test that exhibit boundaries prevent incorrect table combination.""" - text_dict = { - "1": "EXHIBIT A-1\nCommercial rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['A', '1%']]\n-------Table End--------", - "2": "EXHIBIT B-1\nMedicare rates\n-------Table Start--------\n\n[['Service', 'Rate'], ['B', '2%']]\n-------Table End--------", - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - - # Both should be detected as non-continuation tables due to exhibit boundaries - self.assertFalse(table_dict["1"][0].is_continuation) - self.assertFalse(table_dict["2"][0].is_continuation) - - # Combination should not remove any pages - result = combine_continuous_tables(text_dict, table_dict) - self.assertIn("1", result) - self.assertIn("2", result) - self.assertEqual(len(result), 2) - - def test_exhibit_boundary_case_insensitive(self): - """Test that exhibit detection works with different cases.""" - test_cases = [ - "EXHIBIT A-1", - "Exhibit A-1", - "exhibit a-1", - "ATTACHMENT B", - "Schedule C", - "ADDENDUM FOR MEDICARE", - ] - - for exhibit_text in test_cases: - with self.subTest(exhibit=exhibit_text): - text_dict = { - "1": f"{exhibit_text}\nRate information\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - - # Should not be detected as continuation due to exhibit marker - self.assertFalse(table_dict["1"][0].is_continuation) - - def test_exhibit_boundary_detection_beyond_100_chars(self): - """Test that exhibit markers beyond 100 characters don't affect continuation detection.""" - # Create a page with exhibit marker after position 100 - long_prefix = "A" * 95 # 95 characters - text_dict = { - "1": f"{long_prefix}EXHIBIT A-1\nRate info\n-------Table Start--------\n\n[['A', 'B'], ['1%', '2']]\n-------Table End--------" - } - - table_dict = get_table_info(text_dict, self.EXHIBIT_HEADER_MARKERS) - - # Should be treated as continuation since exhibit marker is beyond position 100 - self.assertTrue(table_dict["1"][0].is_continuation) - - def test_table_init_with_empty_list(self): - """Test Table initialization with empty list.""" - table = Table( - page_num="1", - metadata="meta", - header="Empty Table", - table_data=[], - start_index=0, - end_index=100, - ) - - self.assertTrue(table.table.empty) - self.assertEqual(table.table.shape, (0, 0)) - - def test_table_init_with_only_headers(self): - """Test Table initialization with only header row (no data).""" - table = Table( - page_num="1", - metadata="meta", - header="Header Only", - table_data=[["Column A", "Column B"]], - start_index=0, - end_index=100, - ) - - self.assertEqual(table.table.shape, (0, 2)) # 0 rows, 2 columns - self.assertEqual(list(table.table.columns), ["Column A", "Column B"]) - - def test_continuation_table_with_generic_column_names_combination(self): - """Test that continuation tables with generic column names combine correctly.""" - main_table = Table( - page_num="1", - metadata="meta", - header="Main Table", - table_data=[["Service", "Rate"], ["A", "1%"], ["B", "2%"]], - start_index=0, - end_index=100, - ) - - # Continuation table with generic column names - continuation_table = Table( - page_num="2", - metadata="", - header="", - table_data=[["C", "3%"], ["D", "4%"]], - start_index=0, - end_index=100, - is_continuation=True, - ) - - main_table.combine(continuation_table) - - # Should have 4 data rows total - self.assertEqual(main_table.table.shape, (4, 2)) - - # Column names should remain from main table - self.assertEqual(list(main_table.table.columns), ["Service", "Rate"]) - - # Data should be properly combined - self.assertEqual(main_table.table.iloc[0, 0], "A") # Original - self.assertEqual(main_table.table.iloc[2, 0], "C") # Continuation - self.assertEqual(main_table.table.iloc[3, 1], "4%") # Last continuation row - - def test_recreate_page_with_multiple_tables_and_post_text(self): - """Test page recreation with multiple tables and post-table text.""" - tables = [ - Table( - page_num="1", - metadata="Page header", - header="First Table", - table_data=[["A", "B"], ["1%", "2"]], - start_index=0, - end_index=100, - ), - Table( - page_num="1", - metadata="", - header="Second Table", - table_data=[["X", "Y"], ["3%", "4"]], - start_index=200, - end_index=300, - post_table_text="Important footnote\nSignature required", - ), - ] - - recreated = recreate_page(tables) - - # Should contain both tables - self.assertIn("First Table", recreated) - self.assertIn("Second Table", recreated) - - # Should contain post-table text from last table - self.assertIn("Important footnote", recreated) - self.assertIn("Signature required", recreated) - - # Should have proper structure - self.assertIn(START_MARKER, recreated) - self.assertIn(END_MARKER, recreated) - - def test_split_tables_by_rows_preserves_metadata_and_post_text(self): - """Test that table splitting preserves all text components.""" - table_data = [["Code", "Rate"]] + [[f"T{i:04d}", f"${i}.00"] for i in range(12)] - - text_dict = { - "5": f"EXHIBIT F\nRate Schedule\n-------Table Start--------\nLarge Table\n{table_data}\n-------Table End--------\nEffective 2024\nContact: admin@health.net" - } - - result = split_tables_by_rows( - text_dict, self.EXHIBIT_HEADER_MARKERS, row_limit=5 - ) - - # Should create multiple pages - expected_pages = ["5.0", "5.1", "5.2"] - for page in expected_pages: - self.assertIn(page, result) - - # All pages should have metadata and header - for page in expected_pages: - self.assertIn("EXHIBIT F", result[page]) - self.assertIn("Large Table", result[page]) - - # Only last page should have post-table text - self.assertIn("Effective 2024", result["5.2"]) - self.assertIn("Contact: admin@health.net", result["5.2"]) - - # Earlier pages should not have post-table text - self.assertNotIn("Effective 2024", result["5.0"]) - self.assertNotIn("Effective 2024", result["5.1"]) - - def test_convert_str_to_dataframe_edge_cases(self): - """Test edge cases in string to DataFrame conversion.""" - edge_cases = [ - ("[]", True), # Empty list - ("[['A']]", False), # Single cell - ("[['A', 'B'], []]", False), # Empty data row - ("[[], ['1', '2']]", False), # Empty header row - ("[['']]", False), # Single empty string - ] - - for test_input, is_continuation in edge_cases: - with self.subTest(input=test_input, continuation=is_continuation): - df = convert_str_to_dataframe( - test_input, is_continuation=is_continuation - ) - self.assertIsInstance(df, pd.DataFrame) - # Should not raise exceptions - - -if __name__ == "__main__": - unittest.main()