From f26b83bd0bd1dee79aebd08ddb130991fbe2481c Mon Sep 17 00:00:00 2001 From: Praneel Panchigar Date: Wed, 17 Dec 2025 19:15:49 +0000 Subject: [PATCH] Merged in feature/cross-exhibit-dynamic (pull request #789) Feature/cross exhibit dynamic * Merge branch 'feature/deprecate-haiku-3' into feature/1toN-Optimization * fix over-filtering of lesser of * Merged in bugfix/UT-methodology-breakout (pull request #794) Bugfix/UT methodology breakout * updated valid values for AARETE_DERIVED_REIMB_METHOD * removed example reimbursements * prompt update * prompt update * Merged feature/1toN-Optimization into bugfix/nv_issue_fixes * add service term in mb prompts * Merged feature/1toN-Optimization into bugfix/UT-methodology-breakout * print statement removed * Merge branch 'bugfix/UT-methodology-breakout' of https://bitbucket.org/aarete/doczy.ai into bugfix/UT-methodology-breakout * primary prompt update * remove duplicate prompt Approved-by: Katon Minhas * Merge branch 'main' into feature/1toN-Optimization * Merge branch 'main' into feature/1toN-Optimization * Update preprocessing to make the exhibit_chunk_mapping start at first page * Merge remote-tracking branch 'origin/feature/1toN-Optimization' into cross-exhibit-dynamic * Address merge request comments: refactor prompt templates and logging - Make DYNAMIC_PRIMARY_TEXT LOB-specific instructions conditional (only show when field_name is LOB) - Remove MEDICAID FEE SCHEDULE point from DYNAMIC_ASSIGNMENT (point #2) - Generalize SERVICE_TERM context guidance to apply to all fields (LOB, PROGRAM, NETWORK, PRODUCT) - Refactor proximity guidance to emphasize contextual connection over strict section boundaries - Update point #1 to explicitly prevent inferring LOB from Programs alone - Change exhibit inheritance logging from info to debug level * Move function to one_to_n_funcs * Merge row_funcs.py changes from feature/cross-exhibit-dynamic * Merged feature/1toN-Optimization into feature/cross-exhibit-dynamic * Merged in bugfix/UT-grouper-issues (pull request #796) Bugfix/UT grouper issues * updated valid values for AARETE_DERIVED_REIMB_METHOD * removed example reimbursements * prompt update * prompt update * Merged feature/1toN-Optimization into bugfix/nv_issue_fixes * add service term in mb prompts * Merged feature/1toN-Optimization into bugfix/UT-methodology-breakout * print statement removed * Merge branch 'bugfix/UT-methodology-breakout' of https://bitbucket.org/aarete/doczy.ai into bugfix/UT-methodology-breakout * primary prompt update * prompt update * Merge remote-tracking branch 'origin/feature/1toN-Optimization' into bugfix/UT-grouper-issues * removed temp changes * removed temp changes * Update reimb primary Approved-by: Katon Minhas * Merged in bugfix/validation_fixes (pull request #795) bugfix/validation_fixes to feature/1toN-Optimization * updated validation of clean claims reimbursement * Merged feature/1toN-Optimization into bugfix/validation_fixes Approved-by: Katon Minhas * Merge branch 'feature/1toN-Optimization' into feature/cross-exhibit-dynamic * Re-add dynamic codes and reimb-info * Re-structure empty reimbursement prompt * Fix lesser of check overfiltering * Merged feature/1toN-Optimization into feature/cross-exhibit-dynamic * Update LOB inference logic and prompt guidance - Revert DYNAMIC_PRIMARY_TEXT to original template (remove LOB-specific conditional section) - Add explicit guidance in DYNAMIC_ASSIGNMENT: presence of Medicaid programs does NOT imply LOB is Medicaid - Update get_dynamic_one_to_one_fields to only check AARETE_DERIVED_LOB (not raw LOB) when skipping PROGRAM/PRODUCT/NETWORK * Fix LOB_PROGRAM_RELATIONSHIP extraction: add pipe format instructions to LOB_RELATIONSHIP_INSTRUCTION * Update Fidelis Essential Plan mappings in crosswalk_product_lob.json - Add mappings for Essential Plan variants (Aliessa, EP-QHP, EP, Essential Plan) - Essential Plan Aliessa and EP Aliessa map to Medicaid - EP-QHP maps to Commercial - EP and Essential Plan map to Medicaid|Commercial * Remove debug logging statements from prompt_lob_relationship * Merge main into feature/cross-exhibit-dynamic: resolved conflicts, removed debug statements, synced prompt_templates.py with main * Sync non-exhibit-merge files with main before merge * Remove test.py to match main Approved-by: Katon Minhas --- .../src/investment/file_processing.py | 38 +++++++++- .../src/investment/one_to_n_funcs.py | 71 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index ed785e3..7015327 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -6,7 +6,7 @@ from src.investment import preprocessing_funcs, aarete_derived, dynamic_funcs, o from src.utils import io_utils, logging_utils, string_utils from constants.constants import Constants from src import config -from src.prompts.fieldset import FieldSet +from src.prompts.fieldset import FieldSet, Field from src.utils.string_utils import datetime_str @@ -121,6 +121,7 @@ def run_one_to_one_prompts( relationship="one_to_one", file_path=config.FIELD_JSON_PATH ).combine(dynamic_one_to_one_fields) + ################## RUN PROVIDER INFO ################## one_to_one_results, one_to_one_fields = tin_npi_funcs.run_provider_info_fields( contract_text, one_to_one_fields, text_dict, filename @@ -174,6 +175,7 @@ def run_one_to_n_prompts(text_dict: dict[str, str], one_to_n_results = [] first_reimbursement_page = '1' + previous_exhibit = None #################################### PROCESS EACH EXHIBIT SEPARATELY #################################### for exhibit_page, exhibit_page_nums in exhibit_chunk_mapping.items(): @@ -192,7 +194,25 @@ def run_one_to_n_prompts(text_dict: dict[str, str], ) ) # dict, FieldSet logging.debug(f"Exhibit Level Answers for {filename}, Page {exhibit_page}: {exhibit_level_answers}") - + + # Store original values before inheritance (for tracking in previous_exhibit) + original_exhibit_page_nums = exhibit_page_nums.copy() + original_exhibit_text = exhibit_text_original + original_dynamic_reimbursement_fields = dynamic_reimbursement_fields + + ############################### Check and Combine Exhibit Inheritance ############################### + # IF dynamic_reimbursement_fields is empty FieldSet, and previous exhibit's dynamic_reimbursement_fields is NOT empty, + # and the previous exhibit has NO reimbursement rows, then replace dynamic_reimbursement_fields with the previous + # exhibit's dynamic_reimbursement_field values AND set exhibit_page_nums = exhibit_page_nums for exhibit N-1 + exhibit_page_nums for exhibit N + should_inherit, exhibit_page_nums, dynamic_reimbursement_fields = ( + one_to_n_funcs.check_and_combine_exhibit_inheritance( + previous_exhibit, + exhibit_page_nums, + exhibit_text_original, + dynamic_reimbursement_fields, + ) + ) + all_exhibit_reimbursements, all_exhibit_special_case = [], [] #################################### PAGE-BY-PAGE REIMBURSEMENT PRIMARY #################################### for page_num in exhibit_page_nums: @@ -224,6 +244,20 @@ def run_one_to_n_prompts(text_dict: dict[str, str], ################################ Add to Total ############################### one_to_n_results += all_exhibit_rows + ################################ Store Current Exhibit for Next Iteration ############################### + # Store original (non-combined) values for next exhibit to potentially inherit from + # Determine if current exhibit had reimbursements + had_reimbursements = len(all_exhibit_reimbursements) > 0 or len(all_exhibit_special_case) > 0 + + previous_exhibit = { + 'exhibit_page': exhibit_page, + 'exhibit_page_nums': original_exhibit_page_nums, + 'exhibit_text_original': original_exhibit_text, + 'dynamic_reimbursement_fields': original_dynamic_reimbursement_fields, + 'exhibit_level_answers': exhibit_level_answers.copy(), + 'had_reimbursements': had_reimbursements + } + ################## Add N/A Dynamic or Exhibit to One-to-One ################## dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields( one_to_n_results, constants diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 1b131f5..2ad04b7 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -655,3 +655,74 @@ def lesser_of_distribution(reimbursement_level_answers: list[dict[str, str]], ex return final_reimbursement_level_answers +def check_and_combine_exhibit_inheritance( + previous_exhibit: dict | None, + current_exhibit_page_nums: list[str], + current_exhibit_text: str, + current_dynamic_fields: FieldSet, +) -> tuple[bool, list[str], FieldSet]: + """ + Check if inheritance should happen and combine exhibits if needed. + + Conditions for inheritance: + 1. Previous exhibit exists + 2. Current exhibit's dynamic_reimbursement_fields is empty FieldSet + 3. Previous exhibit's dynamic_reimbursement_fields is NOT empty + 4. Previous exhibit had NO reimbursement rows + + When inheritance happens: + - Replace current dynamic_reimbursement_fields with previous exhibit's dynamic_reimbursement_fields + - Combine exhibit_page_nums (previous + current) to provide context through simplify_exhibit + + Args: + previous_exhibit: Dict with keys: 'exhibit_page_nums', 'dynamic_reimbursement_fields', 'had_reimbursements' + current_exhibit_page_nums: List of page numbers for current exhibit + current_exhibit_text: Full text of current exhibit (not used in return, but kept for consistency) + current_dynamic_fields: FieldSet of dynamic fields found in current exhibit + + Returns: + Tuple of: + - should_inherit: bool indicating if inheritance happened + - combined_page_nums: list[str] (combined if inheritance, original otherwise) + - updated_dynamic_fields: FieldSet (replaced with previous if inheritance, original otherwise) + """ + # Check if previous exhibit exists + if previous_exhibit is None: + return False, current_exhibit_page_nums, current_dynamic_fields + + # Check if current exhibit's dynamic_reimbursement_fields is empty + current_field_names = current_dynamic_fields.list_fields() + current_is_empty = len(current_field_names) == 0 + + if not current_is_empty: + return False, current_exhibit_page_nums, current_dynamic_fields + + # Check if previous exhibit's dynamic_reimbursement_fields is NOT empty + prev_dynamic_fields = previous_exhibit['dynamic_reimbursement_fields'] + prev_field_names = prev_dynamic_fields.list_fields() + prev_is_not_empty = len(prev_field_names) > 0 + + if not prev_is_not_empty: + return False, current_exhibit_page_nums, current_dynamic_fields + + # Check if previous exhibit had NO reimbursement rows + prev_no_reimbursements = not previous_exhibit.get('had_reimbursements', True) + + if not prev_no_reimbursements: + return False, current_exhibit_page_nums, current_dynamic_fields + + # All conditions met - inheritance should happen + # Combine exhibit_page_nums (previous + current) + # This ensures simplify_exhibit will include context from both exhibits + combined_page_nums = previous_exhibit['exhibit_page_nums'] + current_exhibit_page_nums + + # Replace current dynamic_reimbursement_fields with previous exhibit's + updated_dynamic_fields = prev_dynamic_fields + + # Minimal logging: inheritance activated, pages being merged, and fields inherited + prev_pages = ', '.join(previous_exhibit.get('exhibit_page_nums', [])) + current_pages = ', '.join(current_exhibit_page_nums) + fields_str = ', '.join(prev_field_names) + logging.debug(f"Exhibit inheritance activated: merging pages {prev_pages} → {current_pages}, inheriting fields: {fields_str}") + + return True, combined_page_nums, updated_dynamic_fields