From ae4eda6f250396b4805cd55cbe2f64aa187d1d22 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Tue, 30 Sep 2025 14:04:47 +0000 Subject: [PATCH] Merged in feature/one-to-n-docs (pull request #715) Feature/one to n docs * Refactor function signatures and improve imports in one_to_n_funcs.py * Enhance type hints and documentation for carveout_check and methodology_breakout functions * Add docstring to methodology_breakout_secondary function for clarity on parameters and return values * Add docstring to special_case_breakout function for clarity on parameters and return values * Enhance docstring for deduplicate_with_accumulator function to clarify behavior and parameters * Enhance docstring for filter_services_without_reimbursements function to clarify validation process and answer structure * Enhance docstring for filter_services_without_reimbursements function to clarify filtering criteria and validation process * Enhance docstring for split_compound_reimbursements function to clarify the splitting process and provide an example * Enhance docstring for split_reimb_dates function to clarify date processing, parameters, return values, and provide an example * Remove unused regex import from one_to_n_funcs.py * Remove unused Delimiter import from one_to_n_funcs.py Approved-by: Katon Minhas --- .../src/investment/one_to_n_funcs.py | 184 +++++++++++++++--- 1 file changed, 160 insertions(+), 24 deletions(-) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index b1068cb..02a0670 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -1,19 +1,17 @@ import logging -import re +import src.investment.dynamic_funcs as dynamic_funcs +import src.investment.lesser_of_funcs as lesser_of_funcs +import src.investment.prompt_calls as prompt_calls import src.prompts.prompt_templates as prompt_templates import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from constants.constants import Constants -from constants.delimiters import Delimiter from src import config from src.prompts.fieldset import FieldSet -import src.investment.dynamic_funcs as dynamic_funcs -import src.investment.prompt_calls as prompt_calls -import src.investment.lesser_of_funcs as lesser_of_funcs -def get_exhibit_header(all_exhibit_headers, exhibit_page): +def get_exhibit_header(all_exhibit_headers: dict, exhibit_page: str) -> str: """ Retrieves the exhibit header for a given exhibit page. Args: @@ -45,7 +43,7 @@ def exhibit_level( reimbursement_level_fields: FieldSet, constants: Constants, filename: str, -): +) -> tuple[dict, FieldSet]: """ Processes exhibit-level fields, including dynamic fields. @@ -71,7 +69,7 @@ def exhibit_level( exhibit_level_answers = prompt_calls.prompt_exhibit_lesser( exhibit_text, exhibit_level_answers, filename ) - + # Dynamic Primary exhibit_level_answers, reimbursement_level_fields = dynamic_funcs.dynamic_primary( exhibit_text, @@ -162,13 +160,13 @@ def reimbursement_level( def clean_reimbursement_primary( - reimbursement_primary_answers, - exhibit_lesser_of, - seen_pairs, - exhibit_page, + reimbursement_primary_answers: list[dict], + exhibit_lesser_of: str, + seen_pairs: set, + exhibit_page: str, constants: Constants, filename: str, -): +) -> list[dict]: """ Cleans and processes reimbursement primary answers through several steps: 1. Split compound reimbursements @@ -182,6 +180,7 @@ def clean_reimbursement_primary( seen_pairs (set): A set accumulator to track tuples of (base_page, SERVICE_TERM, REIMB_TERM) to avoid duplicate processing. constants (Constants): A Constants object exhibit_page (str): The page number of the exhibit being processed (for a unique identifier) + filename (str): The name of the file being processed, for logging Returns: list[dict]: The cleaned and processed reimbursement primary answers. """ @@ -221,7 +220,7 @@ def breakout( special_case_primary_fields: FieldSet, filename: str, constants: Constants, -): +) -> tuple[list[dict], dict[str, list]]: """ Performs breakout processing on reimbursement primary answers and special case primary answers. @@ -254,7 +253,7 @@ def breakout( def carveout_check( reimbursement_primary_answers: list[dict], constants: Constants, filename: str -): +) -> list[dict]: """ Performs carveout check on reimbursement primary answers. @@ -279,14 +278,33 @@ def carveout_check( answer_dict["CARVEOUT_IND"] = "N" else: answer_dict["CARVEOUT_IND"] = "Y" - + # Determine DEFAULT_IND based on specific carveout_answer values - answer_dict["DEFAULT_IND"] = "Y" if carveout_answer in ["DEFAULT_TERM", "UNLISTED_CODE"] else "N" + answer_dict["DEFAULT_IND"] = ( + "Y" if carveout_answer in ["DEFAULT_TERM", "UNLISTED_CODE"] else "N" + ) return reimbursement_primary_answers -def methodology_breakout(reimbursement_primary_answers, constants, filename): +def methodology_breakout( + reimbursement_primary_answers: list[dict], constants: Constants, filename: str +) -> list[dict]: + """Perform methodology breakout on reimbursement primary answers. + + This function loops through each reimbursement primary answer and applies the + methodology breakout process, which may include additional prompts for fee schedule + or grouper details based on the reimbursement method. + + Args: + reimbursement_primary_answers (list[dict]): List of dictionaries containing reimbursement details from the reimbursement primary step. + constants (Constants): A Constants object. + filename (str): The name of the file being processed, for logging. + + Returns: + list[dict]: The processed reimbursement primary answers, augmented with methodology breakout details. + """ + final_answers = [] for answer_dict in reimbursement_primary_answers: final_answers += methodology_breakout_single_row( @@ -369,7 +387,38 @@ def methodology_breakout_secondary( grouper_breakout_questions: FieldSet, constants: Constants, filename: str, -): +) -> dict: + """Performs specialized breakout based on the derived reimbursement methodology type. + + After the primary methodology breakout determines the AARETE_DERIVED_REIMB_METHOD, + this function applies methodology-specific prompts to extract detailed parameters: + + - **Fee Schedule**: Extracts fee schedule type, version, and related details + - **Grouper**: Extracts DRG weights, severity adjustments, transfer policies, etc. + - **Other methodologies**: Sets all specialized fields to "N/A" + + Additionally, it ensures mutually exclusive field population across Fee Schedule and Grouper fields. + (Fee Schedule OR Grouper, never both.) + + Args: + methodology_breakout_dict (dict): Results from primary methodology breakout, + including the AARETE_DERIVED_REIMB_METHOD field which will drive the + branching logic. + service (str): The service term associated with the reimbursement. + reimbursement_method (str): The original reimbursement method text extracted + (pre-AARETE_DERIVED) + fs_breakout_questions (FieldSet): Fee Schedule-specific prompts and field definitions. + grouper_breakout_questions (FieldSet): Grouper-specific prompts and field definitions. + constants (Constants): System constants for processing. + filename (str): The filename used for LLM processing. + + Returns: + dict: Specialized breakout fields populated based on methodology type. + Fee Schedule methodology → fee schedule fields + grouper fields set to "N/A" + Grouper methodology → grouper fields + fee schedule fields set to "N/A" + Other methodologies → all specialized fields set to "N/A" + """ + AARETE_DERIVED_REIMB_METHOD = methodology_breakout_dict.get( "AARETE_DERIVED_REIMB_METHOD", "" ) @@ -417,6 +466,27 @@ def special_case_breakout( special_case_primary_fields: FieldSet, filename: str, ) -> dict[str, list]: + """Applies specialized breakout templates to special case primary answers. + + For each special case field (outliers, discounts, premiums, etc.) that has extracted + terms, this function applies the field-specific breakout template to each term + to extract detailed and custom parameters (thresholds, percentages, conditions, + etc.) + + Args: + special_case_primary_answers (dict): A dictionary containing the primary answers + for special case fields. This will take the form of dict[str, list[str]], + where the key is the field name and the value is a list of extracted terms + for that field. + TODO: Add examples here and to `Returns:` section below. + special_case_primary_fields (FieldSet): FieldSet containing special case primary + field definitions and breakout templates. + filename (str): The name of the file being processed, for logging. + + Returns: + dict[str, list]: A dictionary containing the processed breakout answers for each + special case field. + """ final_answer_dict = {} @@ -444,11 +514,21 @@ def special_case_breakout( def deduplicate_with_accumulator( answers: list[dict], seen_pairs: set, exhibit_page: str, filename: str ) -> list[dict]: - """Remove service-reimbursement pairs that have been seen in previous exhibits. + """Remove service-reimbursement pairs that have been seen in previous pages. + Includes a base page scope to allow only subexhibits to share pairs. This is because + (a) subexhibits often repeat information due to our table splitting logic, and + (b) different exhibits may share the same (service, reimbursement) pairs, + but in meaningfully different contexts (e.g. MEDICARE FEE SCHEDULE on + exhibit page 23 vs. MARKETPLACE FEE SCHEDULE on exhibit 45). In this case + we do NOT want to deduplicate across exhibits. Args: answers (list[dict]): List of dictionaries with SERVICE_TERM and REIMB_TERM keys. seen_pairs (set): A set of previously seen (base_page, SERVICE_TERM, REIMB_TERM) pairs. + e.g. {("23", "Routine Vision", "Provider's Billed Charges"), ...} + This is the "accumulator" that persists across multiple calls. It is + modified in-place to add new pairs as they are encountered, and as such + is not returned by the function. exhibit_page (str): The exhibit page number, for logging. filename (str): The name of the file being processed, for logging @@ -491,10 +571,13 @@ def filter_services_without_reimbursements( """ Filters out services that lack actual reimbursement methodologies. - Uses a single-step approach of LLM validation for remaining entries to determine if they contain actual reimbursement information. + Uses LLM validation to determine if entries contain actual reimbursement information. + Entries that contain service definitions, administrative content, or vague terms + rather than clear reimbursement methodologies are filtered out. Args: answers (list[dict]): List of dictionaries containing reimbursement details. + These will be of the form dict[str, str], with keys including SERVICE_TERM and REIMB_TERM. filename (str): The name of the file being processed. Returns: @@ -533,6 +616,32 @@ def split_compound_reimbursements( """ Splits reimbursement terms that contain multiple distinct methodologies. + The process involves two main steps: + 1. A substring search is performed to identify potential compound reimbursements + using a predefined list of indicators (e.g. "otherwise shall be", "except for", + "for services where there is no", etc.). + 2. For entries flagged as potential compounds, an LLM prompt is invoked to + attempt to split the entry into multiple distinct reimbursement terms. + If the LLM successfully splits the entry into multiple parts, these are added + to the results. If not, the original entry is retained. + + Example: + INPUT: + SERVICE: "Drugs" + REIMBURSEMENT: "lesser of Provider's billed charges or: (i) Fee schedule rate for standard drugs, (ii) WAC + 5% for specialty drugs, not to exceed $50" + + OUTPUT: + [ + {{ + "SERVICE_TERM": "standard drugs", + "REIMB_TERM": "lesser of Provider's billed charges or Fee schedule rate" + }}, + {{ + "SERVICE_TERM": "specialty drugs", + "REIMB_TERM": "lesser of Provider's billed charges or WAC + 5%, not to exceed $50" + }} + ] + Args: answers (list[dict]): List of dictionaries containing reimbursement details. filename (str): The name of the file being processed. @@ -546,7 +655,7 @@ def split_compound_reimbursements( for answer_dict in answers: reimb_term = answer_dict.get("REIMB_TERM", "").lower().strip() - # Check if this looks like a compound reimbursement + # Check if this looks like a compound reimbursement by using a substring search has_compound = any(indicator in reimb_term for indicator in COMPOUND_INDICATORS) if has_compound: @@ -574,8 +683,35 @@ def split_compound_reimbursements( def split_reimb_dates(one_to_n_results: list, filename: str) -> list: - """Extract the date range from the field REIMB_DATES and split the dates into - REIMB_EFFECTIVE_DT and REIMB_TERMINATION_DT by invoking LLM and split the dates based on the context. + """Splits REIMB_DATES field into separate effective and termination date fields. + + Processes records containing REIMB_DATES and uses LLM to parse date ranges into + structured REIMB_EFFECTIVE_DT and REIMB_TERMINATION_DT fields. The original + REIMB_DATES field is removed after processing. + + Args: + one_to_n_results (list): List of dictionaries containing reimbursement records. + Records with valid REIMB_DATES fields will be processed. + filename (str): The name of the file being processed, for logging. + + Returns: + list: Updated list of dictionaries with: + - REIMB_EFFECTIVE_DT: Parsed start date from date range + - REIMB_TERMINATION_DT: Parsed end date from date range + - REIMB_DATES: Removed from all records + + Example: + Input record: {"SERVICE_TERM": "Labs", "REIMB_DATES": "January 1, 2024 through December 31, 2024"} + Output record: { + "SERVICE_TERM": "Labs", + "REIMB_EFFECTIVE_DT": "2024-01-01", + "REIMB_TERMINATION_DT": "2024-12-31" + } + + Note: + - Records without valid REIMB_DATES are left unchanged (but still get the new fields initialized as None) + - Processing continues even if individual records fail to parse + - All records have REIMB_DATES field removed regardless of processing success """ if not one_to_n_results or not isinstance(one_to_n_results, list):