diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 54324cf..bcbfb90 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -910,5 +910,5 @@ def is_exhibit_lesser_of_redundant(reimb_term: str, exhibit_lesser_of: str, file prompt = investment_prompts.CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of) response = llm_utils.invoke_claude(prompt, "haiku_latest", filename=filename) - # logging.debug(f"LLM response for redundancy check in {filename}: {response}") + logging.debug(f"LLM response for redundancy check in {filename}: {response}") return string_utils.extract_text_from_delimiters(response, Delimiter.PIPE).strip().upper() == "YES" \ No newline at end of file diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index 7ae3b50..c274187 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -64,7 +64,7 @@ def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]: text_dict, filename ) exhibit_pages, all_exhibit_headers = preprocessing_funcs.link_exhibit_pages( - all_exhibit_headers, filename + all_exhibit_headers, text_dict, filename ) exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit( text_dict, exhibit_pages diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index f9d49a6..a86d1d2 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -6,6 +6,7 @@ import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from src import config, keywords from src.regex.regex_patterns import PIPE_PATTERN +from src.investment import table_funcs from src import config, keywords @@ -259,7 +260,7 @@ def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> tuple[list[st return exhibit_pages, exhibit_header_dict -def link_exhibit_pages(all_exhibit_headers, filename): +def link_exhibit_pages(all_exhibit_headers: dict[str, str], text_dict: dict[str, str], filename: str): """ Filters exhibit pages to include only those with meaningfully different headers. This function processes exhibit headers sequentially and uses literal comparison and @@ -270,6 +271,7 @@ def link_exhibit_pages(all_exhibit_headers, filename): This function will link contiguous header pages together, so that the final list of exhibit pages will only contain pages that are meaningfully different from the previous page. E.g. "1" with header {"Exhibit 1"} and "2" with header {"Continued exhibit 1"} will be linked together, but "3" with header {"Exhibit 2"} will not be linked to either of them. Args: all_exhibit_headers (dict[str, str]): A dictionary mapping page numbers (as strings) to their exhibit headers. Expected to be in chronological order + text_dict (dict[str, str]): A dictionary mapping page numbers to their text content. filename (str): The name of the file being processed, used for logging and LLM invocation. Returns: tuple[list[str], dict[str, str]]: A tuple containing: @@ -296,12 +298,20 @@ def link_exhibit_pages(all_exhibit_headers, filename): final_exhibit_headers[page_num] = page_header first_exhibit = False else: - # If literally identical, skip prompt - if page_header.upper().strip() == previous_header.upper().strip(): + # Check if this page has a substantial table + has_table = has_substantial_table(text_dict.get(page_num, "")) + + # If literally identical, skip prompt (unless it has a table) + if page_header.upper().strip() == previous_header.upper().strip() and not has_table: exhibit_is_different = "N" elif string_utils.is_empty(previous_header): exhibit_is_different = "Y" previous_header = page_header + elif has_table: + # Always treat pages with substantial tables as different exhibits + logging.debug(f"Page {page_num} has substantial table, treating as separate exhibit") + exhibit_is_different = "Y" + previous_header = page_header else: prompt = investment_prompts.EXHIBIT_LINKAGE(page_header, previous_header) claude_answer_raw = llm_utils.invoke_claude(prompt, "sonnet_latest", filename) @@ -316,6 +326,33 @@ def link_exhibit_pages(all_exhibit_headers, filename): return final_exhibit_pages, final_exhibit_headers +def has_substantial_table(page_text: str) -> bool: + """Check if a page contains a substantial table worth preserving as a separate chunk. + + Args: + page_text (str): The text content of the page to check. + + Returns: + bool: True if the page contains a substantial table, False otherwise. + """ + if not page_text or table_funcs.START_MARKER not in page_text: + return False + + # Count table rows to determine if it's substantial + table_start = page_text.find(table_funcs.START_MARKER) + table_end = page_text.find(table_funcs.END_MARKER) + + if table_start == -1 or table_end == -1: + return False + + table_content = page_text[table_start:table_end] + + # Count rows (each row is a list in the table representation) + row_count = table_content.count("'], ['") # Rough count of table rows + + return row_count >= 5 # Right now, using "5 rows" as "substantial". Alternative: use config.TABLE_ROW_LIMIT + + def chunk_by_exhibit(text_dict: dict, exhibit_pages: list ) -> dict: diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index a208631..eff6b73 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -27,7 +27,7 @@ "field_name": "EXHIBIT_LESSER_OF_STATEMENT", "relationship": "one_to_n", "field_type": "exhibit_level", - "prompt": "Find any 'lesser of' statement in this exhibit that applies to multiple services. If you find one, complete this sentence by filling in the blank:\n\n'The payment is the lesser of the exhibit rates or ________'\n\nReturn ONLY what goes in the blank. Examples:\n- If text says 'lesser of rates or 100% of billed charges' → return '100% of billed charges'\n- If text says 'lesser of fee schedule or Provider's charges' → return 'Provider's charges'\n- If text says 'lesser of contracted rate or Medicare' → return 'Medicare'\n\nReturn ONLY the constraint value that completes the sentence.\n- If text says 'Except as otherwise provided in this Exhibit, the Allowed Amount for Covered Services shall be the lessor of (i) the rates set forth in this Exhibit or (ii) 100% of Provider's billed charges.' → return '100% of Provider's billed charges'.\n If no lesser of statement exists, return N/A." + "prompt": "Look for exhibit-wide payment constraints that apply universally to ALL services in this exhibit, regardless of service category or line of business.\n\nValid exhibit-wide constraints have language like:\n- 'All services under this exhibit are subject to the lesser of...'\n- 'Except as otherwise provided in this Exhibit, the Allowed Amount for Covered Services shall be the lesser of...'\n- 'Payment for any service in this exhibit shall not exceed...'\n- 'Unless specifically stated otherwise, all reimbursements are limited to the lesser of...'\n\nDO NOT extract constraints that:\n- Apply only to specific lines of business (e.g., 'Commercial HMO services are lesser of...')\n- Apply only to specific service categories (e.g., 'Supplies are lesser of...', 'Medications are lesser of...')\n- Are part of individual service methodologies\n- Are introduced by category-specific headers\n\nIf you find a true exhibit-wide constraint, return ONLY the constraint value (what comes after 'lesser of the exhibit rates or'). \n\nExamples:\n- 'Except as otherwise provided in this Exhibit, payment shall be the lesser of contracted rates or Provider's billed charges' → return 'Provider's billed charges'\n- 'All services are subject to the lesser of fee schedule or 100% of charges' → return '100% of charges'\n\nIf no exhibit-wide constraint exists, return 'N/A'." }, { "field_name": "RATE_ESCALATOR_STATEMENT", @@ -45,7 +45,7 @@ "field_name": "AARETE_DERIVED_REIMB_METHOD", "relationship": "one_to_n", "field_type": "methodology_breakout", - "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'\n- In case of MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC/ OPPS or ASC, return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP)\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'\n- Provider interest rates are not considered 'Flat Rate'. \n If no specific rule is matched, map to the most appropriate value from the given valid values.", + "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'\n- In case of MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC/ OPPS or ASC, return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP)\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'\n- Provider interest rates are not considered 'Flat Rate'.\n- Charge master increase caps or limitations on provider pricing behavior are NOT reimbursement methodologies and should be ignored.\n\nIf no specific rule is matched, map to the most appropriate value from the given valid values.", "valid_values": "VALID_REIMB_TERM_FULL" }, { diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index ff7c642..3702785 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -330,7 +330,6 @@ VALID reimbursement methodologies include: - "Lesser of" or "greater of" statements with rates - Capitation rates (PMPM, PMPY, per member costs) - Flat rates per service or episode -- Risk-sharing arrangements with specific rates INVALID entries (return NO): - Service definitions without payment terms @@ -341,6 +340,9 @@ INVALID entries (return NO): - Non-reimbursement statements (e.g., "not payable", "not covered", "excluded", "no reimbursement") - Processing instructions that reference external methodologies without specific rates - Claims processing procedures that don't specify actual payment amounts +- Risk-sharing arrangements, surplus/deficit distributions, or performance bonuses +- Financial reconciliation mechanisms or administrative compensation +- Settlement payments based on overall cost performance Examples: ✅ "reimbursed at 100% of DMAP fee schedule" → YES (clear payment method) @@ -350,6 +352,7 @@ Examples: ❌ "Standard eyeglass lenses as defined by DMAP are CR39 lenses..." → NO (definition only) ❌ "Contact lenses must be prior authorized" → NO (authorization only) ❌ "Not payable per contract" → NO (no reimbursement) +❌ "payment to IPA equal to fifty percent (50%) of the Surplus" → NO (risk-sharing distribution) ❌ "Community shall process Clean Claims based on the then current Medicare or Medicaid Fee Schedule cost index as applicable to services rendered and according to Medicare reimbursement methodology, Including the geographic practice (GPCI), or Medicaid reimbursement methodology as applicable." → NO (definition only) Briefly explain your reasoning, then return YES or NO in |pipes|.""" @@ -744,27 +747,31 @@ Please analyze this date: """ def CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of): - return f"""You are an expert in healthcare reimbursement terms. -Determine if the following exhibit-level lesser-of statement is redundant given the reimbursement methodology. + return f"""You are analyzing whether an exhibit-level lesser-of constraint is semantically redundant with a reimbursement methodology. -REIMBURSEMENT METHODOLOGY: -{reimb_term} +REIMBURSEMENT METHODOLOGY: {reimb_term} -EXHIBIT-LEVEL LESSER-OF CONSTRAINT: -{exhibit_lesser_of} +EXHIBIT CONSTRAINT: {exhibit_lesser_of} -Determine if the exhibit constraint is already covered by explicit language in the reimbursement methodology. Look for: -- "At no time shall Plan pay an amount that exceeds Provider's billed charges" -- "Not to exceed Provider's charges" -- "Lesser of [rate] or Provider's charges" -- Similar limiting language that caps payment at provider charges +Determine if the exhibit constraint is already covered by the reimbursement methodology. -Return YES if the exhibit constraint is redundant (already covered). -Return NO if the exhibit constraint adds new information. +CRITICAL: Pay careful attention to NUMERICAL DIFFERENCES. Different percentages (e.g., 90% vs 100%) are NOT redundant even if they both reference billed charges. -Feel free to explain your reasoning, but the final answer should be either YES or NO, delimited by pipes (|) like this: +Examples of REDUNDANT language: +- Reimb: "lesser of fee schedule or billed charges" + Exhibit: "not to exceed billed charges" → REDUNDANT +- Reimb: "payment shall not exceed provider's charges" + Exhibit: "lesser of rate or charges" → REDUNDANT +- Reimb: "100% of billed charges not to exceed usual and customary" + Exhibit: "not to exceed billed charges" → REDUNDANT -|YES| or |NO|""" +Examples of NON-REDUNDANT language: +- Reimb: "100% of billed charges" + Exhibit: "90% of billed charges" → NOT REDUNDANT (90% is more restrictive than 100%) +- Reimb: "billed charges not to exceed usual and customary" + Exhibit: "90% of billed charges" → NOT REDUNDANT (90% is a specific cap vs general UCR limit) +- Reimb: "fee schedule rate" + Exhibit: "90% of billed charges" → NOT REDUNDANT (different constraint types) + +IMPORTANT: If the exhibit constraint introduces ANY new numerical limit, percentage, or dollar amount not present in the reimbursement methodology, it is NOT redundant. + +Answer YES only if the exhibit constraint adds NO additional limitation beyond what's already in the reimbursement methodology. + +Briefly explain your reasoning, then return YES or NO in |pipes|.""" def DERIVED_TERM_DATE_PROMPT(effective_date:str=None, termination_information:str=None) -> str: