From 0253d7dc4afe2203dc32c64c5c7e43fdf6d3d8e1 Mon Sep 17 00:00:00 2001 From: Venkatakrishna Reddy Avula Date: Thu, 2 Apr 2026 18:09:32 +0000 Subject: [PATCH] Merged in bugfix/DAIP2-2164-reimbursement-primary-issue-fixes (pull request #929) Bugfix/DAIP2-2164 reimbursement primary issue fixes * sample calculations reimbursements removed * split service term added * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * updated prompt and unit test added * added prompt for codes listed below * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Output format at end * Genericize prompt * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * context missing issue fixed * updated service term split prompt for program varints * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * updated split service term prompt * optimized and minimized split service term prompt * updated service term split prompt * Merged DEV into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Merged dev into bugfix/DAIP2-2164-reimbursement-primary-issue-fixes * Updated Prompt * grouped instructions * removed print statements Approved-by: Katon Minhas --- src/pipelines/saas/file_processing.py | 6 + src/pipelines/saas/prompts/prompt_calls.py | 37 ++++ .../shared/extraction/one_to_n_funcs.py | 61 +++++- src/prompts/prompt_templates.py | 195 +++++++++++++++++- src/tests/test_prompt_calls.py | 13 ++ 5 files changed, 310 insertions(+), 2 deletions(-) diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index c965ddc..1acb11c 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -155,6 +155,12 @@ def process_file(file_object, constants: Constants, run_timestamp): f"{datetime_str()} Fields not configured for One to One, Skipping - {filename}" ) + # split the rows based service term and duplicate the other fields accordingly + if not final_results.empty and "SERVICE_TERM" in final_results.columns: + with timing_utils.timed_block("split_service_terms", context=filename): + final_results = one_to_n_funcs.split_service_terms(final_results, filename) + logging.debug(f"{datetime_str()} Split Service Terms Complete - {filename}") + # APPLY CODES IF ONE_TO_N WAS PROCESSED if not one_to_n_results.empty: with timing_utils.timed_block("code_processing", context=filename): diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index 2a4e1a6..582f4b2 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -1281,3 +1281,40 @@ def prompt_aarete_derived_provider_name( ) fallback = cluster_rows[0].get("base_name", "UNKNOWN") return str(fallback).upper() + + +def prompt_split_service_term(service_term: str, filename: str) -> list[str]: + """ + Split a service term into its components (e.g., service description and modifiers). + + Args: + service_term (str): The full service term to be split. + filename (str): The filename to be used for logging or caching purposes. + + + Returns: + list[str]: A list containing the distinct terms of the service term. + """ + prompt, _parser = prompt_templates.SPLIT_SERVICE_TERM(service_term) + + llm_answer_raw = llm_utils.invoke_claude( + prompt, + model_id="haiku_latest", + filename=filename, + cache=True, + instruction=prompt_templates.SPLIT_SERVICE_TERM_INSTRUCTION(), + usage_label="SPLIT_SERVICE_TERM", + ) + logging.debug( + f"Raw LLM response for SPLIT_SERVICE_TERM on '{service_term}': {llm_answer_raw}" + ) + try: + split_result = _parser(llm_answer_raw) + logging.debug( + f"Parsed SPLIT_SERVICE_TERM result for '{service_term}': {split_result}" + ) + return split_result if isinstance(split_result, list) else [] + except Exception as e: + logging.error(f"Error parsing LLM response for SPLIT_SERVICE_TERM: {str(e)}") + logging.error(f"Raw response: {llm_answer_raw}") + return [] diff --git a/src/pipelines/shared/extraction/one_to_n_funcs.py b/src/pipelines/shared/extraction/one_to_n_funcs.py index ac37b3f..7644c22 100644 --- a/src/pipelines/shared/extraction/one_to_n_funcs.py +++ b/src/pipelines/shared/extraction/one_to_n_funcs.py @@ -1,6 +1,6 @@ import concurrent.futures import logging - +import pandas as pd from src.pipelines.shared.extraction import dynamic_funcs from src.pipelines.saas.prompts import prompt_calls from src.pipelines.shared.postprocessing import aarete_derived, postprocessing_funcs @@ -976,3 +976,62 @@ def lesser_of_distribution( final_reimbursement_level_answers.append(answer_dict) return final_reimbursement_level_answers + + +def split_service_terms(one_to_n_results: pd.DataFrame, filename: str) -> pd.DataFrame: + """ + Split SERVICE_TERM values into separate rows. + + For each row, calls the LLM to split multiple services (if any), + and creates one row per service term. If splitting fails, keeps + the original value. + + Args: + one_to_n_results (pd.DataFrame): Input DataFrame with SERVICE_TERM column. + filename (str): Used for LLM call context. + + Returns: + pd.DataFrame: DataFrame with SERVICE_TERM as a string in each row. + """ + + if one_to_n_results.empty or "SERVICE_TERM" not in one_to_n_results.columns: + logging.debug( + "`split_service_terms`: No data or SERVICE_TERM column not found, skipping" + ) + return one_to_n_results + + new_rows = [] + + for _, row in one_to_n_results.iterrows(): + service_term = row["SERVICE_TERM"] + + split_terms = [service_term] + + if not string_utils.is_empty(service_term) and any( + sep in service_term for sep in [",", "and", "/", "and/or"] + ): + try: + result = prompt_calls.prompt_split_service_term(service_term, filename) + + if isinstance(result, list) and all( + isinstance(term, str) for term in result + ): + split_terms = result + else: + logging.warning( + f"LLM returned invalid format for SERVICE_TERM: {service_term} - keeping original" + ) + + except Exception as e: + logging.error( + f"Error splitting SERVICE_TERM '{service_term}': {str(e)} - keeping original" + ) + + for term in split_terms: + new_row = row.copy() + new_row["SERVICE_TERM"] = term + new_rows.append(new_row) + + result_df = pd.DataFrame(new_rows).reset_index(drop=True) + + return result_df diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index face582..98d273a 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -148,6 +148,15 @@ def get_cacheable_instructions(): except Exception: pass + # service term split instruction + if config.FIELDS in ["all", "one_to_n"]: + try: + cacheable_instructions["SPLIT_SERVICE_TERM"] = ( + SPLIT_SERVICE_TERM_INSTRUCTION() + ) + except Exception: + pass + # OUTLIER_BREAKOUT instruction if config.FIELDS in ["all", "one_to_n"]: try: @@ -602,6 +611,11 @@ REIMB_TERM: Describes the method by which the price of the Service is calculated - Include all SERVICE_TERM and REIMB_TERM pairs mentioned, even if they are redundant or overlapping. There should be at least one entry for each SERVICE_TERM and REIMB_TERM pair mentioned. - If different reimbursement percentages or dollar values are mentioned for different effective dates, create separate entries for each effective date. - For the tables: Extract EVERY ROW as a separate entry, including all line items, subtotals, totals, adjustments, charges, calculated ratios, and all relevant information from every column in that row. ALWAYS include column headers and subheaders as part of the SERVICE_TERM to provide full context for the service, forming a complete sentence if necessary. If a row contains multiple reimbursement rate types (e.g., different rate columns such as facility vs non-facility, professional vs technical, inpatient vs outpatient, etc.), create separate entries for EACH rate type using the corresponding rate column header in the SERVICE_TERM. When rows involve different providers/entities, create separate entries for EACH provider/entity and include the provider/entity name in the SERVICE_TERM (e.g., "Provider Name – Service Category"). +- If a reimbursement statement appears either BEFORE or AFTER the table and applies to all listed codes (e.g., "for all codes listed below" or "for all codes listed above" or "codes identified below"), then: + • Extract each table row as a separate entry, AND + • Apply that statement to EVERY row by incorporating it into the REIMB_TERM for each entry. + • Ensure the REIMB_TERM for each row reflects this shared reimbursement language exactly as written. + Example: If the statement says "For all codes listed below, reimbursement is 100% of Medicare," then each row must have REIMB_TERM = "100% of Medicare". - GROUPING RULE: If multiple services are explicitly listed together (e.g., "Reference Lab and DME") AND share the same rate statement, extract as ONE entry preserving the exact combined SERVICE_TERM. Do not split them. - SEPARATION RULE: If services appear in separate sentences or have different reimbursement methods, extract as separate entries. - ENUMERATED SUB-CASES: When a paragraph introduces a service category and then lists sub-cases (i, ii, iii or a, b, c or 1, 2, 3) with different rates, extract each sub-case as a separate entry. For SERVICE_TERM, use the overall service description with only a brief sub-case label (e.g., "primary procedure", "second and subsequent procedures") - do NOT include billing logic, conditional clauses, or payment-determination language in the SERVICE_TERM. For REIMB_TERM, extract only the payment method (rate, percentage, dollar amount) - do NOT include the conditional clause that precedes it. @@ -611,7 +625,7 @@ REIMB_TERM: Describes the method by which the price of the Service is calculated • "Surgical Services: 150% of Medicare" → ONE separate entry [EXCLUSIONS] -- NEVER return reimbursement terms that are labeled as Examples or appear in an EXAMPLE section. These are not actual reimbursements and should not be included in the output. +- NEVER return reimbursement terms that are labeled as Examples,appear in an EXAMPLE section or presented as the sample calculations. These are not actual reimbursements and should not be included in the output. - NEVER return reimbursements from the Reciprocity Agreements section of the contract. These are not actual reimbursements and should not be included in the output. [SPECIAL CASES] @@ -2684,3 +2698,182 @@ STATE_FLAG: """ return (prompt, _json_dict_parser) + + +def SPLIT_SERVICE_TERM( + service_term: str, +) -> Tuple[str, Callable[[str], list]]: + """Returns prompt for splitting a SERVICE_TERM into components. + Call SPLIT_SERVICE_TERM_INSTRUCTION() separately for the cached instruction. + + Args: + service_term (str): The SERVICE_TERM to be split.""" + + prompt = f""" + Analyze the following SERVICE_TERM:{service_term} + """ + return (prompt, _json_list_parser) + + +def SPLIT_SERVICE_TERM_INSTRUCTION() -> str: + return f""" + [OBJECTIVE] + Identify and split a SERVICE_TERM into separate Services only when it clearly represents + multiple independent medical Services. + + [INPUT] + A single SERVICE_TERM string. + + [DEFINITION] + SERVICE_TERM: Describes the medical service or procedure being reimbursed. + It can range from very general (e.g., "Covered Services") to very specific + (e.g., "CPT 98765"), and may include qualifiers such as program names, billing + codes, reimbursement conditions, visit types, or facility names. + + [RULES FOR SPLITTING] + - Only split when there are multiple clearly independent Services present. A Service is + defined as a medical procedure or category of medical procedures. + - Each output item must represent a distinct, standalone Service. + - Preserve original wording exactly (no rephrasing, no expanding abbreviations). + + [MANDATORY: DO NOT SPLIT THESE CASES] + + 1. **Closely Related or Paired Services** + These are terms that are grouped because they are near-synonyms, delivery variants, + or complementary parts of a single clinical service concept. Do not split them. + + Examples included under this case: + - Therapy disciplines bundled together as a single offerings block + (e.g., Physical Therapy, Occupational Therapy, Speech Therapy). Therapeutic program models + that are grouped in a single SERVICE_TERM listing should generally be kept together, not split, if they + represent a coherent care bundle. + - Intensive or program variants of the same foundational model when grouped as one + phrase (e.g., Residential and Intensive variants under a single program category) + - Delivery setting pairs (e.g., Inpatient and Outpatient Services) + - Closely related procedure variants (e.g., Ambulatory Surgical Center and + Hospital Ambulatory Surgical Center services) + - Provider role combinations (e.g., Physician Assistant and Nurse Practitioner services) + - Facility/setting listings (e.g., Behavioral Health Outpatient Clinics/Integrated Clinics, Community Service Agencies) + + 2. **Code Category Descriptions** + When a SERVICE_TERM begins with a code category prefix (such as a billing code + family or procedure code set, HCPCS codes), the items listed after it describe what + falls under that category — not separate services. Do not split. + + 3. **Procedure Groups Tied to Shared Codes or Groupers** + When a SERVICE_TERM contains multiple procedures tied together by shared + billing codes, diagnosis-related groupers, or other classification codes, + do not split. Splitting would sever the reimbursement context. + + 4. **Enumerated Descriptions of a Single Service Concept** + If a comma-separated list is describing components, sub-items, or examples + of a single named service or code group, do not split. The list is clarifying + what is included, not defining separate services. + + 5. **Lines of Business, Programs, or Plan Names** + Do not split when listed items refer to Lines of Business, Programs, Plan names, + or Program-specific service variants rather than distinct medical services. + + 6. **Service and Delivery Modifier Combinations** + If one term modifies how or where another service is delivered (e.g., a setting, + mode, or method qualifier), treat the full phrase as a single service rather + than splitting the modifier from the service. + + 7. **Facility or Setting Types Listed Together** + If the listed items describe types of facilities, clinic settings, or agency + types rather than distinct medical services, do not split. The list is + describing where or through whom a service is delivered, not separate services. + This applies even when a program name or reimbursement prefix precedes the list. + + [WHEN SPLITTING IS APPROPRIATE] + Only split when the items are clearly and independently distinct medical services + that would be separately contracted or reimbursed, and where no shared context, + code grouping, or qualifier is lost by separating them. + + [CRITICAL CONTEXT PRESERVATION] + - DO NOT drop important qualifiers such as: + • Conditions (e.g., "paid when item is over $500") + • Billing codes, procedure codes, diagnosis groupers, or numeric ranges + • Facility names or locations + • Program or plan names + • Visit type qualifiers (e.g., "Per Visit", "Inpatient") + • Payment rules or reimbursement conditions + - If such context applies to all split Services, APPEND the shared context to + each split item. + - If context applies only to part of the string, retain it with the most + relevant Service. + - Never return overly reduced terms that lose meaning. + + [CLEANING] + - Trim whitespace and unnecessary punctuation only. + - Do not alter wording beyond splitting. + + Input: "Home Health, Hospice, Skilled Nursing Facility, Dialysis" + Output: ["Home Health", "Hospice", "Skilled Nursing Facility", "Dialysis"] + Reason: Four clearly distinct, independently reimbursable service categories. + + Input: "Heart and Soul Hospice LLC home hospice Covered Services" + Output: [ + "Heart and Soul Hospice LLC Home Covered Services", + "Heart and Soul Hospice LLC Hospice Covered Services" + ] + Reason: Two distinct services (home and hospice) with shared context (Heart and Soul Hospice LLC, Covered Services) — split and append shared context to each. + + Input: "TFC, ITFC, and Multi Systemic Therapy" + Output: ["TFC, ITFC, and Multi Systemic Therapy"] + Reason: Therapy bundle — do not split. + + Input: "Billing Code Group A: medical and surgical supplies, diabetic supplies and shoes, breast pumps" + Output: ["Billing Code Group A: medical and surgical supplies, diabetic supplies and shoes, breast pumps"] + Reason: Items after a code prefix describe scope of one coded SERVICE_TERM — do not split. + + Input: "HCPCS E-Codes oxygen and oxygen equipment, ventilators, CPAP/BiPAP, nebulizers/compressors/humidifiers" + Output: ["HCPCS E-Codes oxygen and oxygen equipment, ventilators, CPAP/BiPAP, nebulizers/compressors/humidifiers"] + Reason: Items after a code prefix describe scope of one coded SERVICE_TERM — do not split. + + Input: "Craniotomy and Endovascular Intracranial Procedures Diagnosis-Related Group 025, 026, 027" + Output: ["Craniotomy and Endovascular Intracranial Procedures Diagnosis-Related Group 025, 026, 027"] + Reason: Procedures share grouper codes — splitting would sever reimbursement context. + + Input: "CHIP and STAR Outpatient Services" + Output: ["CHIP and STAR Outpatient Services"] + Reason: Program names, not distinct services. + + Input: "Inpatient and Outpatient Services" + Output: ["Inpatient and Outpatient Services"] + Reason: Delivery setting variants of the same service — not independently distinct. + + Input: "Physician Assistant and Nurse Practitioner services for Health Benefit Exchange" + Output: ["Physician Assistant and Nurse Practitioner services for Health Benefit Exchange"] + Reason: Provider types, not service types — do not split. + + Input: "Ambulatory Surgical Center and Hospital Ambulatory Surgical Center services" + Output: ["Ambulatory Surgical Center and Hospital Ambulatory Surgical Center services"] + Reason: Closely related facility type variants of the same service — do not split. + + Input: "Behavioral Health Outpatient Clinics/Integrated Clinics, Community Service Agencies" + Output: ["Behavioral Health Outpatient Clinics/Integrated Clinics, Community Service Agencies"] + Reason: Facility and agency setting types, not distinct reimbursable services. + + Input: "facility and professional Covered Services rendered to a Covered Person" + Output: ["facility and professional Covered Services rendered to a Covered Person"] + Reason: Professional and facility variants of the same service category — do not split. + + Input: "DDD Reimbursement - Behavioral Health Outpatient Clinics/Integrated Clinics, Community Service Agencies" + Output: ["DDD Reimbursement - Behavioral Health Outpatient Clinics/Integrated Clinics, Community Service Agencies"] + Reason: Program-prefixed list of facility settings — not distinct services. + + Input: "Primary and Specialty Care Services for Medicaid Managed Care, Health And Recovery Plan and Essential Plan Program" + Output: [ + "Primary Care Services for Medicaid Managed Care, Health And Recovery Plan and Essential Plan Program", + "Specialty Care Services for Medicaid Managed Care, Health And Recovery Plan and Essential Plan Program" + ] + Reason: Two distinct clinical service categories sharing trailing program context — split and append shared context to each. + + [FINAL GUIDANCE] + If uncertain, do NOT split. The bar for splitting should be high: items must be + clearly independent medical services that stand alone without losing meaning or context. + + [OUTPUT FORMAT] + {JSON_LIST_FORMAT_INSTRUCTIONS} + """ diff --git a/src/tests/test_prompt_calls.py b/src/tests/test_prompt_calls.py index e07f848..6f96854 100644 --- a/src/tests/test_prompt_calls.py +++ b/src/tests/test_prompt_calls.py @@ -985,6 +985,19 @@ class TestPromptCalls(unittest.TestCase): f"[CONTEXT]\n{self.exhibit_text}", ) + @patch("src.utils.llm_utils.invoke_claude") + def test_prompt_split_service_term(self, mock_invoke): + """Test that prompt_split_service_term correctly splits service term.""" + mock_invoke.return_value = '["Service1", "Service2"]' + + result = prompt_calls.prompt_split_service_term( + "Service1 and Service2", self.filename + ) + + # Should return list of split service terms + self.assertIsInstance(result, list) + self.assertEqual(result, ["Service1", "Service2"]) + if __name__ == "__main__": unittest.main()