From f01fa147f88d0242ab1777c646c6c34598a7cc6a Mon Sep 17 00:00:00 2001 From: Faizan Mohiuddin Date: Thu, 23 Apr 2026 16:19:24 +0000 Subject: [PATCH] Merged in feature/DAIP2-pacificsource-reimbursements-issues (pull request #982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Feature/DAIP2 pacificsource reimbursements issues * Tighten PREMIUM_TERM and DISCOUNT_TERM classifier prompts CARVEOUT_CHECK was misrouting table rate rows into special-case fields, dropping them from the reimbursement output: - "110% of CMS allowed" (base fee-schedule rates) was being classified as PREMIUM_TERM because the prompt treated "above 100% of reference" as an implicit premium. Seen on PacificSource Medicare_Attachment_A1 and A2 Facility contracts where Inpatient/Outpatient rows were missing (2556) or silently fell back to 100% fee-schedule (2557). - Per-service discount rates like "Progressive Lenses: 15% discount", "Contact Lenses: 2% discount", "Frame: 20% discount" were being classified as DISCOUNT_TERM because the prompt only required the word "discount" to appear. Seen on PacificSource Commercial_Attachment_A2 and A5 Professional contracts (2558, 2559). Prompts now require the literal keyword AND explicitly exclude the common false-positive patterns, keeping per-service rate rows in the reimbursement output. * Stop splitting multi-page exhibits on repeated page footers Two bugs were collaborating to split a single Attachment into several Exhibit objects, causing intra-exhibit lesser-of search to miss notes living on a later page of the same exhibit: 1. EXHIBIT_HEADER extraction picked up page-footer lines as if they were new section headers. On PacificSource contracts, pages end with a short-form repeat like "Attachment A Oregon Health & Science University" next to the page number and contract date, which the ATTACHMENT-prefix regex and the LLM both accepted as a header. 2. EXHIBIT_HEADER_DEDUP parsing required the LLM output to be wrapped in a strict |pipes| JSON block. When the LLM prefixed the response with prose ("# Analysis ..."), parsing threw ValueError and the code fell back to the original, un-deduped dict — defeating dedup entirely. Combined, these produced 3 Exhibit objects for Commercial_Attachment_A2 (page 1 header + two entries on page 2: repeated header + footer), leaving … * Merged dev into feature/DAIP2-pacificsource-reimbursements-issues Approved-by: Katon Minhas --- src/pipelines/saas/prompts/prompt_calls.py | 10 ++++------ src/prompts/investment_prompts.json | 4 ++-- src/prompts/prompt_templates.py | 1 + 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index f760ada..ae17891 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -5,9 +5,8 @@ import src.config as config import src.prompts.prompt_templates as prompt_templates from src.constants.constants import Constants from src.constants.investment_columns import FIELD_FORMAT_MAPPING -from src.constants.delimiters import Delimiter from src.prompts.fieldset import Field, FieldSet -from src.utils import llm_utils, string_utils +from src.utils import llm_utils, string_utils, json_utils from src.pipelines.shared.preprocessing import preprocessing_funcs from src.utils.formatting_utils import normalize_field_value from src.pipelines.shared.extraction import tin_npi_funcs @@ -650,10 +649,9 @@ def prompt_header_deduplication( ) try: - llm_answer_final = string_utils.extract_text_from_delimiters( - llm_answer_raw, Delimiter.PIPE - ) - deduped_dict = json.loads(llm_answer_final) + # parse_json_dict tolerates prose before/after the JSON body and + # pipe/brace wrappers, unlike strict pipe-delimiter extraction. + deduped_dict = json_utils.parse_json_dict(llm_answer_raw) if not isinstance(deduped_dict, dict) or not deduped_dict: logging.warning( diff --git a/src/prompts/investment_prompts.json b/src/prompts/investment_prompts.json index e15d08c..0e30ffe 100644 --- a/src/prompts/investment_prompts.json +++ b/src/prompts/investment_prompts.json @@ -676,7 +676,7 @@ "field_name": "DISCOUNT_TERM", "relationship": "one_to_n", "field_type": "special_case_check", - "prompt": "Describes a discount applied to the payment. The description must include the term 'Discount' and should specify the discount amount, percentage, or type.", + "prompt": "Describes a contract-wide or methodology-wide discount applied to the standard reimbursement rate (not a per-service rate expressed using discount language). The reimbursement text MUST contain the literal word 'discount' (case-insensitive). Do NOT classify as DISCOUNT_TERM when the discount IS the base rate for a specific service in a rate table — for example, table rows like 'Contact Lenses: 2% discount', 'Frame: 20% discount', or 'Progressive Lenses: 15% discount' are per-service base rates (classify as PAYMENT_CARVEOUT or BASE_COVERED_SERVICES, NOT DISCOUNT_TERM). Only classify as DISCOUNT_TERM when the discount is a separate provision that reduces payment across all services or the fee schedule overall, e.g., 'a 10% discount applies to all Covered Services', 'payments will be discounted by 5% across the board', or 'provider agrees to a 3% prompt-pay discount'.", "breakout_template": "DISCOUNT_BREAKOUT" }, { @@ -707,7 +707,7 @@ "field_name": "PREMIUM_TERM", "relationship": "one_to_n", "field_type": "special_case_check", - "prompt": "Describes a premium to the payment. Look for the term 'Premium'. Classify as a premium only if the text explicitly indicates an increase or addition to the standard payment rate, such as 'an additional premium of X%', 'a premium rate of Y% above standard', or 'premium payment terms apply'.", + "prompt": "Describes an explicit premium or surcharge on top of a base payment. The reimbursement text MUST contain the literal word 'premium' (case-insensitive) to qualify. Do NOT classify as PREMIUM_TERM merely because a rate is expressed as a percentage above 100% of a reference schedule — for example, '110% of CMS allowed', '105% of Medicare', '120% of the fee schedule', or similar are base reimbursement rates (BASE_COVERED_SERVICES), NOT premiums. Only classify as PREMIUM_TERM when the contract language explicitly uses premium/surcharge terminology, such as 'an additional premium of X%', 'a premium rate of Y% above standard', 'premium payment terms apply', or 'Z% premium added to the base rate'.", "breakout_template": "PREMIUM_BREAKOUT" }, { diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 6022c25..94faa01 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -2118,6 +2118,7 @@ Rules for Exclusion: * Do NOT include docusign IDs and other metadata * Do NOT include partial sentences or continuation text * Do NOT include form field labels like "Provider's Legal Name" or "Authorized Representative's Signature" +* Do NOT include page-footer text. Footers appear near the bottom of a page adjacent to page numbers, contract/reference numbers, effective dates, or docusign IDs, and often repeat a SHORTENED form of the real exhibit identifier. For example, if the top-of-page header is "ATTACHMENT A-2 Oregon Health & Science University Effective 07/01/2025 Reimbursement Schedule", a bottom-of-page line like "Attachment A Oregon Health & Science University" next to "Page 2" or a contract number IS a footer, NOT a new section header — exclude it. When the same chunk contains a long-form exhibit header AND a short-form repeat of the same exhibit identifier near page metadata, keep only the long-form header. [OUTPUT FORMAT] Return a JSON dictionary enclosed in |pipes| where: