diff --git a/src/constants/regex_patterns.py b/src/constants/regex_patterns.py index 3c2b2a7..e82d3e5 100644 --- a/src/constants/regex_patterns.py +++ b/src/constants/regex_patterns.py @@ -112,7 +112,7 @@ EFFECTIVE_DATE_FORMAT_PATTERNS = [ # dba patterns DBA_PATTERNS = [r"\bD/B/A\b", r"\bDBA\b", r"\bDOING BUSINESS AS\b", r"\bD B A\b"] -# Exhibit Smart Chunking — Section Header Patterns +# Exhibit Smart Chunking - Section Header Patterns # Used to identify section/exhibit header boundaries when splitting exhibit text into chunks. # Matches numbered sections, lettered sections, roman numerals, ARTICLE/SECTION/EXHIBIT keywords, # legal preambles, and attachment/addendum/schedule markers. @@ -124,7 +124,7 @@ SECTION_HEADER_PATTERNS = [ # Roman numerals: I., II., III., IV., etc. r"(?:^|\n)\s*([IVXivx]+\.)\s+", # ARTICLE headers: ARTICLE ONE, ARTICLE 1, ARTICLE I - r"(?:^|\n)\s*(ARTICLE\s+(?:ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|\d+|[IVX]+)(?:\s*[-–—]\s*[A-Z][A-Z\s]+)?)", + r"(?:^|\n)\s*(ARTICLE\s+(?:ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN|\d+|[IVX]+)(?:\s*[-–-]\s*[A-Z][A-Z\s]+)?)", # SECTION headers: SECTION 1, Section 1.1 r"(?:^|\n)\s*(SECTION\s+\d+(?:\.\d+)*)", # Legal preambles diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 594bac2..5c0a849 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -324,7 +324,7 @@ def process_page_reimbursements( individually through reimbursement_level. If empty/None, page is skipped. """ - # No relevant chunks on this page — skip reimbursement extraction + # No relevant chunks on this page - skip reimbursement extraction if not relevant_chunks: logging.debug( f"{datetime_str()} Page {page_num}: No relevant chunks, skipping - {filename}" diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index db9cb8e..d7f8a95 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -963,7 +963,7 @@ def update_grouper_base_rate_and_grouper_pct_rate(df): def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame: """ Ensures the given AARETE_DERIVED_SID columns exist in the DataFrame, fills missing values, - and creates a unique 'AARETE_DERIVED_SID' identifier column — while keeping all original columns. + and creates a unique 'AARETE_DERIVED_SID' identifier column - while keeping all original columns. Steps: 1. Verify that all columns listed in sid_cols exist in the DataFrame. diff --git a/src/prompts/investment_prompts.json b/src/prompts/investment_prompts.json index 5824a66..c32b592 100644 --- a/src/prompts/investment_prompts.json +++ b/src/prompts/investment_prompts.json @@ -40,7 +40,7 @@ "field_name": "REIMB_CONVERSION_FACTOR", "relationship": "one_to_n", "field_type": "methodology_breakout", - "prompt": "What is the conversion factor mentioned in the text? A conversion factor is a dollar amount or multiplier value used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit — RVU) into an actual payment amount. In the case of 'RBRVS', 'RVU', or American Society of Anesthesiology ('ASA'), any dollar amount mentioned refers to a conversion factor, and not to a flat fee rate. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)." + "prompt": "What is the conversion factor mentioned in the text? A conversion factor is a dollar amount or multiplier value used to determine payment for healthcare services, typically under a fee-for-service or Diagnosis-Related Group (DRG) reimbursement model. It serves as a multiplier that converts a relative value (such as a DRG weight or Relative Value Unit - RVU) into an actual payment amount. In the case of 'RBRVS', 'RVU', or American Society of Anesthesiology ('ASA'), any dollar amount mentioned refers to a conversion factor, and not to a flat fee rate. If no conversion factor is mentioned, return 'N/A'. If it is mentioned as a dollar amount, return it as a numeric value only (e.g., 1.25 for $1.25)." }, { "field_name": "UNIT_OF_MEASURE", @@ -297,7 +297,7 @@ "field_name": "EFFECTIVE_DT", "relationship": "one_to_one", "field_type": "smart_chunked", - "prompt": "Please analyze the following contract and extract ONLY the effective date of the CURRENT contract or CURRENT amendment. THE DATE MAY OR MAY NOT BE EXPLICITLY LABELED AS AN EFFECTIVE DATE. Follow these retrieval rules in order:\n\n[RETRIEVAL RULES]\nLOOK FOR dates that appear after these common phrases but not limited to (case-insensitive matching):\n'Effective Date:' FOLLOWED BY a date\n'is entered into as of' FOLLOWED BY a date\n'is made and entered into as of the' FOLLOWED BY a date\n'effective as of' FOLLOWED BY a date\n'shall begin on' FOLLOWED BY a date\n'Effective Date of Agreement:' FOLLOWED BY a date\n'Effective Date of Amendment' FOLLOWED BY a date\n'made this' FOLLOWED BY a date\n'made and entered into' FOLLOWED BY a date\n\nALSO LOOK FOR dates in phrases that semantically indicate an effective date (may or may not be explictly labelled as effective date), even if they do not exactly match the phrases above.\nExamples include but ARE NOT LIMITED TO:\nPhrases indicating when a contract or amendment begins or takes effect, Phrases specifying the start date of the contract or amendment, Phrases defining when the contract or amendment becomes operational, or Any phrase indicating contract or amendment creation or execution date.\n\nIt is possible that the contract will specify that the effective date is derived from elsewhere in the contract. One specific case is outlined:\n\nSIGNATURE DATE HANDLING:\nPRIMARY RULE: DO NOT USE signature dates unless STRICTLY meeting the criteria below\nWHEN TO USE a signature date: ONLY IF that SPECIFIC signature date has an EXPLICIT statement marking IT as the effective date\nHere are some example scenarios:\n✓ USE: 'Effective Date: 1/13/2024' + 'Date: [blank]' below\n✓ USE: 'Effective Date of Agreement: 2/15/2016' + 'Date: [blank]' below\n\nWHEN NOT TO USE a signature date:\n- Even if there is an empty effective date section or field nearby\nHere are some example scenarios but not limited to:\n✗ DO NOT USE: 'Date: 1/1/2024' + 'Effective Date: [blank]' below\n✗ DO NOT USE: 'Effective Date: [blank]' + 'Date: 1/1/2024' below\n\nSECONDARY RULE – UNREFERENCED DATE NEAR “EFFECTIVE DATE:” LABEL:\nIf no explicit effective date is found in the contract, and the “Effective Date:” label appears with a blank or empty field, you MAY consider a nearby date as the effective date ONLY IF ALL of the following conditions are true:\n- The nearby date is NOT referenced by any signature, title, or labeled field (i.e., it is unassigned or free-floating)\n- Other nearby dates ARE explicitly labeled (e.g., under “Date” near signature lines or roles)\n- The unassigned date appears directly under or beside the “Effective Date:” label\n- The document does not provide a clearer effective date elsewhere\n\nIn such cases, treat the unreferenced date near the “Effective Date:” label as the effective date. Preserve the date's exact format as it appears.\n\n[OUTPUT DATE HANDLING]\nCRITICAL - DATE FORMAT PRESERVATION:\n- You MUST preserve the EXACT format of the date AS IT APPEARS in the document\n- DO NOT modify, standardize, or reformat the date in ANY way\n- Preserve ALL original: Spacing (including multiple spaces), punctuation, separators (commas, hyphens, forward slashes, etc.), leading/trailing spaces, or partial or incomplete dates\n\nExamples of EXACT DATE preservation:\n✓ If document shows: '3-1 , , 03' → return \"3-1 , , 03\"\n✓ If document shows: 'June 1st,2010' → return \"June 1st,2010\"\n✓ If document shows: 'Feb 1st day of , 10' → return \"Feb 1st day of , 10\"\n✓ If document shows: '6 /1/ 2010' → return \"6 /1/ 2010\"\n✓ If document shows: '12-1-20' → return \"12-1-20\"\n✓ If document shows: 'made this day of Jan, 2009' → return \"day of Jan, 2009\"\n✓ If document shows: 'to be effective as of , 20 ( the 'Effective Date').' → return \", 20\"\n\nIt is POSSIBLE that the intended effective date could be blank (left blank, not filled), empty, incomplete, or partial. DO NOT ASSUME/INFER anything about the date or its components.\nReturn N/A only if NO valid effective date is found (e.g. cases with blank/empty/incomplete dates/placeholder values).\n\nIF MULTIPLE matches are retrieved:\n- VERIFY that you have PRECISELY followed the retrieval rules\n- PRIORITIZE the effective date of the CURRENT contract or CURRENT amendment over any attachment or exhibit\n- ONLY IF multiple valid effective dates for the CURRENT contract/amendment are found, SELECT the LATEST one\n- MAINTAIN the EXACT FORMAT of the selected date as it appears in the document\n\nBefore providing the final answer, SHOW your reasoning:\n- LIST ALL dates found that match patterns in the retrieval rules\n- For EACH date found:\n* EXPLAIN why it is considered for effective date or not\n* Include the EXACT text snippet where the date was found\n- JUSTIFY your final date selection or why N/A is being returned\n\nFinal Answer Format:\n- Return your answer as a valid JSON dictionary with the key \"EFFECTIVE_DT\" and the value as the exact date string\n- The date value MUST be an EXACT COPY of how the date appears in the document\n- DO NOT add or remove ANY characters, spaces, or formatting\n- Example: {\"EFFECTIVE_DT\": \"3-1 , , 03\"} or {\"EFFECTIVE_DT\": \"N/A\"}", + "prompt": "Analyze the contract and extract the effective date of the CURRENT contract or amendment (not exhibits/reimbursements). The date may not be explicitly labeled.\nRETRIEVAL RULES (in order):\n1. Look for these trigger phrases followed by a date:'Effective Date:', 'is entered into as of', 'effective as of', 'shall begin on', 'Effective Date of Amendment', 'made this', 'made and entered into' - or any phrase semantically indicating when the contract/amendment takes effect, begins, or was created.\n2. Resolve referenced dates: If the effective date is defined by a formula or reference (e.g., \"the earlier of [reference] or [date]\"), locate the actual date(s) the reference points to, substitute them, then apply the formula to determine the final date.\n3. Signature dates: Use ONLY if explicitly marked as the effective date, or if the contract states it is effective as of the earlier of the signature date and another date.\n4. Blank \"Effective Date:\" label: If no explicit date exists but the label appears with a blank field, use a nearby unassigned/free-floating date ONLY IF other nearby dates are explicitly labeled and no clearer date exists elsewhere.\nDATE FORMAT: Return the date EXACTLY as it appears - preserve all spacing, punctuation, and formatting. Do not infer missing components; return incomplete/blank dates as-is. If multiple matches: Prioritize the current contract/amendment over attachments. If still multiple, return the latest.\nReturn \"N/A\" only if no valid date is found.\nOutput: Show reasoning (list all candidate dates, explain each), then return:{\"EFFECTIVE_DT\": \"\"} or {\"EFFECTIVE_DT\": \"N/A\"}", "keywords": [ "effective date", "effective date:", diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index 6d517cb..8017acd 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -592,7 +592,7 @@ REIMB_TERM: Describes the method by which the price of the Service is calculated - Extract ONE entry per RATE STATEMENT in the contract. A rate statement includes percentages, dollar amounts, weight-based calculations (e.g., DRG weight), or other reimbursement methodologies. - GROUPING RULE: If multiple services are explicitly listed together (e.g., "Reference Lab and DME") AND share the same rate statement, extract as ONE entry preserving the exact combined SERVICE_TERM. Do not split them. - SEPARATION RULE: If services appear in separate sentences or have different reimbursement methods, extract as separate entries. -- ENUMERATED SUB-CASES: When a paragraph introduces a service category and then lists sub-cases (i, ii, iii or a, b, c or 1, 2, 3) with different rates, extract each sub-case as a separate entry. For SERVICE_TERM, use the overall service description with only a brief sub-case label (e.g., "primary procedure", "second and subsequent procedures") — do NOT include billing logic, conditional clauses, or payment-determination language in the SERVICE_TERM. For REIMB_TERM, extract only the payment method (rate, percentage, dollar amount) — do NOT include the conditional clause that precedes it. +- ENUMERATED SUB-CASES: When a paragraph introduces a service category and then lists sub-cases (i, ii, iii or a, b, c or 1, 2, 3) with different rates, extract each sub-case as a separate entry. For SERVICE_TERM, use the overall service description with only a brief sub-case label (e.g., "primary procedure", "second and subsequent procedures") - do NOT include billing logic, conditional clauses, or payment-determination language in the SERVICE_TERM. For REIMB_TERM, extract only the payment method (rate, percentage, dollar amount) - do NOT include the conditional clause that precedes it. - Examples: • "Lab and Imaging Services: 110% of Medicare" → ONE entry with SERVICE_TERM "Lab and Imaging Services" • "Emergency Services: $500 per visit" → ONE separate entry @@ -633,12 +633,12 @@ DECISION LOGIC: 0. UNDERSTAND THE TASK We are looking for overarching "lesser of" or "not to exceed" constraints that apply to SERVICE. - METHODOLOGY is the base rate — we may wrap it with constraints we find. + METHODOLOGY is the base rate - we may wrap it with constraints we find. 1. COLLECT APPLICABLE STATEMENTS From cross-exhibit templates (if provided): - → Include these—they're already verified to apply. + → Include these-they're already verified to apply. From intra-exhibit text: → INCLUDE statements referencing "this exhibit", "rates below", "contracted rates" (generic) @@ -670,11 +670,11 @@ DECISION LOGIC: If NO → it's part of the excepted rule and does NOT apply c) ALWAYS re-scan the exhibit text for standalone constraints after excluding the payment rule. - Do not stop at step (a) — even if the main rule is excluded, independent constraints MUST be applied. + Do not stop at step (a) - even if the main rule is excluded, independent constraints MUST be applied. Examples: - SEPARATE sentence: "In no event shall reimbursement exceed the amount billed by Provider." - → Still applies (passes independence test — standalone constraint) + → Still applies (passes independence test - standalone constraint) - SAME sentence: "Compensation shall be the lesser of 80% of the scheduled fee or usual and customary charges, capped at the Provider's submitted amount" → Does NOT apply (entire grammatical unit is the excepted rule) @@ -694,7 +694,7 @@ DECISION LOGIC: Substitution: - Replace generic "rates" / "contracted rates" with METHODOLOGY value - - Preserve original phrasing exactly — do not paraphrase, abbreviate, or alter wording + - Preserve original phrasing exactly - do not paraphrase, abbreviate, or alter wording - Do not include service code or description in the output - Deduplicate identical constraints - Join multiple terms preserving meaning through nesting or other structures: e.g., "[the lesser of [the greater of A or B] and C]", "lesser of X, Y, or Z". Ensure brackets and nesting are used appropriately to maintain the original meaning. @@ -703,7 +703,7 @@ DECISION LOGIC: - The output string MUST contain METHODOLOGY value - If METHODOLOGY is a simple rate (e.g., "$30.00") and is missing from the final string, prepend it - If METHODOLOGY is a complex formula (e.g., "80% of State Medicaid fee schedule", "the lesser of (i) X or (ii) Y"), verify it appears as the base of the output - - The output MUST contain the FULL original METHODOLOGY — every rate, tier, and condition. + - The output MUST contain the FULL original METHODOLOGY - every rate, tier, and condition. The lesser-of constraint wraps METHODOLOGY as an outer ceiling; it never replaces or summarizes it. Before returning, verify METHODOLOGY appears intact. If any part is missing, re-nest: "the lesser of [full METHODOLOGY] or [constraint ceiling]" @@ -714,7 +714,7 @@ DECISION LOGIC: - ["the lesser of [the greater of 100% of the State Medicaid fee schedule or Affinity's standard fee schedule] and billed charges"] - ["N/A"] -Example — Intra-exhibit statement references another exhibit → N/A: +Example - Intra-exhibit statement references another exhibit → N/A: - METHODOLOGY: "per visit basis at payment rates established in Attachment A" - CROSS-EXHIBIT TEMPLATES: None - EXHIBIT TEXT contains: "lesser of payment rates established in Attachment A or 100% of Medicare" @@ -790,7 +790,7 @@ def LESSER_OF_CHECK_INSTRUCTION() -> str: 2. **EXHIBIT_SPECIFIC (CURRENT)** - References only this exhibit - Phrases: "this exhibit", "rates below", "listed below", "set forth below", "in this schedule" - These are templates/preambles that apply to rates defined within the same exhibit - - NOT standalone rates—they reference other rates rather than being complete themselves + - NOT standalone rates-they reference other rates rather than being complete themselves 3. **STANDALONE** - A complete, self-contained reimbursement rate - This IS a rate (a fee schedule entry, a line item) @@ -957,7 +957,7 @@ Analyze a given term from a Payer-Provider contract and extract key fields. [INSTRUCTION] - Always identify **all enforceable reimbursement clauses**, whether they appear as standalone sentences or within a comparative structure. -- When a reimbursement methodology includes **multiple enforceable limits**—for example, "100% of billed charges not to exceed $500"—treat **each distinct limit** as a separate reimbursement methodology. Output one dictionary for "100% of billed charges" and another for "$500 cap". +- When a reimbursement methodology includes **multiple enforceable limits**-for example, "100% of billed charges not to exceed $500"-treat **each distinct limit** as a separate reimbursement methodology. Output one dictionary for "100% of billed charges" and another for "$500 cap". - If a clause includes a **"lesser of" (or similar comparative)** structure: - Carefully extract **every individual reimbursement clause within the comparison**. - For example: "Lesser of (i) 80% of billed charges, (ii) Medicare rate, or (iii) $400" should yield **three separate JSON dictionaries**. @@ -1301,7 +1301,7 @@ Extract explicit procedure, revenue, diagnosis, and other healthcare codes from - If the text describes an entire Code Category (not one specific code) based on a start letter, return the Category in the form "Category: X". e.g. "A-Codes" --> "Category: A", "K-Codes" --> "Category K", etc. - If the text explicitly says that there is no published or established rate, write "NOT_ESTABLISHED" for the PROCEDURE_CD value. - If any of the codes are not found, do not write a list for that field. Simply populate the field with an empty list []. -- If a standalone 3-digit code appears without explicit labels (e.g., "DRG," "revenue," "Rev code"), analyze the surrounding context for clues. Look for any direct or indirect references—no matter how subtle—that may suggest a connection to either a Revenue Code or a Grouper Code. If any contextual clues imply relevance to either classification, categorize the code accordingly. +- If a standalone 3-digit code appears without explicit labels (e.g., "DRG," "revenue," "Rev code"), analyze the surrounding context for clues. Look for any direct or indirect references-no matter how subtle-that may suggest a connection to either a Revenue Code or a Grouper Code. If any contextual clues imply relevance to either classification, categorize the code accordingly. - Return only valid codes that match the stated format (e.g. CPT exactly 5 digits, HCPCS 1 letter + 4 digits); do not invent or guess codes. [FIELDS] @@ -2036,10 +2036,10 @@ Your task is to deduplicate this dictionary so that each unique section appears [RULES FOR IDENTIFYING DUPLICATES] * Two headers belong to the same section if they share the same core section identifier AND the same content-type descriptors after case-normalization. * Minor case variations do NOT make headers different sections. -* Extra sub-lines that are facility names, entity names, or provider names do NOT make it a different section — these are just contextual metadata repeated on continuation pages. +* Extra sub-lines that are facility names, entity names, or provider names do NOT make it a different section - these are just contextual metadata repeated on continuation pages. * However, sub-lines that describe the TYPE OF CONTENT or SERVICE CATEGORY DO make it a DIFFERENT section, even if the top-level exhibit/addendum identifier is the same. These descriptors indicate distinct sub-sections with different subject matter under a shared parent exhibit. * A genuinely NEW section has either a different top-level identifier OR a different content-type/service-category descriptor under the same identifier. -* If a single header on one page bundles multiple distinct section identifiers together (e.g., lists several addenda, exhibits, or schedules within one header block), it is likely a table of contents or index page — NOT an actual section start. When the individual sections referenced in such a bundled header each appear as their own standalone headers on later pages, DROP the bundled/index header entirely and KEEP the individual section headers from later pages. +* If a single header on one page bundles multiple distinct section identifiers together (e.g., lists several addenda, exhibits, or schedules within one header block), it is likely a table of contents or index page - NOT an actual section start. When the individual sections referenced in such a bundled header each appear as their own standalone headers on later pages, DROP the bundled/index header entirely and KEEP the individual section headers from later pages. [CRITICAL RULE] * Do NOT modify the header text of the first occurrence of any section. The exact original text MUST be preserved character-for-character because it is used downstream for regex matching to split the document into exhibits. Any modification will break downstream processing. @@ -2101,7 +2101,7 @@ Valid payment methods: Note: A REIMBURSEMENT TERM is valid as long as it contains a concrete rate or payment method, even if it also includes surrounding descriptive or billing-scenario language. Focus on whether a specific rate is present, not on the framing around it. Invalid - return NO if REIMBURSEMENT TERM is: -- Empty reimbursements: structure of a payment method but actual rate/percentage is not stated or references another location (e.g., "paid at rates in the next section", "the percentage listed below", "per the attached schedule", "as set forth in Exhibit A"). Note: formulas that define calculation methodology (e.g., "base rate multiplied by DRG weight") ARE valid even if specific dollar amounts aren't stated. Also valid: a percentage tied to a specific CPT/HCPCS code (e.g., "100% of S5125-U3") — the percentage IS the concrete rate. +- Empty reimbursements: structure of a payment method but actual rate/percentage is not stated or references another location (e.g., "paid at rates in the next section", "the percentage listed below", "per the attached schedule", "as set forth in Exhibit A"). Note: formulas that define calculation methodology (e.g., "base rate multiplied by DRG weight") ARE valid even if specific dollar amounts aren't stated. Also valid: a percentage tied to a specific CPT/HCPCS code (e.g., "100% of S5125-U3") - the percentage IS the concrete rate. - Vague or undefined amounts: uses non-specific language like "appropriate amount", "applicable rate", "as determined", "reasonable charges", "customary charges" without a concrete percentage or dollar figure. A fee schedule reference is only valid if paired with a specific percentage (e.g., "100% of fee schedule" is valid; "appropriate amount under fee schedule" is NOT valid). - Service definitions without payment terms - Authorization or coverage requirements only diff --git a/src/tests/test_qc_qa_pipeline.py b/src/tests/test_qc_qa_pipeline.py index 4200e0f..ebb159c 100644 --- a/src/tests/test_qc_qa_pipeline.py +++ b/src/tests/test_qc_qa_pipeline.py @@ -713,7 +713,7 @@ class TestPipelineFunctions: mocker.patch("src.config.S3_CLIENT", mock_s3_client) mocker.patch("builtins.open", mocker.mock_open(read_data=b"")) - # Should not raise — failure is caught and logged as warning + # Should not raise - failure is caught and logged as warning save_qc_qa_outputs( validated_df=sample_dataframe, stats_df=mock_stats_df,