diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 09897b8..8995ea5 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -217,13 +217,13 @@ pipelines: TARGET_BRANCH="${BITBUCKET_PR_DESTINATION_BRANCH}" # Gate: feature branches can only target dev ALLOWED="dev" - echo "Source: ${SOURCE_BRANCH}" - echo "Target: ${TARGET_BRANCH} (allowed: ${ALLOWED})" + echo "Source: $SOURCE_BRANCH" + echo "Target: $TARGET_BRANCH (allowed: $ALLOWED)" IFS=',' read -ra targets <<< "$ALLOWED" for t in "${targets[@]}"; do [ "${TARGET_BRANCH}" = "$t" ] && exit 0 done - echo "BLOCKED: target must be one of: ${ALLOWED}"; exit 1 + echo "BLOCKED: target must be one of: $ALLOWED"; exit 1 - step: *ai-code-review # ------------------------------------------------------------------------- diff --git a/src/constants/delimiters.py b/src/constants/delimiters.py index f95270d..389d128 100644 --- a/src/constants/delimiters.py +++ b/src/constants/delimiters.py @@ -16,3 +16,6 @@ class Delimiter(Enum): # Text parsing markers inserted by Textract table extraction TABLE_START_MARKER = "-------Table Start--------" TABLE_END_MARKER = "-------Table End--------" + +# Keywords used to identify exhibit/attachment headers within table regions +TABLE_EXHIBIT_KEYWORDS = ("EXHIBIT", "ADDENDUM", "ATTACHMENT") diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index ae17891..13187e0 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -598,7 +598,7 @@ def prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename): prompt, "sonnet_latest", filename, - max_tokens=1024, + max_tokens=4096, cache=True, instruction=prompt_templates.EXHIBIT_HEADER_INSTRUCTION(), usage_label="EXHIBIT_HEADER", diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index da86ec2..8faef5e 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -8,6 +8,8 @@ from datetime import datetime import pandas as pd +import src.prompts.prompt_templates as prompt_templates +import src.utils.llm_utils as llm_utils from src.utils import string_utils from src.utils import json_utils diff --git a/src/pipelines/shared/preprocessing/preprocessing_funcs.py b/src/pipelines/shared/preprocessing/preprocessing_funcs.py index 0302cd5..5ec4164 100644 --- a/src/pipelines/shared/preprocessing/preprocessing_funcs.py +++ b/src/pipelines/shared/preprocessing/preprocessing_funcs.py @@ -5,7 +5,11 @@ from typing import TYPE_CHECKING, Optional import src.utils.string_utils as string_utils from src.pipelines.shared.prompts import prompt_calls from src import config -from src.constants.delimiters import TABLE_START_MARKER, TABLE_END_MARKER +from src.constants.delimiters import ( + TABLE_START_MARKER, + TABLE_END_MARKER, + TABLE_EXHIBIT_KEYWORDS, +) if TYPE_CHECKING: from src.pipelines.shared.extraction.page_funcs import Page @@ -559,8 +563,38 @@ def identify_section_boundaries(page_text: str, context_chars: int = 100) -> str inside_table = False continue - # Skip any lines inside table boundaries + # Inside table: include lines that contain exhibit/addendum/attachment headers. + # Two forms are checked: + # 1. Plain text title lines (not starting with '[[') + # 2. Cell data lines ('[[...]]'): only the first cell of the first row if inside_table: + stripped = line.strip() + header_text = None + + if stripped and not stripped.startswith("[["): + upper = stripped.upper() + if any(kw in upper for kw in TABLE_EXHIBIT_KEYWORDS): + header_text = stripped + elif stripped.startswith("[["): + first_cell_match = re.match( + r"""\[\[['\"]((?:[^'\"\\]|\\.)*?)['\"]\s*[,\]]""", stripped + ) + if first_cell_match: + first_cell = first_cell_match.group(1) + upper_cell = first_cell.upper() + if any(kw in upper_cell for kw in TABLE_EXHIBIT_KEYWORDS): + header_text = first_cell + + if header_text: + char_pos = len("\n".join(lines[:line_idx])) + potential_boundaries.append( + { + "line_idx": line_idx, + "char_pos": char_pos, + "type": "TABLE_EXHIBIT", + "header_text": header_text, + } + ) continue # Skip empty lines diff --git a/src/prompts/prompt_templates.py b/src/prompts/prompt_templates.py index ff0b2ff..86ff22c 100644 --- a/src/prompts/prompt_templates.py +++ b/src/prompts/prompt_templates.py @@ -2120,7 +2120,7 @@ Rules for Inclusion: * Include ALL text that appears to be part of a section header * Full header names and numbers/letters (e.g., "Attachment C: Commercial-Exchange") * Document titles, agreement names, and section identifiers -* Any words that specify what type of information is contained, such as "Compensation Schedule" or "Definitions" +* Content-type descriptors (e.g., "Reimbursement", "Fee Schedule", "Compensation Schedule") ONLY when they appear as part of a named exhibit/attachment/amendment header — not when they appear as standalone headings within exhibit body text * Provider/Entity names when listed with exhibit information * Keep exact capitalization and formatting as shown in document @@ -2130,6 +2130,7 @@ Rules for Exclusion: * Do NOT include docusign IDs and other metadata * Do NOT include partial sentences or continuation text * Do NOT include form field labels like "Provider's Legal Name" or "Authorized Representative's Signature" +* Do NOT include standalone headings that label a category of content within an exhibit's body (e.g., a heading that introduces a block of notes, definitions, billing rules, or procedural instructions). A valid exhibit header answers "which section/exhibit/amendment is this?" — a body-level label answers "what kind of content follows here?" Only the former should be included. Ask: would this heading appear in the document's table of contents as a top-level entry? If not, exclude it. * Do NOT include page-footer text. Footers appear near the bottom of a page adjacent to page numbers, contract/reference numbers, effective dates, or docusign IDs, and often repeat a SHORTENED form of the real exhibit identifier. For example, if the top-of-page header is "ATTACHMENT A-2 Oregon Health & Science University Effective 07/01/2025 Reimbursement Schedule", a bottom-of-page line like "Attachment A Oregon Health & Science University" next to "Page 2" or a contract number IS a footer, NOT a new section header — exclude it. When the same chunk contains a long-form exhibit header AND a short-form repeat of the same exhibit identifier near page metadata, keep only the long-form header. [OUTPUT FORMAT] @@ -2209,7 +2210,10 @@ def EXHIBIT_HEADER_DEDUP_INSTRUCTION() -> str: "[OBJECTIVE]\n" "Deduplicate a dictionary of page-number-to-header mappings from a contract.\n" "Keep only the first page occurrence of each unique section.\n" - "Return the deduplicated JSON dict in |pipes|." + "Return the deduplicated JSON dict in |pipes|.\n\n" + "[EFFECTIVE DATE RULE]\n" + "An exhibit or section that reappears with a different effective date is still a duplicate of the first occurrence — " + "a change in effective date alone does NOT create a distinct section. Drop the later-dated repetition and keep only the first." ) @@ -2237,6 +2241,7 @@ Your task is to deduplicate this dictionary so that each unique section appears * Extra sub-lines that are facility names, entity names, or provider names do NOT make it a different section - these are just contextual metadata repeated on continuation pages. * However, sub-lines that describe the TYPE OF CONTENT or SERVICE CATEGORY DO make it a DIFFERENT section, even if the top-level exhibit/addendum identifier is the same. These descriptors indicate distinct sub-sections with different subject matter under a shared parent exhibit. * A genuinely NEW section has either a different top-level identifier OR a different content-type/service-category descriptor under the same identifier. +* If an exhibit or section header reappears with a different effective date, it is still a duplicate of the first occurrence. A change in effective date alone does NOT create a distinct section — drop the later-dated repetition. * If a single header on one page bundles multiple distinct section identifiers together (e.g., lists several addenda, exhibits, or schedules within one header block), it is likely a table of contents or index page - NOT an actual section start. When the individual sections referenced in such a bundled header each appear as their own standalone headers on later pages, DROP the bundled/index header entirely and KEEP the individual section headers from later pages. [CRITICAL RULE] diff --git a/src/tests/test_preprocessing_funcs.py b/src/tests/test_preprocessing_funcs.py index 3bcc500..b5f588d 100644 --- a/src/tests/test_preprocessing_funcs.py +++ b/src/tests/test_preprocessing_funcs.py @@ -3,6 +3,7 @@ from src.pipelines.shared.preprocessing.preprocessing_funcs import ( chunk_by_exhibit, clean_law_symbols, filter_quick_review, + identify_section_boundaries, remove_page_indicators, simplify_exhibit, ) @@ -254,3 +255,87 @@ class TestPreprocessingFuncs: # Should still process and replace table, stopping at end of text assert "Table 1" in result or "-------Table Start--------" in result + + +class TestIdentifySectionBoundaries: + TABLE_START = "-------Table Start--------" + TABLE_END = "-------Table End--------" + + def _make_table(self, content: str) -> str: + return f"{self.TABLE_START}\n{content}\n{self.TABLE_END}" + + def test_exhibit_keyword_in_table_is_detected(self): + page = self._make_table( + "HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025" + ) + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" in result + assert ( + "HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025" + in result + ) + + def test_addendum_keyword_in_table_is_detected(self): + page = self._make_table("ADDENDUM 1 TO THE AGREEMENT EFFECTIVE 2025") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" in result + + def test_attachment_keyword_in_table_is_detected(self): + page = self._make_table("ATTACHMENT A - FEE SCHEDULE") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" in result + + def test_first_cell_with_exhibit_keyword_is_detected(self): + page = self._make_table( + "[['EXHIBIT B-1 Effective 1/1/24', '', '', ''], ['row2col1', 'row2col2']]" + ) + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" in result + assert "EXHIBIT B-1 Effective 1/1/24" in result + + def test_first_cell_with_addendum_keyword_is_detected(self): + page = self._make_table("[['ADDENDUM 2 FEE SCHEDULE', ''], ['data', 'data']]") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" in result + + def test_first_cell_without_keywords_is_skipped(self): + page = self._make_table("[['RATES AND FEES', '100', '200']]") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" not in result + + def test_non_first_cell_with_keyword_does_not_produce_boundary(self): + """Only the first cell is checked — keywords in other cells are ignored.""" + page = self._make_table("[['RATES', 'EXHIBIT B', ''], ['data', 'data']]") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" not in result + + def test_table_without_keywords_produces_no_table_exhibit_boundary(self): + page = self._make_table("RATES AND FEES SCHEDULE\n[['100', '200']]") + result = identify_section_boundaries(page) + assert "TABLE_EXHIBIT" not in result + + def test_title_line_and_cell_data_both_with_keyword_produce_one_boundary_each(self): + """Title line and first cell both match — two separate TABLE_EXHIBIT boundaries.""" + content = ( + "HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025\n" + "[['EXHIBIT B-1 Effective 1/1/24', '', '']]" + ) + page = self._make_table(content) + result = identify_section_boundaries(page) + assert result.count("TABLE_EXHIBIT") == 2 + + def test_title_line_with_keyword_and_cell_without_keyword(self): + """Title line matched; cell data first cell has no keyword — only one boundary.""" + content = ( + "HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025\n" + "[['RATES AND FEES', '']]" + ) + page = self._make_table(content) + result = identify_section_boundaries(page) + assert result.count("TABLE_EXHIBIT") == 1 + + def test_no_table_regions_unaffected(self): + page = "EXHIBIT A - SERVICES\nSome contract text here." + result = identify_section_boundaries(page) + assert "EXHIBIT" in result + assert "TABLE_EXHIBIT" not in result