Merged in hotfix/exhibit-header-in-tables (pull request #987)
Hotfix/exhibit header in tables * Merged in feature/FixplaceholderIssue (pull request #977) Remove curly braces from echo statements in dev->stg * Remove curly brances from echo statements in dev->stg * Removed curly braces in echo statements in feature-> dev gate Approved-by: Sujit Deokar * Merged in feature/standardized-services (pull request #958) Feature/standardized services * service term standardization * prompt update * prompt updates for standardization * only service standardization * new file * add supporting files and test scripts for standardization work Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * remove old files * Merge remote-tracking branch 'origin/dev' into feature/standardized-services * final fixes * Merged dev into feature/standardized-services * additional features * removed unwanted files * remove unwanted files * Merge branch 'dev' into feature/standardized-services * Merge remote-tracking branch 'origin/dev' into feature/standardized-services * reversed vendor changes * Merged dev into feature/standardized-services * addressed PR comments * Merge branch 'dev' into feature/standardized-services * Merged dev into feature/standardized-services * addressed 3 remaining comments inPR * black formatting * incorporated feedback from AI c… * prompt added for amendment intent * amendment intent language * update field name * amendment intent tag * prompt update * Merge remote-tracking branch 'origin/dev' into feature/active-rates * Merge remote-tracking branch 'origin/dev' into feature/active-rates * remove docusign lines * add unit tests for clean_header_footer docusign/deleted_lines changes Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * black forrmatting * Merge branch 'feature/active-rates' into temp-branch * initial commit * exhibit header in table * Merge branch 'temp-branch' into hotfix/exhibit-header-in-tables * exhibit header within table * Merge branch 'dev' into hotfix/exhibit-header-in-tables * deduplicaton prompt update * foramatting * removed amend intent code * Merged dev into hotfix/exhibit-header-in-tables * added keywords to constants * changes reverted * formatting Approved-by: Katon Minhas
This commit is contained in:
Karan Desai
committed by
Katon Minhas
Katon Minhas
parent
0c3636f13b
commit
b70759fda0
@@ -217,13 +217,13 @@ pipelines:
|
||||
TARGET_BRANCH="${BITBUCKET_PR_DESTINATION_BRANCH}"
|
||||
# Gate: feature branches can only target dev
|
||||
ALLOWED="dev"
|
||||
echo "Source: ${SOURCE_BRANCH}"
|
||||
echo "Target: ${TARGET_BRANCH} (allowed: ${ALLOWED})"
|
||||
echo "Source: $SOURCE_BRANCH"
|
||||
echo "Target: $TARGET_BRANCH (allowed: $ALLOWED)"
|
||||
IFS=',' read -ra targets <<< "$ALLOWED"
|
||||
for t in "${targets[@]}"; do
|
||||
[ "${TARGET_BRANCH}" = "$t" ] && exit 0
|
||||
done
|
||||
echo "BLOCKED: target must be one of: ${ALLOWED}"; exit 1
|
||||
echo "BLOCKED: target must be one of: $ALLOWED"; exit 1
|
||||
- step: *ai-code-review
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
@@ -16,3 +16,6 @@ class Delimiter(Enum):
|
||||
# Text parsing markers inserted by Textract table extraction
|
||||
TABLE_START_MARKER = "-------Table Start--------"
|
||||
TABLE_END_MARKER = "-------Table End--------"
|
||||
|
||||
# Keywords used to identify exhibit/attachment headers within table regions
|
||||
TABLE_EXHIBIT_KEYWORDS = ("EXHIBIT", "ADDENDUM", "ATTACHMENT")
|
||||
|
||||
@@ -598,7 +598,7 @@ def prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename):
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
max_tokens=1024,
|
||||
max_tokens=4096,
|
||||
cache=True,
|
||||
instruction=prompt_templates.EXHIBIT_HEADER_INSTRUCTION(),
|
||||
usage_label="EXHIBIT_HEADER",
|
||||
|
||||
@@ -8,6 +8,8 @@ from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
import src.prompts.prompt_templates as prompt_templates
|
||||
import src.utils.llm_utils as llm_utils
|
||||
from src.utils import string_utils
|
||||
from src.utils import json_utils
|
||||
|
||||
|
||||
@@ -5,7 +5,11 @@ from typing import TYPE_CHECKING, Optional
|
||||
import src.utils.string_utils as string_utils
|
||||
from src.pipelines.shared.prompts import prompt_calls
|
||||
from src import config
|
||||
from src.constants.delimiters import TABLE_START_MARKER, TABLE_END_MARKER
|
||||
from src.constants.delimiters import (
|
||||
TABLE_START_MARKER,
|
||||
TABLE_END_MARKER,
|
||||
TABLE_EXHIBIT_KEYWORDS,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.pipelines.shared.extraction.page_funcs import Page
|
||||
@@ -559,8 +563,38 @@ def identify_section_boundaries(page_text: str, context_chars: int = 100) -> str
|
||||
inside_table = False
|
||||
continue
|
||||
|
||||
# Skip any lines inside table boundaries
|
||||
# Inside table: include lines that contain exhibit/addendum/attachment headers.
|
||||
# Two forms are checked:
|
||||
# 1. Plain text title lines (not starting with '[[')
|
||||
# 2. Cell data lines ('[[...]]'): only the first cell of the first row
|
||||
if inside_table:
|
||||
stripped = line.strip()
|
||||
header_text = None
|
||||
|
||||
if stripped and not stripped.startswith("[["):
|
||||
upper = stripped.upper()
|
||||
if any(kw in upper for kw in TABLE_EXHIBIT_KEYWORDS):
|
||||
header_text = stripped
|
||||
elif stripped.startswith("[["):
|
||||
first_cell_match = re.match(
|
||||
r"""\[\[['\"]((?:[^'\"\\]|\\.)*?)['\"]\s*[,\]]""", stripped
|
||||
)
|
||||
if first_cell_match:
|
||||
first_cell = first_cell_match.group(1)
|
||||
upper_cell = first_cell.upper()
|
||||
if any(kw in upper_cell for kw in TABLE_EXHIBIT_KEYWORDS):
|
||||
header_text = first_cell
|
||||
|
||||
if header_text:
|
||||
char_pos = len("\n".join(lines[:line_idx]))
|
||||
potential_boundaries.append(
|
||||
{
|
||||
"line_idx": line_idx,
|
||||
"char_pos": char_pos,
|
||||
"type": "TABLE_EXHIBIT",
|
||||
"header_text": header_text,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
# Skip empty lines
|
||||
|
||||
@@ -2120,7 +2120,7 @@ Rules for Inclusion:
|
||||
* Include ALL text that appears to be part of a section header
|
||||
* Full header names and numbers/letters (e.g., "Attachment C: Commercial-Exchange")
|
||||
* Document titles, agreement names, and section identifiers
|
||||
* Any words that specify what type of information is contained, such as "Compensation Schedule" or "Definitions"
|
||||
* Content-type descriptors (e.g., "Reimbursement", "Fee Schedule", "Compensation Schedule") ONLY when they appear as part of a named exhibit/attachment/amendment header — not when they appear as standalone headings within exhibit body text
|
||||
* Provider/Entity names when listed with exhibit information
|
||||
* Keep exact capitalization and formatting as shown in document
|
||||
|
||||
@@ -2130,6 +2130,7 @@ Rules for Exclusion:
|
||||
* Do NOT include docusign IDs and other metadata
|
||||
* Do NOT include partial sentences or continuation text
|
||||
* Do NOT include form field labels like "Provider's Legal Name" or "Authorized Representative's Signature"
|
||||
* Do NOT include standalone headings that label a category of content within an exhibit's body (e.g., a heading that introduces a block of notes, definitions, billing rules, or procedural instructions). A valid exhibit header answers "which section/exhibit/amendment is this?" — a body-level label answers "what kind of content follows here?" Only the former should be included. Ask: would this heading appear in the document's table of contents as a top-level entry? If not, exclude it.
|
||||
* Do NOT include page-footer text. Footers appear near the bottom of a page adjacent to page numbers, contract/reference numbers, effective dates, or docusign IDs, and often repeat a SHORTENED form of the real exhibit identifier. For example, if the top-of-page header is "ATTACHMENT A-2 Oregon Health & Science University Effective 07/01/2025 Reimbursement Schedule", a bottom-of-page line like "Attachment A Oregon Health & Science University" next to "Page 2" or a contract number IS a footer, NOT a new section header — exclude it. When the same chunk contains a long-form exhibit header AND a short-form repeat of the same exhibit identifier near page metadata, keep only the long-form header.
|
||||
|
||||
[OUTPUT FORMAT]
|
||||
@@ -2209,7 +2210,10 @@ def EXHIBIT_HEADER_DEDUP_INSTRUCTION() -> str:
|
||||
"[OBJECTIVE]\n"
|
||||
"Deduplicate a dictionary of page-number-to-header mappings from a contract.\n"
|
||||
"Keep only the first page occurrence of each unique section.\n"
|
||||
"Return the deduplicated JSON dict in |pipes|."
|
||||
"Return the deduplicated JSON dict in |pipes|.\n\n"
|
||||
"[EFFECTIVE DATE RULE]\n"
|
||||
"An exhibit or section that reappears with a different effective date is still a duplicate of the first occurrence — "
|
||||
"a change in effective date alone does NOT create a distinct section. Drop the later-dated repetition and keep only the first."
|
||||
)
|
||||
|
||||
|
||||
@@ -2237,6 +2241,7 @@ Your task is to deduplicate this dictionary so that each unique section appears
|
||||
* Extra sub-lines that are facility names, entity names, or provider names do NOT make it a different section - these are just contextual metadata repeated on continuation pages.
|
||||
* However, sub-lines that describe the TYPE OF CONTENT or SERVICE CATEGORY DO make it a DIFFERENT section, even if the top-level exhibit/addendum identifier is the same. These descriptors indicate distinct sub-sections with different subject matter under a shared parent exhibit.
|
||||
* A genuinely NEW section has either a different top-level identifier OR a different content-type/service-category descriptor under the same identifier.
|
||||
* If an exhibit or section header reappears with a different effective date, it is still a duplicate of the first occurrence. A change in effective date alone does NOT create a distinct section — drop the later-dated repetition.
|
||||
* If a single header on one page bundles multiple distinct section identifiers together (e.g., lists several addenda, exhibits, or schedules within one header block), it is likely a table of contents or index page - NOT an actual section start. When the individual sections referenced in such a bundled header each appear as their own standalone headers on later pages, DROP the bundled/index header entirely and KEEP the individual section headers from later pages.
|
||||
|
||||
[CRITICAL RULE]
|
||||
|
||||
@@ -3,6 +3,7 @@ from src.pipelines.shared.preprocessing.preprocessing_funcs import (
|
||||
chunk_by_exhibit,
|
||||
clean_law_symbols,
|
||||
filter_quick_review,
|
||||
identify_section_boundaries,
|
||||
remove_page_indicators,
|
||||
simplify_exhibit,
|
||||
)
|
||||
@@ -254,3 +255,87 @@ class TestPreprocessingFuncs:
|
||||
|
||||
# Should still process and replace table, stopping at end of text
|
||||
assert "Table 1" in result or "-------Table Start--------" in result
|
||||
|
||||
|
||||
class TestIdentifySectionBoundaries:
|
||||
TABLE_START = "-------Table Start--------"
|
||||
TABLE_END = "-------Table End--------"
|
||||
|
||||
def _make_table(self, content: str) -> str:
|
||||
return f"{self.TABLE_START}\n{content}\n{self.TABLE_END}"
|
||||
|
||||
def test_exhibit_keyword_in_table_is_detected(self):
|
||||
page = self._make_table(
|
||||
"HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025"
|
||||
)
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" in result
|
||||
assert (
|
||||
"HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025"
|
||||
in result
|
||||
)
|
||||
|
||||
def test_addendum_keyword_in_table_is_detected(self):
|
||||
page = self._make_table("ADDENDUM 1 TO THE AGREEMENT EFFECTIVE 2025")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" in result
|
||||
|
||||
def test_attachment_keyword_in_table_is_detected(self):
|
||||
page = self._make_table("ATTACHMENT A - FEE SCHEDULE")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" in result
|
||||
|
||||
def test_first_cell_with_exhibit_keyword_is_detected(self):
|
||||
page = self._make_table(
|
||||
"[['EXHIBIT B-1 Effective 1/1/24', '', '', ''], ['row2col1', 'row2col2']]"
|
||||
)
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" in result
|
||||
assert "EXHIBIT B-1 Effective 1/1/24" in result
|
||||
|
||||
def test_first_cell_with_addendum_keyword_is_detected(self):
|
||||
page = self._make_table("[['ADDENDUM 2 FEE SCHEDULE', ''], ['data', 'data']]")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" in result
|
||||
|
||||
def test_first_cell_without_keywords_is_skipped(self):
|
||||
page = self._make_table("[['RATES AND FEES', '100', '200']]")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" not in result
|
||||
|
||||
def test_non_first_cell_with_keyword_does_not_produce_boundary(self):
|
||||
"""Only the first cell is checked — keywords in other cells are ignored."""
|
||||
page = self._make_table("[['RATES', 'EXHIBIT B', ''], ['data', 'data']]")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" not in result
|
||||
|
||||
def test_table_without_keywords_produces_no_table_exhibit_boundary(self):
|
||||
page = self._make_table("RATES AND FEES SCHEDULE\n[['100', '200']]")
|
||||
result = identify_section_boundaries(page)
|
||||
assert "TABLE_EXHIBIT" not in result
|
||||
|
||||
def test_title_line_and_cell_data_both_with_keyword_produce_one_boundary_each(self):
|
||||
"""Title line and first cell both match — two separate TABLE_EXHIBIT boundaries."""
|
||||
content = (
|
||||
"HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025\n"
|
||||
"[['EXHIBIT B-1 Effective 1/1/24', '', '']]"
|
||||
)
|
||||
page = self._make_table(content)
|
||||
result = identify_section_boundaries(page)
|
||||
assert result.count("TABLE_EXHIBIT") == 2
|
||||
|
||||
def test_title_line_with_keyword_and_cell_without_keyword(self):
|
||||
"""Title line matched; cell data first cell has no keyword — only one boundary."""
|
||||
content = (
|
||||
"HACKENSACK MERIDIAN HEALTH EXHIBIT B REIMBURSEMENT EFFECTIVE 1/1/2025\n"
|
||||
"[['RATES AND FEES', '']]"
|
||||
)
|
||||
page = self._make_table(content)
|
||||
result = identify_section_boundaries(page)
|
||||
assert result.count("TABLE_EXHIBIT") == 1
|
||||
|
||||
def test_no_table_regions_unaffected(self):
|
||||
page = "EXHIBIT A - SERVICES\nSome contract text here."
|
||||
result = identify_section_boundaries(page)
|
||||
assert "EXHIBIT" in result
|
||||
assert "TABLE_EXHIBIT" not in result
|
||||
|
||||
Reference in New Issue
Block a user