Merged in feature/lesser-table-caching-refactor-hybrid (pull request #847)
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
This commit is contained in:
Katon Minhas
@@ -0,0 +1,579 @@
|
||||
import logging
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import src.utils.string_utils as string_utils
|
||||
from src.pipelines.saas.prompts import prompt_calls
|
||||
from src import config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.pipelines.shared.extraction.page_funcs import Page
|
||||
from src.pipelines.shared.extraction.exhibit_funcs import Exhibit
|
||||
|
||||
|
||||
def remove_page_indicators(contract_text: str) -> str:
|
||||
"""Clean textract output by removing page number indicators in the form of "Page X of Y"
|
||||
|
||||
This function processes input text to remove lines that indicate page numbers (e.g. 'Page 1 of 10')
|
||||
|
||||
Args:
|
||||
contract_text (str): Raw text output from Textract to be cleaned
|
||||
|
||||
Returns:
|
||||
str: cleaned text with newlines and page number indicators removed
|
||||
"""
|
||||
|
||||
if contract_text:
|
||||
cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text)
|
||||
else:
|
||||
cleaned_text = contract_text
|
||||
return cleaned_text
|
||||
|
||||
|
||||
def split_text(text: str) -> dict[str, str]:
|
||||
"""Split text on pages by the string `Start of Page No. = '
|
||||
|
||||
Args:
|
||||
text (str): Raw text output from Textract to be split
|
||||
|
||||
Returns:
|
||||
dict[str, str]: A dictionary, keyed by the string page number and valued by the page text.
|
||||
"""
|
||||
temp_list = text.split("Start of Page No. = ")
|
||||
text_list = re.split(r"Start of Page No. = [0-9]+\n", text)
|
||||
|
||||
text_dict = {}
|
||||
for i in range(len(text_list)):
|
||||
if temp_list[i]:
|
||||
text_dict[temp_list[i].split()[0]] = text_list[
|
||||
i
|
||||
] # splits on whitespace characters, which includes spaces, tabs, and newline characters
|
||||
|
||||
return {k: v for k, v in text_dict.items() if k != "Document"}
|
||||
|
||||
|
||||
def clean_law_symbols(contract_text):
|
||||
if not contract_text:
|
||||
return contract_text
|
||||
# First correction: Replace '$$' with '$'
|
||||
contract_text = contract_text.replace("$$", "$")
|
||||
|
||||
contract_text = re.sub(
|
||||
r"(U\.?S\.?C\.?) \$", r"\1§", contract_text
|
||||
) # replaces $ with § when it follows abbreviations like "U.S.C." (United States Code).
|
||||
contract_text = re.sub(
|
||||
r"(C\.?F\.?R\.?) \$", r"\1§", contract_text
|
||||
) # replaces $ with § when it follows abbreviations like "C.F.R." (Code of Federal Regulations).
|
||||
|
||||
# Second correction: Replace '$' with '§' when followed by a number with three decimal places
|
||||
contract_text = re.sub(r"\$(?=\d+\.\d{3})", "§", contract_text)
|
||||
|
||||
return contract_text
|
||||
|
||||
|
||||
def clean_newlines(contract_text: str) -> str:
|
||||
"""Clean textract output by removing newlines and page number indicators.
|
||||
|
||||
This function processes input text to:
|
||||
1. Remove all newline characters.
|
||||
2. Remove lines that indicate page numbers (e.g. 'Page 1 of 10')
|
||||
|
||||
Args:
|
||||
contract_text (str): Raw text output from Textract to be cleaned
|
||||
|
||||
Returns:
|
||||
str: cleaned text with newlines and page number indicators removed
|
||||
"""
|
||||
cleaned_text = re.sub(r"(?<!\n)\n(?!\n)", " ", contract_text)
|
||||
cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text)
|
||||
return cleaned_text
|
||||
|
||||
|
||||
def filter_quick_review(text_dict):
|
||||
"""
|
||||
Cover Sheets (aka Top Sheets or Quick Review pages) are pages stapled to the front of the contract that contain manually written summaries of the contract's contents.
|
||||
They are NOT legally binding documents, and because they are often manually filled out and hand-written, are more prone to have erroneous information than the rest of the contract.
|
||||
For Doczy.AI, we should not pull any information from top sheets, except as a very last resort.
|
||||
|
||||
filter_quick_review() splits the input dictionary into two dictionaries, one containing the contract pages, the other containing the top sheet pages.
|
||||
|
||||
The function checks each page's text for the keywords "QUICK REVIEW", "TOP SHEET", and "COVER SHEET". It categorizes the text into two separate dictionaries: one for texts that do not contain any of these keywords, and another for texts that do.
|
||||
|
||||
Parameters:
|
||||
text_dict (dict): A dictionary where the key is the page number and the value is the text of that page.
|
||||
|
||||
Returns:
|
||||
tuple of two dicts:
|
||||
- The first dictionary contains pages that do not have the specified keywords.
|
||||
- The second dictionary includes pages that contain any of the specified keywords.
|
||||
"""
|
||||
keywords = [
|
||||
"QUICK REVIEW",
|
||||
"TOP SHEET",
|
||||
"COVER SHEET",
|
||||
"PROVIDER SUBMISSION FORM",
|
||||
"CONTRACT REVIEW FORM",
|
||||
]
|
||||
regular_pages = {}
|
||||
quick_review_pages = {}
|
||||
|
||||
for page_num, page_text in text_dict.items():
|
||||
page_text_upper = page_text.upper()
|
||||
if any(keyword in page_text_upper for keyword in keywords):
|
||||
quick_review_pages[page_num] = page_text
|
||||
else:
|
||||
regular_pages[page_num] = page_text
|
||||
|
||||
return regular_pages, quick_review_pages
|
||||
|
||||
|
||||
def get_exhibit_pages(
|
||||
text_dict: Optional[dict[str, str]] = None,
|
||||
pages_dict: Optional[dict[str, "Page"]] = None,
|
||||
EXHIBIT_HEADER_MARKERS: Optional[list] = None,
|
||||
filename: Optional[str] = None,
|
||||
) -> tuple[list[str], dict[str, str]]:
|
||||
"""
|
||||
Identify exhibit pages from the text dictionary or pages dictionary.
|
||||
|
||||
This function processes the dictionary to identify pages that are considered exhibits.
|
||||
It uses the first page as an exhibit and checks subsequent pages for exhibit headers.
|
||||
If a page is identified as an exhibit, it is added to the exhibit pages list and its
|
||||
header is stored in the exhibit header dictionary.
|
||||
If a page does not have a header or is not an exhibit, it is skipped.
|
||||
|
||||
Args:
|
||||
text_dict: A dictionary where keys are page numbers (as strings) and values are
|
||||
the text of those pages (for backward compatibility)
|
||||
pages_dict: A dictionary where keys are page numbers and values are Page objects
|
||||
(preferred)
|
||||
EXHIBIT_HEADER_MARKERS: List of markers to identify exhibit headers
|
||||
filename: The name of the file being processed, used for logging and LLM invocation.
|
||||
|
||||
Returns:
|
||||
tuple[list[str], dict[str, str]]: A tuple containing:
|
||||
- A list of page numbers (as strings) that are identified as exhibit pages.
|
||||
- A dictionary where keys are page numbers and values are the headers of those exhibits.
|
||||
|
||||
Note:
|
||||
Either text_dict or pages_dict must be provided. If both are provided,
|
||||
pages_dict takes precedence.
|
||||
"""
|
||||
if EXHIBIT_HEADER_MARKERS is None:
|
||||
EXHIBIT_HEADER_MARKERS = []
|
||||
if filename is None:
|
||||
filename = ""
|
||||
|
||||
# Determine which dictionary to use
|
||||
if pages_dict is not None:
|
||||
page_dict = pages_dict
|
||||
use_pages = True
|
||||
elif text_dict is not None:
|
||||
page_dict = text_dict
|
||||
use_pages = False
|
||||
else:
|
||||
raise ValueError("Either text_dict or pages_dict must be provided")
|
||||
|
||||
sorted_pages = sorted(page_dict.keys(), key=string_utils.page_key_sort)
|
||||
exhibit_pages = []
|
||||
exhibit_header_dict = {}
|
||||
|
||||
for page_num in sorted_pages:
|
||||
# Get page content - handle both Page objects and strings
|
||||
if use_pages:
|
||||
page_obj = page_dict[page_num]
|
||||
page_content = page_obj.get_text()
|
||||
else:
|
||||
page_content = page_dict[page_num]
|
||||
|
||||
if "." not in page_num or ".0" in page_num:
|
||||
exhibit_header = prompt_calls.prompt_exhibit_header(
|
||||
page_content, EXHIBIT_HEADER_MARKERS, filename
|
||||
)
|
||||
if "N/A" in exhibit_header:
|
||||
is_exhibit = False
|
||||
else:
|
||||
exhibit_pages.append(page_num)
|
||||
exhibit_header_dict[page_num] = exhibit_header
|
||||
is_exhibit = True
|
||||
elif is_exhibit == True:
|
||||
exhibit_pages.append(page_num)
|
||||
return exhibit_pages, exhibit_header_dict
|
||||
|
||||
|
||||
def link_exhibit_pages(
|
||||
all_exhibit_headers: dict[str, str],
|
||||
text_dict: Optional[dict[str, str]] = None,
|
||||
pages_dict: Optional[dict[str, "Page"]] = None,
|
||||
filename: Optional[str] = None,
|
||||
):
|
||||
"""
|
||||
Filters exhibit pages to include only those with meaningfully different headers.
|
||||
|
||||
This function processes exhibit headers sequentially and uses literal comparison and
|
||||
LLM-based semantic comparison to determine when exhibits represent continuation pages
|
||||
vs. distinct exhibits. Continuation pages are filtered out of the headers list and
|
||||
mapping, and when used downstream are included in their parent exhibit's text chunk.
|
||||
Links exhibit pages based on their headers. If the header of a page is identical
|
||||
(either literally or semantically) to the previous page's header, it is not added
|
||||
to the final list of exhibit pages.
|
||||
This function will link contiguous header pages together, so that the final list of
|
||||
exhibit pages will only contain pages that are meaningfully different from the
|
||||
previous page. E.g. "1" with header {"Exhibit 1"} and "2" with header {"Continued
|
||||
exhibit 1"} will be linked together, but "3" with header {"Exhibit 2"} will not be
|
||||
linked to either of them.
|
||||
|
||||
Args:
|
||||
all_exhibit_headers: A dictionary mapping page numbers (as strings) to their
|
||||
exhibit headers. Expected to be in chronological order
|
||||
text_dict: A dictionary mapping page numbers to their text content
|
||||
(for backward compatibility)
|
||||
pages_dict: A dictionary mapping page numbers to Page objects (preferred)
|
||||
filename: The name of the file being processed, used for logging and LLM invocation.
|
||||
|
||||
Returns:
|
||||
tuple[list[str], dict[str, str]]: A tuple containing:
|
||||
- A list of page numbers (as strings) that are identified as exhibit pages.
|
||||
- A dictionary where keys are page numbers and values are the headers of those exhibits.
|
||||
|
||||
Example:
|
||||
Input: {"1": "Exhibit A", "2": "Exhibit A (continued)", "3": "Exhibit B"}
|
||||
Output (["1", "3"], {"1": "Exhibit A", "3": "Exhibit B"})
|
||||
|
||||
Notes:
|
||||
- First exhibit page is always included.
|
||||
- Literal header comparison is performed first to optimize LLM usage and latency.
|
||||
- Uses `EXHIBIT_LINKAGE` prompt for semantic similarity assessment
|
||||
- Expects LLM to return "Y" or "N" enclosed in pipe delimiters
|
||||
- Pages with non-distinct headers are excluded from the final results.
|
||||
- Either text_dict or pages_dict must be provided (pages_dict takes precedence).
|
||||
"""
|
||||
if filename is None:
|
||||
filename = ""
|
||||
first_exhibit = True
|
||||
final_exhibit_pages = []
|
||||
final_exhibit_headers = {}
|
||||
for page_num, page_header in all_exhibit_headers.items():
|
||||
if first_exhibit:
|
||||
exhibit_is_different = "N"
|
||||
previous_header = page_header
|
||||
final_exhibit_pages.append(page_num)
|
||||
final_exhibit_headers[page_num] = page_header
|
||||
first_exhibit = False
|
||||
else:
|
||||
# If literally identical, skip prompt (unless it has a table)
|
||||
if page_header.upper().strip() == previous_header.upper().strip():
|
||||
exhibit_is_different = "N"
|
||||
elif string_utils.is_empty(previous_header):
|
||||
exhibit_is_different = "Y"
|
||||
previous_header = page_header
|
||||
else:
|
||||
exhibit_is_different = prompt_calls.prompt_exhibit_linkage(
|
||||
page_header, previous_header, filename
|
||||
)
|
||||
previous_header = page_header
|
||||
|
||||
# If the prompt has identified that the current exhibit is meaningfully different than the previous
|
||||
if "Y" in exhibit_is_different:
|
||||
final_exhibit_pages.append(page_num)
|
||||
final_exhibit_headers[page_num] = page_header
|
||||
|
||||
return final_exhibit_pages, final_exhibit_headers
|
||||
|
||||
|
||||
def chunk_by_exhibit(
|
||||
text_dict: Optional[dict[str, str]] = None,
|
||||
pages_dict: Optional[dict[str, "Page"]] = None,
|
||||
exhibit_pages: Optional[list] = None,
|
||||
*args, # For backward compatibility with positional arguments
|
||||
) -> dict:
|
||||
"""
|
||||
Organizes pages into groups based on their association with specific exhibits.
|
||||
|
||||
This function assigns each page number from the dictionary to an exhibit
|
||||
based on the `exhibit_pages` list. Pages are grouped under the nearest preceding
|
||||
page number in `exhibit_pages`. If a page number is itself in `exhibit_pages`,
|
||||
it starts a new exhibit group.
|
||||
|
||||
Parameters:
|
||||
text_dict: A dictionary where keys are page numbers and values are page text
|
||||
(for backward compatibility)
|
||||
pages_dict: A dictionary where keys are page numbers and values are Page objects
|
||||
(preferred)
|
||||
exhibit_pages: A list of page numbers that mark the beginning of a new exhibit.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary where keys are exhibit identifiers (page numbers from exhibit_pages)
|
||||
and values are lists of page numbers that belong to that exhibit.
|
||||
Example: {'1': ['1', '2'], '3': ['3', '4', '5']}
|
||||
|
||||
Note:
|
||||
Either text_dict or pages_dict must be provided. If both are provided,
|
||||
pages_dict takes precedence.
|
||||
For backward compatibility: can be called as chunk_by_exhibit(text_dict, exhibit_pages)
|
||||
"""
|
||||
# Backward compatibility: detect when called with old positional signature
|
||||
# Old signature: chunk_by_exhibit(text_dict, exhibit_pages)
|
||||
# When called positionally, Python binds: text_dict=dict, pages_dict=list, exhibit_pages=None
|
||||
if (
|
||||
pages_dict is not None
|
||||
and isinstance(pages_dict, list)
|
||||
and exhibit_pages is None
|
||||
):
|
||||
# This is the old signature - pages_dict is actually exhibit_pages
|
||||
exhibit_pages = pages_dict
|
||||
pages_dict = None
|
||||
|
||||
if exhibit_pages is None:
|
||||
exhibit_pages = []
|
||||
|
||||
# Determine which dictionary to use
|
||||
if pages_dict is not None:
|
||||
if not isinstance(pages_dict, dict):
|
||||
raise TypeError(f"pages_dict must be a dict, got {type(pages_dict)}")
|
||||
page_dict = pages_dict
|
||||
elif text_dict is not None:
|
||||
if not isinstance(text_dict, dict):
|
||||
raise TypeError(f"text_dict must be a dict, got {type(text_dict)}")
|
||||
page_dict = text_dict
|
||||
else:
|
||||
raise ValueError("Either text_dict or pages_dict must be provided")
|
||||
|
||||
if len(exhibit_pages) == 0:
|
||||
return {key: [key] for key in page_dict.keys()}
|
||||
|
||||
exhibit_chunk_mapping = {}
|
||||
current_exhibit = list(page_dict.keys())[0]
|
||||
|
||||
for page_num in page_dict.keys():
|
||||
if page_num in exhibit_pages:
|
||||
current_exhibit = page_num
|
||||
# Initialize the list for this exhibit if not already present
|
||||
if current_exhibit not in exhibit_chunk_mapping:
|
||||
exhibit_chunk_mapping[current_exhibit] = []
|
||||
|
||||
# Add the page to the current exhibit's list
|
||||
if current_exhibit not in exhibit_chunk_mapping:
|
||||
exhibit_chunk_mapping[current_exhibit] = []
|
||||
exhibit_chunk_mapping[current_exhibit].append(page_num)
|
||||
|
||||
return exhibit_chunk_mapping
|
||||
|
||||
|
||||
def simplify_exhibit(
|
||||
text_dict: Optional[dict[str, str]] = None,
|
||||
pages_dict: Optional[dict[str, "Page"]] = None,
|
||||
exhibit: Optional["Exhibit"] = None,
|
||||
exhibit_page_nums: Optional[list[str]] = None,
|
||||
current_page_num: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Simplify exhibit text by replacing table content with 'Table X' placeholders on all pages
|
||||
except the current page being processed.
|
||||
|
||||
Args:
|
||||
text_dict: Dictionary mapping page numbers to page text (for backward compatibility)
|
||||
pages_dict: Dictionary mapping page numbers to Page objects (preferred)
|
||||
exhibit: Exhibit object (preferred - uses exhibit.pages and exhibit.exhibit_page_nums)
|
||||
exhibit_page_nums: List of page numbers that belong to this exhibit
|
||||
(required if exhibit is not provided)
|
||||
current_page_num: The current page being processed (will NOT be simplified)
|
||||
(required if exhibit is not provided)
|
||||
|
||||
Returns:
|
||||
str: The exhibit text with tables simplified on all pages except current_page_num
|
||||
|
||||
Note:
|
||||
Either (exhibit) or (text_dict/pages_dict + exhibit_page_nums + current_page_num) must be provided.
|
||||
If exhibit is provided, it takes precedence.
|
||||
"""
|
||||
# Check if any parameter that should be a dict is actually an Exhibit object
|
||||
if text_dict is not None and hasattr(text_dict, "pages"):
|
||||
# text_dict is actually an Exhibit object
|
||||
exhibit = text_dict
|
||||
text_dict = None
|
||||
elif pages_dict is not None and hasattr(pages_dict, "pages"):
|
||||
# pages_dict is actually an Exhibit object
|
||||
exhibit = pages_dict
|
||||
pages_dict = None
|
||||
|
||||
# Handle Exhibit object (preferred approach) - but only if it's actually an Exhibit
|
||||
if exhibit is not None and hasattr(exhibit, "pages"):
|
||||
if exhibit.pages is not None:
|
||||
pages_dict = exhibit.pages
|
||||
else:
|
||||
# Fall back to text_dict if pages not available
|
||||
text_dict = {
|
||||
page_id: exhibit.get_page_text(page_id)
|
||||
for page_id in exhibit.exhibit_page_nums
|
||||
}
|
||||
exhibit_page_nums = exhibit.exhibit_page_nums
|
||||
# For current_page_num, we need to determine which base page the current_page_num belongs to
|
||||
# If current_page_num is a sub-page like "27.1", the base page is "27"
|
||||
if current_page_num is None:
|
||||
# Try to infer from exhibit_page_nums - use first page as default
|
||||
current_page_num = exhibit_page_nums[0] if exhibit_page_nums else ""
|
||||
else:
|
||||
# Extract base page if it's a sub-page
|
||||
if "." in current_page_num and not current_page_num.startswith("."):
|
||||
current_page_num = current_page_num.rsplit(".", 1)[0]
|
||||
|
||||
# Validate required parameters
|
||||
if exhibit_page_nums is None:
|
||||
raise ValueError(
|
||||
"exhibit_page_nums must be provided if exhibit is not provided"
|
||||
)
|
||||
if current_page_num is None:
|
||||
raise ValueError("current_page_num must be provided if exhibit is not provided")
|
||||
|
||||
# Determine which dictionary to use
|
||||
use_pages = False
|
||||
if pages_dict is not None:
|
||||
page_dict = pages_dict
|
||||
use_pages = True
|
||||
elif text_dict is not None:
|
||||
page_dict = text_dict
|
||||
use_pages = False
|
||||
else:
|
||||
raise ValueError("Either text_dict, pages_dict, or exhibit must be provided")
|
||||
|
||||
def replace_tables_with_placeholder(
|
||||
page_text: str, starting_table_num: int
|
||||
) -> tuple[str, int]:
|
||||
"""Replace table blocks with 'Table X' labels and return next table number."""
|
||||
if not page_text:
|
||||
return page_text, starting_table_num
|
||||
|
||||
start_marker = "-------Table Start--------"
|
||||
end_marker = "-------Table End--------"
|
||||
result = []
|
||||
pos = 0
|
||||
table_num = starting_table_num
|
||||
|
||||
while True:
|
||||
start_idx = page_text.find(start_marker, pos)
|
||||
if start_idx == -1:
|
||||
# No more tables, append remaining text
|
||||
result.append(page_text[pos:])
|
||||
break
|
||||
|
||||
# Append text before table
|
||||
result.append(page_text[pos:start_idx])
|
||||
|
||||
# Find end of table
|
||||
end_idx = page_text.find(end_marker, start_idx + len(start_marker))
|
||||
|
||||
# Replace table with placeholder
|
||||
result.append(f"Table {table_num}")
|
||||
table_num += 1
|
||||
|
||||
# Move position past the table end marker
|
||||
if end_idx != -1:
|
||||
pos = end_idx + len(end_marker)
|
||||
else:
|
||||
# No end marker found, stop processing
|
||||
break
|
||||
|
||||
return "".join(result), table_num
|
||||
|
||||
# Process each page/sub-page in the exhibit
|
||||
simplified_exhibit_pages = []
|
||||
table_counter = 1
|
||||
|
||||
# Extract base page from current_page_num if it's a sub-page
|
||||
current_base_page = current_page_num
|
||||
if "." in current_page_num and not current_page_num.startswith("."):
|
||||
current_base_page = current_page_num.rsplit(".", 1)[0]
|
||||
|
||||
for page_id in exhibit_page_nums:
|
||||
# Extract base page for comparison
|
||||
base_page = page_id
|
||||
if "." in page_id and not page_id.startswith("."):
|
||||
base_page = page_id.rsplit(".", 1)[0]
|
||||
|
||||
# Get page text
|
||||
if use_pages:
|
||||
# Handle sub-pages: "27.1" means get sub-page "1" from page "27"
|
||||
if "." in page_id and not page_id.startswith("."):
|
||||
base_page_num, sub_page = page_id.rsplit(".", 1)
|
||||
if base_page_num in page_dict:
|
||||
page_text = page_dict[base_page_num].get_text(sub_page)
|
||||
else:
|
||||
page_text = ""
|
||||
elif page_id in page_dict:
|
||||
page_text = page_dict[page_id].get_text()
|
||||
else:
|
||||
page_text = ""
|
||||
else:
|
||||
page_text = page_dict.get(page_id, "")
|
||||
|
||||
# Check if this is the current page (match base page)
|
||||
if base_page == current_base_page:
|
||||
# Don't simplify the current page - keep it as-is
|
||||
simplified_exhibit_pages.append(page_text)
|
||||
# But still need to count tables to keep numbering consistent
|
||||
if "-------Table Start--------" in page_text:
|
||||
_, table_counter = replace_tables_with_placeholder(
|
||||
page_text, table_counter
|
||||
)
|
||||
else:
|
||||
# Simplify other pages by replacing tables with placeholders
|
||||
if "-------Table Start--------" in page_text:
|
||||
simplified_page, table_counter = replace_tables_with_placeholder(
|
||||
page_text, table_counter
|
||||
)
|
||||
simplified_exhibit_pages.append(simplified_page)
|
||||
else:
|
||||
simplified_exhibit_pages.append(page_text)
|
||||
|
||||
return "\n".join(simplified_exhibit_pages)
|
||||
|
||||
|
||||
def split_large_tables(
|
||||
text_dict: dict[str, str],
|
||||
max_cells: Optional[int] = None,
|
||||
min_rows_to_split: Optional[int] = None,
|
||||
) -> dict[str, "Page"]:
|
||||
"""
|
||||
Identify and split large tables into sub-pages.
|
||||
|
||||
This function converts a text dictionary to Page objects and splits
|
||||
large tables based on cell count (rows × columns).
|
||||
|
||||
Args:
|
||||
text_dict: Original page number to text mapping
|
||||
max_cells: Maximum cells (rows × columns) per table sub-page.
|
||||
If None, uses config.MAX_TABLE_CELLS_PER_SUBPAGE
|
||||
min_rows_to_split: Minimum rows to trigger splitting.
|
||||
If None, uses config.MIN_TABLE_ROWS_TO_SPLIT
|
||||
|
||||
Returns:
|
||||
dict mapping page numbers to Page objects
|
||||
"""
|
||||
from src.pipelines.shared.extraction.page_funcs import Page
|
||||
|
||||
if not config.TABLE_SPLITTING_ENABLED:
|
||||
# If table splitting is disabled, create Page objects without splitting
|
||||
return {
|
||||
page_num: Page(page_num, page_text)
|
||||
for page_num, page_text in text_dict.items()
|
||||
}
|
||||
|
||||
if max_cells is None:
|
||||
max_cells = config.MAX_TABLE_CELLS_PER_SUBPAGE
|
||||
if min_rows_to_split is None:
|
||||
min_rows_to_split = config.MIN_TABLE_ROWS_TO_SPLIT
|
||||
|
||||
pages_dict = {}
|
||||
|
||||
for page_num, page_text in text_dict.items():
|
||||
page = Page(page_num, page_text)
|
||||
|
||||
# Check if page has a table and if it needs splitting
|
||||
if page.has_table():
|
||||
# Split table if it exceeds thresholds
|
||||
page.split_table(max_cells=max_cells, min_rows_to_split=min_rows_to_split)
|
||||
|
||||
pages_dict[page_num] = page
|
||||
|
||||
return pages_dict
|
||||
Reference in New Issue
Block a user