jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/exhibit-page-fix-pipeline (pull request #390)

Browse Source 
Bugfix/exhibit page fix pipeline

* Fix broken get_exhibit_pages tests

* Add __repr__ method to FieldSet for better string representation


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-02-10 22:59:10 +00:00
parent d243f35c63
commit 524968f09f
3 changed files with 36 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/preprocessing_funcs.py
+21 -11
View File
@@ -203,23 +203,33 @@ def filter_quick_review(text_dict):
or "COVER SHEET" in page_text.upper()
}
def get_exhibit_pages(text_dict, filename):
def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> list[str]:
"""Extract beginning-of-exhibit pages from a contract. The first page is also
always included in the exhibit pages.
Args:
text_dict (dict[str, str]): Dictionary valued by string-formatted page numbers and valued by page text
filename (str): Filename of the contract (for tracking purposes)
Returns:
list[str]: A list of exhibit page numbers extracted from the contract.
"""
exhibit_pages = []
first_page = True
for page_num, page in text_dict.items():
if first_page: # Include the first page in the exhibit pages
exhibit_pages.append(page_num)
first_page = False
continue
prompt = preprocessing_prompts.EXHIBIT_CHECK(page[0:100])
claude_answer_raw = llm_utils.invoke_claude(
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10 # TODO: low priority, try increasing max_tokens and maybe pass multiple pages in to reduce overall calls
)
claude_answer_extracted = string_utils.extract_text_from_delimiters(
claude_answer_raw, Delimiter.PIPE
)
if "Y" in claude_answer_extracted:
exhibit_pages.append(page_num)
else:
prompt = preprocessing_prompts.EXHIBIT_CHECK(page[0:100])
claude_answer_raw = llm_utils.invoke_claude(
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10 # TODO: low priority, try increasing max_tokens and maybe pass multiple pages in to reduce overall calls
)
claude_answer_extracted = string_utils.extract_text_from_delimiters(
claude_answer_raw, Delimiter.PIPE
)
if "Y" in claude_answer_extracted:
exhibit_pages.append(page_num)
return exhibit_pages
def chunk_by_exhibit(text_dict: dict,