jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in task/unit-tests-investment-and-client (pull request #354)

Browse Source 
Task/unit tests investment and client

* move io_utils.py

* add __init__.py

* refactor: move string_funcs and update import paths to use utils namespace

* rename string_funcs to string_utils

* move and rename table_utils.py

* refactor: rename claude_funcs to llm_utils in multiple files

* update unit tests

* added unit tests for src/string_utils.py

* added unit tests for src/table_utils.py

* added unit tests for src/llm_utils.py

* added unit tests for src/io_utils.py

* Merge branch 'main' into task/unit-tests-investment-and-client; TODO: fix unit tests

* Unit tests to fix

* fix unit tests

* fix invalid references

* add unit tests for preprocessing_funcs.py

* move pytest-mock to test dependencies

* Merge remote-tracking branch 'origin/main' into task/unit-tests-investment-and-client

* add tests/string_utils_test.py

* removed filename parameters for a few functions

* save postprocessing_funcs_test midway

* save postprocessing_funcs_test

* postprocessing_funcs unit tests complete

* Merge branch 'main' into task/unit-tests-investment-and-client


Approved-by: Katon Minhas
This commit is contained in:
Siddhant Medar
2025-01-17 17:15:32 +00:00
committed by Alex Galarce
parent 64baf930f8
commit 04d268a2f8
7 changed files with 1120 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/preprocessing_funcs.py
+22 -14
View File
@@ -5,9 +5,9 @@ from itertools import chain, count, groupby
from src.prompts import preprocessing_prompts
import src.utils.llm_utils as llm_utils
import src.utils.string_utils as string_utils
from src import config, keywords, smart_chunking_funcs
from src.regex.regex_patterns import PIPE_PATTERN
from src import config, keywords, smart_chunking_funcs
from src.enums.delimiters import Delimiter
def clean_newlines(contract_text: str) -> str:
"""Clean textract output by removing newlines and page number indicators.
@@ -22,8 +22,9 @@ def clean_newlines(contract_text: str) -> str:
Returns:
str: cleaned text with newlines and page number indicators removed
"""
cleaned_text = re.sub(r"(?<!\n)\n(?!\n)", " ", contract_text)
cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text)
cleaned_text = re.sub(r"(?<!\n)\n(?!\n)", " ", contract_text) # replaces standalone newlines (not part of double newlines) with spaces
cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", cleaned_text) # TODO: fix in main branch
return cleaned_text
@@ -36,21 +37,26 @@ def split_text(text: str) -> dict[str, str]:
Returns:
dict[str, str]: A dictionary, keyed by the string page number and valued by the page text.
"""
temp_list = text.split("Start of Page No. = ")
text_list = re.split(r"Start of Page No. = [0-9]+\n", text)
if isinstance(text, str):
if not text:
return {}
temp_list = text.split("Start of Page No. = ")
text_list = re.split(r"Start of Page No. = [0-9]+\n", text)
text_dict = {}
for i in range(len(text_list)):
text_dict[temp_list[i].split()[0]] = text_list[i]
text_dict = {}
for i in range(len(text_list)):
text_dict[temp_list[i].split()[0]] = text_list[i] # splits on whitespace characters, which includes spaces, tabs, and newline characters
return {k: v for k, v in text_dict.items() if k != "Document"}
return {k: v for k, v in text_dict.items() if k != "Document"}
def clean_law_symbols(contract_text):
contract_text = contract_text.replace("$$", "$")
contract_text = re.sub(r"(U\.?S\.?C\.?) \$", r"\1§", contract_text)
contract_text = re.sub(r"(C\.?F\.?R\.?) \$", r"\1§", contract_text)
contract_text = re.sub(r"(U\.?S\.?C\.?) \$", r"\1§", contract_text) # replaces $ with § when it follows abbreviations like "U.S.C." (United States Code).
contract_text = re.sub(r"(C\.?F\.?R\.?) \$", r"\1§", contract_text) # replaces $ with § when it follows abbreviations like "C.F.R." (Code of Federal Regulations).
# Second correction: Replace '$' with '§' when followed by a number with three decimal places
contract_text = re.sub(r"\$(?=\d+\.\d{3})", "§", contract_text)
@@ -247,7 +253,7 @@ def smart_chunk_ac(
raise ValueError(
"methodology must be 'hierarchy' or 'or' - using entire contract as chunk"
)
if field_group == "initial_page_group":
if field_group == "initial_page_group": #TODO: this needs to be removed? as it's not present in GROUPED_KEYWORD_MAPPINGS
keyword_page_dict["initial_pages"] = [1, 2]
# chunks[field_group] = concatenated page texts
@@ -303,7 +309,9 @@ def get_exhibit_pages(text_dict, filename):
claude_answer_raw = llm_utils.invoke_claude(
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10
)
claude_answer_extracted = re.findall(pattern=PIPE_PATTERN, string=claude_answer_raw)[-1]
claude_answer_extracted = string_utils.extract_text_from_delimiters(
claude_answer_raw, Delimiter.PIPE
)
if "Y" in claude_answer_extracted:
exhibit_pages.append(page_num)
return exhibit_pages