Merged in feature/integrate-standard-and-complex (pull request #356)

Feature/integrate standard and complex

* last edits to merge_tables_draft_v2 before moving to table_funcs

* migrate to table_utils.py

* fix typehint that caused mypy error

* fix mypy errors

* black and isort

* fix error with multi-table pages

* update poetry.lock

* add tests, fix bug when table start marker or table end marker is missing

* added more tests

* tests for get_str_dictionaries_from_text()

* fix edge case in get_str_dictionaries_from_text

* fix mypy error

* more tests

* add tests, add handling for invalid dictionaries

* add tests

* add tests for insert_column_headers()

* update requirements

* add new simple/complex logic to preprocess.py

* remove files used solely for testing

* split pages into sub-pages by tables

* add table split on end marker.  Also add tests

* add docstrings, change control flow

* remove unused tests, add TODO for new tests

* get in prompt changes and intermediate decisions

* TODO for future enhancement for get_exhibit_pages


Approved-by: Katon Minhas

fieldExtraction/src/preprocessing_funcs.py

@@ -25,6 +25,7 @@ def remove_page_indicators(contract_text: str) -> str:
     return cleaned_text
 
 
+# TODO: write unit tests
 def split_text(text: str) -> dict[str, str]:
     """Split text on pages by the string `Start of Page No. = '
 
@@ -34,19 +35,14 @@ def split_text(text: str) -> dict[str, str]:
     Returns:
         dict[str, str]: A dictionary, keyed by the string page number and valued by the page text.
     """
-        
-    if isinstance(text, str):
-        if not text:
-            return {}
-    
-        temp_list = text.split("Start of Page No. = ")
-        text_list = re.split(r"Start of Page No. = [0-9]+\n", text)
+    temp_list = text.split("Start of Page No. = ")
+    text_list = re.split(r"Start of Page No. = [0-9]+\n", text)
 
-        text_dict = {}
-        for i in range(len(text_list)):
-            text_dict[temp_list[i].split()[0]] = text_list[i]   # splits on whitespace characters, which includes spaces, tabs, and newline characters
+    text_dict = {}
+    for i in range(len(text_list)):
+        text_dict[temp_list[i].split()[0]] = text_list[i]   # splits on whitespace characters, which includes spaces, tabs, and newline characters
 
-        return {k: v for k, v in text_dict.items() if k != "Document"}
+    return {k: v for k, v in text_dict.items() if k != "Document"}
 
 
 def clean_law_symbols(contract_text):
@@ -304,7 +300,7 @@ def get_exhibit_pages(text_dict, filename):
     for page_num, page in text_dict.items():
         prompt = preprocessing_prompts.EXHIBIT_CHECK(page[0:100])
         claude_answer_raw = llm_utils.invoke_claude(
-            prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10
+            prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=10 # TODO: low priority, try increasing max_tokens and maybe pass multiple pages in to reduce overall calls
         )
         claude_answer_extracted = string_utils.extract_text_from_delimiters(
                             claude_answer_raw, Delimiter.PIPE