Merged in feature/integrate-standard-and-complex (pull request #356)

Feature/integrate standard and complex

* last edits to merge_tables_draft_v2 before moving to table_funcs

* migrate to table_utils.py

* fix typehint that caused mypy error

* fix mypy errors

* black and isort

* fix error with multi-table pages

* update poetry.lock

* add tests, fix bug when table start marker or table end marker is missing

* added more tests

* tests for get_str_dictionaries_from_text()

* fix edge case in get_str_dictionaries_from_text

* fix mypy error

* more tests

* add tests, add handling for invalid dictionaries

* add tests

* add tests for insert_column_headers()

* update requirements

* add new simple/complex logic to preprocess.py

* remove files used solely for testing

* split pages into sub-pages by tables

* add table split on end marker.  Also add tests

* add docstrings, change control flow

* remove unused tests, add TODO for new tests

* get in prompt changes and intermediate decisions

* TODO for future enhancement for get_exhibit_pages


Approved-by: Katon Minhas

fieldExtraction/src/utils/string_utils.py

@@ -223,27 +223,64 @@ def primary_string_to_dict(string_dict, filename):
             data.append(dict_)
     return data
 
+reimbursement_strings = [ # These are for `method='keyword'`
+    "%",
+    "$",
+    "percent",
+    "compensation schedule",
+    "reimbursement schedule",
+]
 
-def contains_reimbursement(text, page="1"): # string_funcs.py
-    if isinstance(text, dict):
-        return page.isdigit() and (
-            "%" in text[page]
-            or "$" in text[page]
-            or "percent " in text[page]
-            or "compensation schedule" in text[page].lower()
-            or "reimbursement schedule" in text[page].lower()
-        )
-    elif isinstance(text, str):
-        return (
-            "%" in text
-            or "$" in text
-            or "percent " in text
-            or "compensation schedule" in text.lower()
-            or "reimbursement schedule" in text.lower()
-        )
+# Maybe drop compensation / reimbursement schedules
+# contains and count should do a number followed by a percent or a number and a dollar
+# Also could be like "one hundred percent" or "one hundred dollars"
+# limit reimbursement strings to the first three
+
+# Then for counting AND containing, use the regex
+
+# maybe have `contains_reimbursement_keywords` and `contains_reimbursement_regex` functions
+# method='keyword' or method='regex'
+
+reimb_regex = r"(?<![$%])(?:\$\d+|\d+[$%])(?![$%])"
+
+def contains_reimbursement(text, page="1", method="keyword"): # string_funcs.py
+    """
+    Checks if the given text contains any reimbursement-related keywords or patterns.
+
+    Args:
+        text (str or dict): The text to check. If a dictionary is provided, it should have page numbers as keys.
+        page (str, optional): The page number to check in the dictionary. Defaults to "1".
+        method (str, optional): The method to use for checking. Options are "keyword"
+         and "regex". Defaults to "keyword".
+
+    Returns:
+        bool: True if any reimbursement-related keyword or pattern is found, False otherwise.
+    """
+    if method == "keyword":
+        if isinstance(text, dict):
+            text_to_check = text.get(page, "").lower()
+        elif isinstance(text, str):
+            text_to_check = text.lower()
+        else:
+            print("contains_reimbursement - Invalid data type")
+            return False
+        return any(keyword in text_to_check for keyword in reimbursement_strings)
+    elif method == "regex":
+        return bool(re.search(reimb_regex, text))
     else:
-        print("contains_reimbursement - Invalid data type")
+        raise ValueError("Invalid method. Choose 'keyword' or 'regex'.")
 
+def count_reimbursements_in_exhibit(exhibit_text: str) -> int: #JUST by regex
+    """Counts reimbursements in an exhibit text.  Reimbursements are detected by a regex
+    search as defined by `reimb_regex`.
+
+    Args:
+        exhibit_text (str): Input exhibit text
+
+    Returns:
+        int: Number of reimbursements detected
+    """
+    return len(re.findall(reimb_regex, exhibit_text))
 
 def is_empty(value): # string_funcs.py
     if pd.isna(value):
@@ -251,7 +288,7 @@ def is_empty(value): # string_funcs.py
     else:
         empty_values = [None, "", "N/A", "NA", "null", "none", "NaN", np.nan, "nan"]
         return value in empty_values
-    
+
 
 def get_exhibit_chunk(text_dict: dict, 
                       exhibit_chunk_mapping: dict,