Merged in feature/integrate-standard-and-complex (pull request #356)

Feature/integrate standard and complex

* last edits to merge_tables_draft_v2 before moving to table_funcs

* migrate to table_utils.py

* fix typehint that caused mypy error

* fix mypy errors

* black and isort

* fix error with multi-table pages

* update poetry.lock

* add tests, fix bug when table start marker or table end marker is missing

* added more tests

* tests for get_str_dictionaries_from_text()

* fix edge case in get_str_dictionaries_from_text

* fix mypy error

* more tests

* add tests, add handling for invalid dictionaries

* add tests

* add tests for insert_column_headers()

* update requirements

* add new simple/complex logic to preprocess.py

* remove files used solely for testing

* split pages into sub-pages by tables

* add table split on end marker.  Also add tests

* add docstrings, change control flow

* remove unused tests, add TODO for new tests

* get in prompt changes and intermediate decisions

* TODO for future enhancement for get_exhibit_pages


Approved-by: Katon Minhas

fieldExtraction/tests/string_utils_test.py

@@ -1,6 +1,6 @@
 import pytest
 import src.utils as utils
-from src.utils.string_utils import extract_text_from_delimiters, json_parsing_search, secondary_string_to_dict, primary_string_to_dict, contains_reimbursement, is_empty
+from src.utils.string_utils import extract_text_from_delimiters, json_parsing_search, secondary_string_to_dict, primary_string_to_dict, contains_reimbursement, is_empty, count_reimbursements_in_exhibit
 import src.utils.llm_utils as llm_utils
 import numpy as np
 import pandas as pd
@@ -175,7 +175,7 @@ class TestStringUtils:
         text = ["This is a list, not a dictionary or string."]
         page = "1"
         result = contains_reimbursement(text, page)
-        assert result is None
+        assert result is False
         captured = capsys.readouterr()
         assert "contains_reimbursement - Invalid data type" in captured.out
 
@@ -194,3 +194,17 @@ class TestStringUtils:
     def test_is_empty(self, value, expected):
         result = is_empty(value)
         assert result == expected
+
+class TestCountReimbursementsInExhibit:
+    @pytest.mark.parametrize("exhibit_text, expected_count", [
+        ("This exhibit includes a 10% reimbursement and a $100 reimbursement.", 2),
+        ("No reimbursements mentioned here.", 0),
+        ("Reimbursement of 50% and another reimbursement of $200.", 2),
+        ("100 percent reimbursement and fifty dollars reimbursement.", 0),
+        ("", 0),
+        ("Reimbursement: 20% and $300.", 2),
+        ("Reimbursement: 20% and $300. Another 10% reimbursement.", 3),
+    ])
+    def test_count_reimbursements_in_exhibit(self, exhibit_text, expected_count):
+        result = count_reimbursements_in_exhibit(exhibit_text)
+        assert result == expected_count