Merged in feature/integrate-standard-and-complex (pull request #356)

Feature/integrate standard and complex

* last edits to merge_tables_draft_v2 before moving to table_funcs

* migrate to table_utils.py

* fix typehint that caused mypy error

* fix mypy errors

* black and isort

* fix error with multi-table pages

* update poetry.lock

* add tests, fix bug when table start marker or table end marker is missing

* added more tests

* tests for get_str_dictionaries_from_text()

* fix edge case in get_str_dictionaries_from_text

* fix mypy error

* more tests

* add tests, add handling for invalid dictionaries

* add tests

* add tests for insert_column_headers()

* update requirements

* add new simple/complex logic to preprocess.py

* remove files used solely for testing

* split pages into sub-pages by tables

* add table split on end marker.  Also add tests

* add docstrings, change control flow

* remove unused tests, add TODO for new tests

* get in prompt changes and intermediate decisions

* TODO for future enhancement for get_exhibit_pages


Approved-by: Katon Minhas

fieldExtraction/tests/preprocessing_funcs_test.py

@@ -1,4 +1,5 @@
 import pytest
+from src.preprocessing_funcs import split_text
 from src.preprocessing_funcs import (
     remove_page_indicators,
     split_text,
@@ -21,31 +22,6 @@ class TestPreprocessingFuncs:
     def test_clean_newlines(self, input_text, expected_output):
         assert remove_page_indicators(input_text) == expected_output
 
-    # Test cases for split_text
-    @pytest.mark.parametrize("input_text, expected_output", [
-        # Splits on page markers
-        (
-            "Title Start of Page No. = 1\nContent1\nStart of Page No. = 2\nContent2",
-            {"Title" : "Title ", "1": "Content1\n", "2": "Content2"}
-        ),
-        # Ignores "Document" key
-        (
-            "Document\nStart of Page No. = 1\nContent1",
-            {"1": "Content1"}
-        ),
-        # Handles empty input
-        ("", {}),
-        # Handles no page markers
-        ("This is a single page.", {"This": "This is a single page."}),
-        # Handles multiple page markers
-        (
-            "Introduction\nStart of Page No. = 1\nContent1\nStart of Page No. = 2\nContent2\nStart of Page No. = 3\nContent3",
-            {'Introduction': 'Introduction\n', '1': 'Content1\n', '2': 'Content2\n', '3': 'Content3'}
-        ),
-    ])
-    def test_split_text(self, input_text, expected_output):
-        assert split_text(input_text) == expected_output
-
     # Test cases for clean_law_symbols
     @pytest.mark.parametrize("input_text, expected_output", [
         # Replaces double dollar signs