Merged in feature/dynamic_and_generalized_branch (pull request #361)

feature/dynamic and generalized branch

* included txt files for nltk_data

* move nltk_data to src

* Fix last upload count

* Last upload count bugfix

* Fixed B processing

* Remove client-specific postprocessing

* split consolidate_output

* Run by file

* Fix output

* Reconfigure smart_chunk fields

* Add Full Context

* Fix merge conflicts

* Regex, Smart-Chunked, and Full working - not adding smart-chunked-->full when necessary

* Modernized run_full_context_fields()

* Switched set to list in field_context

* Move fields from smart_chunked to full_context as part of 'field_context' function

* Working version with placeholders

* Update poetry and pyproject

* Update s3 output

* Remove deprecated unit test

* Updated error messages

* Updated smart chunk ac name to one to one

* Update dependencies - end-to-end test for write s3 functional

* Add basic multithreading

* Send individual output to s3/local


Approved-by: Alex Galarce

fieldExtraction/src/utils/string_utils.py

@@ -223,27 +223,24 @@ def primary_string_to_dict(string_dict, filename):
             data.append(dict_)
     return data
 
+def universal_json_load(string_dict):
+    match = re.search(r'\{.*\}|\[.*\]', string_dict, re.DOTALL)
+    if match:
+        try:
+            return json.loads(match.group())
+        except json.JSONDecodeError:
+            pass
+    raise ValueError("No valid JSON object found in the input string")
+
+
 reimbursement_strings = [ # These are for `method='keyword'`
     "%",
     "$",
     "percent",
-    "compensation schedule",
-    "reimbursement schedule",
 ]
-
-# Maybe drop compensation / reimbursement schedules
-# contains and count should do a number followed by a percent or a number and a dollar
-# Also could be like "one hundred percent" or "one hundred dollars"
-# limit reimbursement strings to the first three
-
-# Then for counting AND containing, use the regex
-
-# maybe have `contains_reimbursement_keywords` and `contains_reimbursement_regex` functions
-# method='keyword' or method='regex'
-
 reimb_regex = r"(?<![$%])(?:\$\d+|\d+[$%])(?![$%])"
 
-def contains_reimbursement(text, page="1", method="keyword"): # string_funcs.py
+def contains_reimbursement(text, page="1", method="keyword"):
     """
     Checks if the given text contains any reimbursement-related keywords or patterns.
 
@@ -282,7 +279,16 @@ def count_reimbursements_in_exhibit(exhibit_text: str) -> int: #JUST by regex
     """
     return len(re.findall(reimb_regex, exhibit_text))
 
-def is_empty(value): # string_funcs.py
+def is_empty(value):
+    """
+    Checks if a value is considered empty or invalid.
+
+    Args:
+        value: The value to check.
+
+    Returns:
+        bool: True if the value is empty or invalid, False otherwise.
+    """
     if pd.isna(value):
         return True
     else: