Merged in hotfix/file-errors (pull request #448)

Hotfix/file errors

* Set up test file

* Fix regex pattern in json.loads

*  bug fix No valid JSON object found in the input string

* bug fixes

* moved try statement to parse fn

* invalid literal for int() with base 10 error fixed

* remove test

* remove excess print statements

* Raise error

* Remove todo

* Remove print statement


Approved-by: Katon Minhas

fieldExtraction/src/investment/code_funcs.py

@@ -381,8 +381,13 @@ def get_code_description(code, code_mapping):
                 description = code_mapping.mapping.get(expanded_code_range, "mismap")
                 # If mismap, try to handle numeric codes
                 if string_utils.is_empty(description) or description == "mismap":
-                    x_range_code_list = [int(f"{base_code}{i}") for i in range(10)]
-                    description = get_code_description(x_range_code_list, code_mapping)
+                    try:
+                        x_range_code_list = [int(f"{base_code}{i}") for i in range(10)]
+                        description = get_code_description(x_range_code_list, code_mapping)
+                    except Exception as e:
+                        print(f"Error message: {e}")
+                        description = "N/A"
+
 
             elif "-" in code_str and all(part.isdigit() for part in code_str.split("-")):
                 try:

fieldExtraction/src/investment/file_processing.py

@@ -95,6 +95,7 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi
 
     ################## RUN PROMPTS ##################
     one_to_n_results = []
+    # print(f"Total number of exhibits in {filename}: {len(exhibit_pages)}")
     for exhibit_page in exhibit_pages:
         exhibit_chunk = string_utils.get_exhibit_chunk(text_dict, exhibit_chunk_mapping, exhibit_page)
         if string_utils.contains_reimbursement(exhibit_chunk):

fieldExtraction/src/investment/one_to_n_funcs.py

@@ -15,6 +15,7 @@ import src.investment.carveout_funcs as carveout_funcs
 import src.investment.special_case_funcs as special_case_funcs
 from src.config import FIELD_JSON_PATH
 from src.regex.regex_utils import find_regex_matches
+from src.utils.string_utils import datetime_str
 
 def get_exhibit_header(exhibit_chunk, filename):
     """
@@ -227,7 +228,7 @@ def get_methodology_breakout(reimbursement_primary_answers: list[dict], filename
                 try:
                     methodology_breakout_answer = string_utils.universal_json_load(llm_response)
                 except ValueError as e:
-                    print(f"universal_string_load failed for input {llm_response} with error {e}\nassigning methodology_breakout_answer to []")
+                    # print(f"universal_string_load failed for input {llm_response} with error {e}\nassigning methodology_breakout_answer to []")
                     methodology_breakout_answer = []
                 for methodology_breakout_dict in methodology_breakout_answer:
                     # Fee Schedule Breakout
@@ -360,14 +361,18 @@ def reimbursement_level(exhibit_text, filename, reimbursement_level_fields, all_
         dict: The parsed LLM response as a list of dictionaries with unique identifiers
     """
     reimbursement_primary_answers = get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename) # Returns list of dicts
-    
+
+    # print(f"{datetime_str()} reimbursement_primary_answers complete for exhibit page {exhibit_page} of {filename}...")
     if reimbursement_primary_answers: # If any reimbursements found        
         carveout_answers = get_special_cases(reimbursement_primary_answers, filename)
+        # print(f"{datetime_str()} carveout_answers complete for exhibit page {exhibit_page} of {filename}...")
         
         # Later, we will only run methodology breakout for those without certain carveout indicators. We will run carveout-specific breakouts for the others
         methodology_breakout_answers = get_methodology_breakout(carveout_answers, filename)
-        # service_breakout_answers = get_service_breakout(methodology_breakout_answers, filename) # Not using as of now - keep temporarily while field initialization is finishing.
+        # print(f"{datetime_str()} methodology_breakout_answers complete for exhibit page {exhibit_page} of {filename}...")
+        
         code_breakout_answers = code_funcs.get_code_breakout(methodology_breakout_answers, filename, all_dataset)
+        # print(f"{datetime_str()} code_breakout_answers complete for exhibit page {exhibit_page} of {filename}...")
 
         # Add unique identifiers to each row composed of filename, exhibit page, service code, and index
         for idx, row in enumerate(code_breakout_answers):

fieldExtraction/src/investment/table_funcs.py

@@ -305,7 +305,7 @@ def split_long_tables(text_dict, table_dict, row_limit):
 
         subpage_index = 0
         for table in tables:
-            if len(table.table[list(table.table.keys())[0]]) > row_limit:
+            if table.table and len(table.table[list(table.table.keys())[0]]) > row_limit:
                 split_tables = table.split(row_limit)
                 for split_table in split_tables:
                     new_page_num = f"{page_num}.{subpage_index}"

fieldExtraction/src/preprocessing_funcs.py

@@ -22,7 +22,10 @@ def remove_page_indicators(contract_text: str) -> str:
         str: cleaned text with newlines and page number indicators removed
     """
     
-    cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text) # TODO: fix in main branch
+    if contract_text:
+        cleaned_text = re.sub(r"Page [0-9]+ of [0-9]+\n\n", " ", contract_text)
+    else:
+        cleaned_text = contract_text
     return cleaned_text
 
 

fieldExtraction/src/utils/string_utils.py

@@ -226,13 +226,15 @@ def primary_string_to_dict(string_dict, filename):
 
 def universal_json_load(string_dict):
     """Matches json dicts and list of dicts - NOT simple lists"""
-    match = re.search(r'\{.*\}|\[\{.*\}\]', string_dict, re.DOTALL)
+    match = re.search(r'\{.*\}|\[\s*?\{.*\}\s*?\]', string_dict, re.DOTALL) # Updated regex
     if match:
         try:
             return json.loads(match.group())
         except json.JSONDecodeError:
             pass
-    raise ValueError("No valid JSON object found in the input string")
+        raise ValueError("No valid JSON object found in the input string")
+    else:
+        return {}
 
 reimbursement_strings = [ # These are for `method='keyword'`
     "%",