Merged in feature/improved-error-out-logging (pull request #637)

Feature/improved error out logging

* Improve error logging in file processing and JSON parsing

* Enhance JSON parsing error handling for CSV compatibility

* black formatting

* isort

* Merge remote-tracking branch 'origin/main' into feature/improved-error-out-logging


Approved-by: Katon Minhas

fieldExtraction/src/utils/string_utils.py

@@ -6,15 +6,13 @@ from datetime import datetime
 
 import numpy as np
 import pandas as pd
+
 import src.utils.llm_utils as llm_utils
 from src import config
 from src.enums.delimiters import Delimiter
 from src.prompts import preprocessing_prompts
-from src.regex.regex_patterns import (
-    BACKTICK_PATTERN,
-    PIPE_PATTERN,
-    TRIPLE_BACKTICK_PATTERN,
-)
+from src.regex.regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
+                                      TRIPLE_BACKTICK_PATTERN)
 
 
 def extract_text_from_delimiters(
@@ -274,19 +272,26 @@ def universal_json_load(string_dict: str):
     # Try to match dictionary or list of dictionaries first
     dict_matches = re.findall(r"\{.*\}|\[\s*?\{.*\}\s*?\]", string_dict, re.DOTALL)
     if dict_matches:
-        matched_str = dict_matches[-1] # Take the last match in the string
+        matched_str = dict_matches[-1]  # Take the last match in the string
         try:
             return json.loads(matched_str)
         except json.JSONDecodeError as e:
             print(f"Failed to parse JSON: {e}")
             print(f"Original string: {string_dict}")
             print(f"Traceback: {traceback.format_exc()}")
-            raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
-
+            # Use placeholders to preserve JSON structure characters for CSV compatibility
+            csv_safe_string = (
+                string_dict.replace(",", "[COMMA]")
+                .replace("\n", "[NEWLINE]")
+                .replace('"', "[QUOTE]")
+            )
+            raise ValueError(
+                f"JSON parsing failed: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
+            ) from e
     # If no dict pattern found, try to match list of strings
     list_matches = re.findall(r"\[(.*?)\]", string_dict, re.DOTALL)
     if list_matches:
-        matched_str = f"[{list_matches[-1]}]" # Take the last match in the string
+        matched_str = f"[{list_matches[-1]}]"  # Take the last match in the string
         try:
             # Clean up the string list before parsing
             cleaned_str = re.sub(r'(?<!\\)"', '"', matched_str)  # Handle escaped quotes
@@ -298,7 +303,15 @@ def universal_json_load(string_dict: str):
             print(f"Failed to parse list: {e}")
             print(f"Original string: {string_dict}")
             print(f"Traceback: {traceback.format_exc()}")
-            raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
+            # Use placeholders to preserve JSON structure characters for CSV compatibility
+            csv_safe_string = (
+                string_dict.replace(",", "[COMMA]")
+                .replace("\n", "[NEWLINE]")
+                .replace('"', "[QUOTE]")
+            )
+            raise ValueError(
+                f"JSON parsing failed: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
+            ) from e
 
     return {}
 
@@ -590,6 +603,7 @@ def normalize_state_to_abbreviation(state_value: str) -> str:
     # If no match found, return original value
     return state_value
 
+
 def flatten_to_strings(items) -> list[str]:
     """Helper function to flatten and stringify any nested structure
 
@@ -608,5 +622,5 @@ def flatten_to_strings(items) -> list[str]:
                 result.append(str(item).strip())
     else:
         result.append(str(items).strip())
-    
-    return result
+
+    return result