Merged in bugfix/mi-reimb-terms (pull request #740)

Bugfix/mi reimb terms

* test changes

* logging added

* Merge branch 'main' into bugfix/generic-reimb-terms

* parsing fixes

* fix missing reimbursements

* make service terms unique by adding additional context

* Merge branch 'main' into bugfix/mi-reimb-terms

* remomoving test changes

* remomoving test changes

* remomoving test changes

* test case for universal_json_load added


Approved-by: Katon Minhas

fieldExtraction/src/utils/string_utils.py

@@ -196,24 +196,34 @@ def universal_json_load(string_dict: str):
     take the last one positionally.
     """
     # Try to match dictionary or list of dictionaries first
-    dict_matches = re.findall(r"\{.*?\}|\[\s*?\{.*?\}\s*?\]", string_dict, re.DOTALL)
+    # It will find any substring starting with { or [ and ending with } or ].
+    # dict_matches = re.findall(r"\{.*?\}|\[\s*?\{.*?\}\s*?\]", string_dict, re.DOTALL)
+    dict_matches = re.findall(r'(\[.*\]|\{.*?\})', string_dict, re.DOTALL)
     if dict_matches:
         matched_str = dict_matches[-1]  # Take the last match in the string
         try:
             return json.loads(matched_str)
-        except json.JSONDecodeError as e:
-            logging.error(f"Failed to parse JSON: {e}")
-            logging.error(f"Original string: {string_dict}")
-            logging.error(f"Traceback: {traceback.format_exc()}")
-            # Use placeholders to preserve JSON structure characters for CSV compatibility
-            csv_safe_string = (
-                string_dict.replace(",", "[COMMA]")
-                .replace("\n", "[NEWLINE]")
-                .replace('"', "[QUOTE]")
-            )
-            raise ValueError(
-                f"JSON parsing failed: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
-            ) from e
+        except json.JSONDecodeError:
+            # If it fails, it might be a malformed list (e.g., "[{...} text {...}]").
+            # As a fallback, find the LAST dictionary within that malformed string.
+            fallback_matches = re.findall(r'\{.*?\}', matched_str, re.DOTALL)
+            if fallback_matches:
+                last_dict_in_string = fallback_matches[-1]
+                try:
+                    return json.loads(last_dict_in_string)
+                except json.JSONDecodeError as e:
+                    logging.error(f"Failed to parse JSON in fallback: {e}")
+                    logging.error(f"Original string: {string_dict}")
+                    logging.error(f"Traceback: {traceback.format_exc()}")
+                    # Use placeholders to preserve JSON structure characters for CSV compatibility
+                    csv_safe_string = (
+                        string_dict.replace(",", "[COMMA]")
+                        .replace("\n", "[NEWLINE]")
+                        .replace('"', "[QUOTE]")
+                    )
+                    raise ValueError(
+                        f"JSON parsing failed in fallback: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
+                    ) from e
 
     # If no dict pattern found, try to match list of strings
     list_matches = re.findall(r"\[(.*?)\]", string_dict, re.DOTALL)