Merged in feature/implicit-proc-mapping (pull request #531)

Feature/implicit proc mapping

* Testing scripts

* Refactor code primary

* implicit draft

* Merge branch 'main' into feature/implicit-proc-mapping

* Prompt update

* Prompt changes

* Experiments with preprocessing

* code updates

* Merged main into feature/implicit-proc-mapping

* Add condition to avoid Covered Services

* Add functions to main script

* remove explanations

* Error handling

* Genericize prompt

* Final prompt updates

* remove test file

* prep for merge

* Merged main into feature/implicit-proc-mapping

* Merged main into feature/implicit-proc-mapping


Approved-by: Alex Galarce

fieldExtraction/src/utils/string_utils.py

@@ -264,19 +264,34 @@ def universal_json_load(string_dict: str):
         the traceback as well as the offending string are printed.
     
     """
-    match = re.search(r'\{.*\}|\[\s*?\{.*\}\s*?\]', string_dict, re.DOTALL) # Updated regex
-    if match:
-        matched_str = match.group()
+    # Try to match dictionary or list of dictionaries first
+    dict_match = re.search(r'\{.*\}|\[\s*?\{.*\}\s*?\]', string_dict, re.DOTALL)
+    if dict_match:
+        matched_str = dict_match.group()
         try:
             return json.loads(matched_str)
         except json.JSONDecodeError as e:
             print(f"Failed to parse JSON: {e}")
-            # print(f"Problematic JSON string: {matched_str}")
             print(f"Original string: {string_dict}")
             print(f"Traceback: {traceback.format_exc()}")
             raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
-    else:
-        return {}
+    
+    # If no dict pattern found, try to match list of strings
+    list_match = re.search(r'\[(.*?)\]', string_dict, re.DOTALL)
+    if list_match:
+        matched_str = list_match.group()
+        try:
+            # Clean up the string list before parsing
+            cleaned_str = re.sub(r'(?<!\\)"', '"', matched_str)  # Handle escaped quotes
+            cleaned_str = re.sub(r"'", '"', cleaned_str)  # Replace single quotes with double quotes
+            return json.loads(cleaned_str)
+        except json.JSONDecodeError as e:
+            print(f"Failed to parse list: {e}")
+            print(f"Original string: {string_dict}")
+            print(f"Traceback: {traceback.format_exc()}")
+            raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e
+    
+    return {}
 
 reimbursement_strings = [ # These are for `method='keyword'`
     "%",