Merged in bugfix/cpt-codes (pull request #812)

Bugfix/cpt codes

* formatting fix for revenue_cd

* format fix for procedure_codes

* bugfix for revenue_codes

* working version from code main

* code formatting from main and code main

* removed dubug prints

* removed local file paths

* standardize state fields

* updated test scenerios

* minor fix

* Merge branch 'main' into bugfix/cpt-codes
merging main into cpt code fix

* checks for all fields

* reused is_empty functionality

* moved normalize_to_json_list to utils

* Merge branch 'main' into bugfix/cpt-codes
merging upto date main

* minor touches

* pytest fix


Approved-by: Katon Minhas

fieldExtraction/src/utils/string_utils.py

@@ -1,4 +1,5 @@
 import json
+import ast
 import logging
 import re
 import traceback
@@ -631,7 +632,7 @@ def parse_state_field_to_list(raw_value):
         logging.error(f"Could not convert value to string: {raw_value!r}")
         return []
 
-def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
+def normalize_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
     if field_name not in answers_dict or not answers_dict[field_name]:
         answers_dict[field_name] = []
         return answers_dict
@@ -639,7 +640,7 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID
     raw_value = answers_dict[field_name]
 
     # ---- Step 1: Parse into list ----
-    state_list = parse_state_field_to_list(raw_value)
+    state_list = parse_raw_state_value(raw_value)
 
     # ---- Step 2: Normalize each state ----
     normalized_states = []
@@ -652,3 +653,100 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID
 
     answers_dict[field_name] = normalized_states
     return answers_dict
+
+
+def parse_raw_state_value(raw_value):
+    """
+    Parses various input formats into a flat list of state strings.
+    Handles: strings, lists, nested lists, and various delimiters.
+    """
+    if not raw_value:
+        return []
+    
+    # If it's a string that looks like a list, try to parse it
+    if isinstance(raw_value, str):
+        stripped = raw_value.strip()
+        
+        # Try to parse as Python literal (handles "[['NC', 'WA'], 'CA|UT']")
+        if stripped.startswith('['):
+            try:
+                parsed = ast.literal_eval(stripped)
+                return parse_raw_state_value(parsed)  # Recursive call with parsed list
+            except (ValueError, SyntaxError):
+                pass  # Not valid Python literal, continue with string parsing
+        
+        # Regular string splitting on delimiters
+        states = re.split(r'[|,;\s]+', stripped)
+        return [s.strip() for s in states if s.strip()]
+    
+    if isinstance(raw_value, list):
+        result = []
+        for item in raw_value:
+            result.extend(parse_raw_state_value(item))  # Recursive call
+        return result
+    
+    return [str(raw_value)]
+
+
+def normalize_to_json_list(val):
+    """
+    Normalizes a value to a JSON list format.
+    Handles: pipe-delimited strings, string representations of Python lists,
+    actual lists with pipe-delimited items, and single values.
+    """    
+    # Handle None and empty values first
+    if is_empty(val):
+        return val
+    
+    # Handle pandas NA/NaN - check for scalar first
+    try:
+        if pd.isna(val):
+            return val
+    except (ValueError, TypeError):
+        # pd.isna fails on arrays/lists, which is fine - we'll handle them below
+        pass
+    
+    if isinstance(val, str):
+        # Check if it's a string representation of a Python list (e.g., "['item1', 'item2']")
+        if val.startswith("[") and val.endswith("]"):
+            try:
+                # Try JSON parse first
+                parsed = json.loads(val)
+            except json.JSONDecodeError:
+                # If JSON fails, try ast.literal_eval for Python list syntax
+                try:
+                    parsed = ast.literal_eval(val)
+                except (ValueError, SyntaxError):
+                    return val
+            
+            if isinstance(parsed, list):
+                expanded = []
+                for item in parsed:
+                    if isinstance(item, str) and "|" in item:
+                        expanded.extend(item.split("|"))
+                    else:
+                        expanded.append(item)
+                result = json.dumps(expanded)
+                return result
+            return val
+        
+        # Pipe-delimited format - convert to JSON list
+        if "|" in val:
+            result = json.dumps(val.split("|"))
+            return result
+        
+        # Single value - convert to JSON list
+        result = json.dumps([val])
+        return result
+    
+    if isinstance(val, list):
+        # Check if list contains pipe-delimited strings and expand them
+        expanded = []
+        for item in val:
+            if isinstance(item, str) and "|" in item:
+                expanded.extend(item.split("|"))
+            else:
+                expanded.append(item)
+        result = json.dumps(expanded)
+        return result
+    return val