Merged in bugfix/cpt-codes (pull request #812)

Bugfix/cpt codes

* formatting fix for revenue_cd

* format fix for procedure_codes

* bugfix for revenue_codes

* working version from code main

* code formatting from main and code main

* removed dubug prints

* removed local file paths

* standardize state fields

* updated test scenerios

* minor fix

* Merge branch 'main' into bugfix/cpt-codes
merging main into cpt code fix

* checks for all fields

* reused is_empty functionality

* moved normalize_to_json_list to utils

* Merge branch 'main' into bugfix/cpt-codes
merging upto date main

* minor touches

* pytest fix


Approved-by: Katon Minhas

fieldExtraction/src/codes/code_funcs.py

@@ -1,3 +1,5 @@
+import ast
+import json
 import logging
 import os
 import re
@@ -29,12 +31,18 @@ def clean_service(service, constants: Constants) -> str:
 
     # Map common values
     for acronym, full_form in constants.SYNONYM_MAP.items():
+        if string_utils.is_empty(acronym) or string_utils.is_empty(full_form):
+            continue
         service = re.sub(rf"\b{re.escape(acronym)}\b", full_form, service)
 
     # Remove other values
     for term in constants.REMOVAL_LIST:
+        if term is None:
+            continue
         service = re.sub(rf"\b{re.escape(term)}\b", "", service).strip()
         service = re.sub(r"\s+", " ", service)
+    if service is None:
+        return ""
     service = service.replace(" - ", "-")
 
     if any([v in service for v in ["UNLISTED", "UNCATEGORIZED", "NON-LISTED"]]):
@@ -148,15 +156,8 @@ def code_category(service, proc_category, hcpcs_level2_mapping, filename):
             code = list(hcpcs_level2_mapping.keys())[
                 list(hcpcs_level2_mapping.values()).index(answer)
             ]
-            code_answer_dict["PROCEDURE_CD"].append(code)
+            code_answer_dict["PROCEDURE_CD"].append(str(code))
-            code_answer_dict["PROCEDURE_CD_DESC"].append(answer)
+            code_answer_dict["PROCEDURE_CD_DESC"].append(str(answer))
-
-    if len(code_answer_dict["PROCEDURE_CD"]) == 1:
-        code_answer_dict["PROCEDURE_CD"] = code_answer_dict["PROCEDURE_CD"][0]
-        code_answer_dict["PROCEDURE_CD_DESC"] = code_answer_dict["PROCEDURE_CD_DESC"][0]
-    else:
-        code_answer_dict["PROCEDURE_CD"] = str(code_answer_dict["PROCEDURE_CD"])
-        code_answer_dict["PROCEDURE_CD_DESC"] = str(code_answer_dict["PROCEDURE_CD_DESC"])
     
     return code_answer_dict
 
@@ -181,13 +182,13 @@ def code_implicit_special(service, filename):
     )
 
     special_case_mapping = {
-        "Drugs": "J0000-J9999",
+        "Drugs": ["J0000-J9999"],
-        "Vaccines": "J0000-J9999|90471‑90474|90620‑90621|90633|90647‑90648|90651|90670|90672|90680‑90681|90686|90696|90698|90700|90707|90710|90713|90714",
+        "Vaccines": ["J0000-J9999", "90471‑90474", "90620‑90621", "90633", "90647‑90648", "90651", "90670", "90672", "90680‑90681", "90686", "90696", "90698", "90700", "90707", "90710", "90713", "90714"],
-        "Surgery": "10004-69990",
+        "Surgery": ["10004-69990"],
-        "PT/OT/ST": "92507-92508|92526|97014|97110|97112|97116|97150|97161-97168|97530|97535",
+        "PT/OT/ST": ["92507-92508", "92526", "97014", "97110", "97112", "97116", "97150", "97161-97168", "97530", "97535"],
-        "PT": "PT Codes TBD",
+        "PT": ["PT Codes TBD"],
-        "OT": "OT Codes TBD",
+        "OT": ["OT Codes TBD"],
-        "ST": "ST Codes TBD",
+        "ST": ["ST Codes TBD"],
     }
     code_answer_dict = {}
     if claude_answer_final in special_case_mapping:
@@ -343,33 +344,39 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants):
                     str(key) for key, val in cpt_mapping.items() if val == description
                 ]
                 if matching_codes:
-                    proc_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        proc_codes.extend(code.split("|"))
                     proc_descs.append(description)
             if description in hcpcs_mapping.values():
                 matching_codes = [
                     str(key) for key, val in hcpcs_mapping.items() if val == description
                 ]
                 if matching_codes:
-                    proc_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        proc_codes.extend(code.split("|"))
                     proc_descs.append(description)
             if description in rev_mapping.values():
                 matching_codes = [
                     str(key) for key, val in rev_mapping.items() if val == description
                 ]
                 if matching_codes:
-                    rev_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        rev_codes.extend(code.split("|"))
                     rev_descs.append(description)
 
         # Combine answers
         if proc_codes:
-            code_answer_dict["PROCEDURE_CD"] = proc_codes[0] if len(proc_codes) == 1 else str(proc_codes)
+            code_answer_dict["PROCEDURE_CD"] = proc_codes
-            code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs[0] if len(proc_descs) == 1 else str(proc_descs)
+            code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs
             code_answer_dict["CODE_METHODOLOGY"] = (
                 f"Implicit - Level {level_dict['level_suffix']}"
             )
         if rev_codes:
-            code_answer_dict["REVENUE_CD"] = rev_codes[0] if len(rev_codes) == 1 else str(rev_codes)
+            code_answer_dict["REVENUE_CD"] = rev_codes
-            code_answer_dict["REVENUE_CD_DESC"] = rev_descs[0] if len(rev_descs) == 1 else str(rev_descs)
+            code_answer_dict["REVENUE_CD_DESC"] = rev_descs
             code_answer_dict["CODE_METHODOLOGY"] = (
                 f"Implicit - Level {level_dict['level_suffix']}"
             )
@@ -521,6 +528,24 @@ def get_implicit_runs(answer_dict):
     return run_dict
 
 
+def normalize_answer_dict_codes(answer_dict):
+    """
+    Normalizes code fields in an answer dictionary to JSON list format.
+    
+    Args:
+        answer_dict (dict): Dictionary containing code fields to normalize.
+    Returns:
+        dict: Dictionary with normalized code fields.
+    """
+    fields_to_normalize = ["REVENUE_CD", "PROCEDURE_CD", "CPT4_PROC_CD"]
+    
+    for field in fields_to_normalize:
+        if field in answer_dict:
+            answer_dict[field] = string_utils.normalize_to_json_list(answer_dict[field])
+    
+    return answer_dict
+
+
 def extract_codes_from_service(answer_dict, constants: Constants):
     """
     Extracts codes from the service term in the answer dictionary.
@@ -548,8 +573,8 @@ def extract_codes_from_service(answer_dict, constants: Constants):
     service, bill_type = answer_dict.get("SERVICE_TERM"), answer_dict.get(
         "BILL_TYPE_CD_DESC"
     )
-    filename = answer_dict.get("FILENAME")
+    filename = answer_dict.get("FILENAME", "")
-    methodology = answer_dict.get("REIMB_TERM")
+    methodology = answer_dict.get("REIMB_TERM", "")
 
     if string_utils.is_empty(service):
         return {}
@@ -573,14 +598,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
     if service_clean == "UNLISTED":
         answer_dict["PROCEDURE_CD_DESC"] = "UNLISTED"
         answer_dict["CODE_METHODOLOGY"] = "UNLISTED"
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)
 
     # Exit point if N/A or in DO_NOT_RUN list
     if string_utils.is_empty(service_clean) or any(
         [v in service_clean for v in constants.DO_NOT_RUN]
     ):
         answer_dict["CODE_METHODOLOGY"] = "Generic - Before Prompts"
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)
 
     # Explicit Codes (run always)
     code_answer_dict = code_explicit(service_clean, methodology, filename)
@@ -595,7 +620,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
         else:
             code_answer_dict["CODE_METHODOLOGY"] = "Explicit"
         answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)
 
     # Implicit Codes: Code Categories
     if any(["Category:" in v for v in code_answer_dict.get("PROCEDURE_CD", "")]):
@@ -610,14 +635,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
         ):  # if any code value is not empty, return
             code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Letter Category"
             answer_dict.update(code_answer_dict)
-            return answer_dict
+            return normalize_answer_dict_codes(answer_dict)
 
     # Implicit Codes: Special Categories
     code_answer_dict = code_implicit_special(service_clean, "")
     if code_answer_dict:
         code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Special Case"
         answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)
 
     # Implicit Codes: Levels 1 and 2
     code_answer_dict = code_implicit_rag(
@@ -627,7 +652,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
         not string_utils.is_empty(value) for value in code_answer_dict.values()
     ):  # if any code value is not empty, return
         answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)
 
     # No Match - Why? Generic or Specific
     last_check_answer = code_last_check(service_clean, "")
@@ -636,7 +661,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
     elif last_check_answer == "Specific":
         answer_dict["CODE_METHODOLOGY"] = "Specific - No Match"
 
-    return answer_dict
+    return normalize_answer_dict_codes(answer_dict)
 
 
 def fill_claim_type(answer_dicts):
@@ -689,8 +714,14 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
         for answer_dict in answer_dicts_with_code
     ]
 
-    return pd.DataFrame(answer_dicts_with_code)
+    df = pd.DataFrame(answer_dicts_with_code)
 
+    # Normalize code columns to JSON list format
+    for col in ["PROCEDURE_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]:
+        if col in df.columns:
+            df[col] = df[col].apply(string_utils.normalize_to_json_list)
+
+    return df
 
 def grouper_breakout(results_with_code: pd.DataFrame):
     """
@@ -729,4 +760,4 @@ def grouper_breakout(results_with_code: pd.DataFrame):
             answer_dict.update(grouper_answer)
         final_answer_dicts.append(answer_dict)
 
-    return pd.DataFrame(final_answer_dicts)
+    return pd.DataFrame(final_answer_dicts)

fieldExtraction/src/codes/main.py

@@ -1,7 +1,10 @@
 import concurrent.futures
+import json
 
+import pandas as pd
 import src.codes.code_funcs as code_funcs
 import src.config as config
+import src.utils.string_utils as string_utils
 import src.utils.io_utils as io_utils
 from constants.constants import Constants
 from sentence_transformers import SentenceTransformer
@@ -9,11 +12,10 @@ from sentence_transformers import SentenceTransformer
 
 def main():
     # Constants
-    INPUT_FILE_PATH = "Codes-Test-Doc.xlsx"
+    INPUT_FILE_PATH = ""
-    OUTPUT_FILE_PATH = "Doczy-Codes-Test-9.csv"
+    OUTPUT_FILE_PATH = ""
 
     df = io_utils.read_local(INPUT_FILE_PATH)
-
     # Read Constants
     constants = Constants()
     model = SentenceTransformer("all-roberta-large-v1")
@@ -35,7 +37,7 @@ def main():
             }
 
             # Process the row
-            result_dict = code_funcs.extract_codes_from_service(answer_dict)
+            result_dict = code_funcs.extract_codes_from_service(answer_dict, constants)
             return idx, result_dict
         except Exception as e:
             print(f"Error processing row {idx}: {e}")
@@ -66,16 +68,16 @@ def main():
             if key not in df.columns:
                 df[key] = None
 
-            # Ensure value is scalar
-            if isinstance(value, (list, dict)):
-                value = str(value)  # Convert lists/dicts to strings
-
             try:
-                # Update the specific cell
                 df.at[idx, key] = value
             except Exception as e:
                 print(f"Error updating DataFrame at row {idx}, key '{key}': {e}")
 
+    # NORMALIZE ALL CODE COLUMNS
+    for col in ["CPT4_PROC_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]:
+        if col in df.columns:
+            df[col] = df[col].apply(string_utils.normalize_to_json_list)
+
     # Save the updated DataFrame
     try:
         if OUTPUT_FILE_PATH:
@@ -85,7 +87,6 @@ def main():
             print("No output path specified. Results not saved.")
     except Exception as e:
         print(f"Error saving results: {e}")
-        # Try to save to a backup location
         backup_path = "backup_results.csv"
         try:
             df.to_csv(backup_path, index=False)
@@ -97,4 +98,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py

@@ -318,7 +318,7 @@ def run_hybrid_smart_chunked_fields(
             )
 
         # Normalize PROVIDER_STATE  
-        answers_dict = string_utils.normalize_provider_state_field(answers_dict)
+        answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PROVIDER_STATE")
 
         # Normalize PAYER_STATE to standardized two-letter abbreviation
         for state_field in ["PAYER_STATE"]:
@@ -328,7 +328,8 @@ def run_hybrid_smart_chunked_fields(
                         answers_dict[state_field]
                     )
                 )
-
+        # Normalize PAYER_STATE  
+        answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PAYER_STATE")
         return answers_dict
 
     except Exception as e:

fieldExtraction/src/investment/one_to_one_funcs.py

@@ -258,11 +258,13 @@ def run_full_context_fields(
         full_context_answers_dict["PAYER_STATE"] = string_utils.normalize_state_to_abbreviation(
             full_context_answers_dict["PAYER_STATE"]
         )
+    full_context_answers_dict = string_utils.normalize_state_field(
+        full_context_answers_dict, field_name="PAYER_STATE")
 
     # normalize PROVIDER_STATE to two-letter abbreviation
     if "PROVIDER_STATE" in full_context_answers_dict:
-        full_context_answers_dict = string_utils.normalize_provider_state_field(
+        full_context_answers_dict = string_utils.normalize_state_field(
-            full_context_answers_dict
+            full_context_answers_dict, field_name="PROVIDER_STATE"
         )
     # Run Special Case Breakout on any breakout terms
     full_context_answers_dict = one_to_one_breakout(full_context_answers_dict, filename)

fieldExtraction/src/utils/string_utils.py

@@ -1,4 +1,5 @@
 import json
+import ast
 import logging
 import re
 import traceback
@@ -631,7 +632,7 @@ def parse_state_field_to_list(raw_value):
         logging.error(f"Could not convert value to string: {raw_value!r}")
         return []
 
-def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
+def normalize_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
     if field_name not in answers_dict or not answers_dict[field_name]:
         answers_dict[field_name] = []
         return answers_dict
@@ -639,7 +640,7 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID
     raw_value = answers_dict[field_name]
 
     # ---- Step 1: Parse into list ----
-    state_list = parse_state_field_to_list(raw_value)
+    state_list = parse_raw_state_value(raw_value)
 
     # ---- Step 2: Normalize each state ----
     normalized_states = []
@@ -652,3 +653,100 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID
 
     answers_dict[field_name] = normalized_states
     return answers_dict
+
+
+def parse_raw_state_value(raw_value):
+    """
+    Parses various input formats into a flat list of state strings.
+    Handles: strings, lists, nested lists, and various delimiters.
+    """
+    if not raw_value:
+        return []
+    
+    # If it's a string that looks like a list, try to parse it
+    if isinstance(raw_value, str):
+        stripped = raw_value.strip()
+        
+        # Try to parse as Python literal (handles "[['NC', 'WA'], 'CA|UT']")
+        if stripped.startswith('['):
+            try:
+                parsed = ast.literal_eval(stripped)
+                return parse_raw_state_value(parsed)  # Recursive call with parsed list
+            except (ValueError, SyntaxError):
+                pass  # Not valid Python literal, continue with string parsing
+        
+        # Regular string splitting on delimiters
+        states = re.split(r'[|,;\s]+', stripped)
+        return [s.strip() for s in states if s.strip()]
+    
+    if isinstance(raw_value, list):
+        result = []
+        for item in raw_value:
+            result.extend(parse_raw_state_value(item))  # Recursive call
+        return result
+    
+    return [str(raw_value)]
+
+
+def normalize_to_json_list(val):
+    """
+    Normalizes a value to a JSON list format.
+    Handles: pipe-delimited strings, string representations of Python lists,
+    actual lists with pipe-delimited items, and single values.
+    """    
+    # Handle None and empty values first
+    if is_empty(val):
+        return val
+    
+    # Handle pandas NA/NaN - check for scalar first
+    try:
+        if pd.isna(val):
+            return val
+    except (ValueError, TypeError):
+        # pd.isna fails on arrays/lists, which is fine - we'll handle them below
+        pass
+    
+    if isinstance(val, str):
+        # Check if it's a string representation of a Python list (e.g., "['item1', 'item2']")
+        if val.startswith("[") and val.endswith("]"):
+            try:
+                # Try JSON parse first
+                parsed = json.loads(val)
+            except json.JSONDecodeError:
+                # If JSON fails, try ast.literal_eval for Python list syntax
+                try:
+                    parsed = ast.literal_eval(val)
+                except (ValueError, SyntaxError):
+                    return val
+            
+            if isinstance(parsed, list):
+                expanded = []
+                for item in parsed:
+                    if isinstance(item, str) and "|" in item:
+                        expanded.extend(item.split("|"))
+                    else:
+                        expanded.append(item)
+                result = json.dumps(expanded)
+                return result
+            return val
+        
+        # Pipe-delimited format - convert to JSON list
+        if "|" in val:
+            result = json.dumps(val.split("|"))
+            return result
+        
+        # Single value - convert to JSON list
+        result = json.dumps([val])
+        return result
+    
+    if isinstance(val, list):
+        # Check if list contains pipe-delimited strings and expand them
+        expanded = []
+        for item in val:
+            if isinstance(item, str) and "|" in item:
+                expanded.extend(item.split("|"))
+            else:
+                expanded.append(item)
+        result = json.dumps(expanded)
+        return result
+    return val

fieldExtraction/tests/test_code_funcs.py

@@ -108,9 +108,9 @@ class TestCodeFuncs(unittest.TestCase):
             "J CODES", proc_category, hcpcs_mapping, "test.pdf"
         )
 
-        # Verify the results
+        # Verify the results - code_category returns lists (JSON conversion happens in extract_codes_from_service)
-        self.assertEqual(result["PROCEDURE_CD"], "['J0001', 'J0002']")
+        self.assertEqual(result["PROCEDURE_CD"], ["J0001", "J0002"])
-        self.assertEqual(result["PROCEDURE_CD_DESC"], "['Drug A', 'Drug B']")
+        self.assertEqual(result["PROCEDURE_CD_DESC"], ["Drug A", "Drug B"])
         self.assertIn("J0001", result["PROCEDURE_CD"])
         self.assertIn("J0002", result["PROCEDURE_CD"])
         self.assertIn("Drug A", result["PROCEDURE_CD_DESC"])
@@ -124,9 +124,9 @@ class TestCodeFuncs(unittest.TestCase):
         mock_invoke_claude.return_value = "mock_response"
         mock_extract.return_value = "Drugs"
 
-        # Test drug category
+        # Test drug category - returns a list
         result = code_funcs.code_implicit_special("DRUG SERVICE", "test.pdf")
-        self.assertEqual(result["PROCEDURE_CD"], "J0000-J9999")
+        self.assertEqual(result["PROCEDURE_CD"], ["J0000-J9999"])
         self.assertEqual(result["PROCEDURE_CD_DESC"], "Drugs")
 
         # Test no match
@@ -221,9 +221,9 @@ class TestCodeFuncs(unittest.TestCase):
                 "TEST SERVICE", implicit_run_dict, "test.pdf", self.constants
             )
 
-            # Verify results
+            # Verify results - code_implicit_rag returns lists
-            self.assertEqual(result["PROCEDURE_CD"], "12345")
+            self.assertEqual(result["PROCEDURE_CD"], ["12345"])
-            self.assertEqual(result["PROCEDURE_CD_DESC"], "Test Procedure")
+            self.assertEqual(result["PROCEDURE_CD_DESC"], ["Test Procedure"])
             self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Level 1")
 
         # Test exception handling
@@ -333,12 +333,12 @@ class TestCodeFuncs(unittest.TestCase):
         }
         mock_implicit_runs.return_value = {"PROCEDURE_CD": True, "REVENUE_CD": True}
 
-        # Test explicit code match
+        # Test explicit code match - extract_codes_from_service normalizes to JSON list format
         mock_explicit.return_value = {"PROCEDURE_CD": "12345"}
         result = code_funcs.extract_codes_from_service(
             {"SERVICE_TERM": "TEST SERVICE"}, self.constants
         )
-        self.assertEqual(result["PROCEDURE_CD"], "12345")
+        self.assertEqual(result["PROCEDURE_CD"], '["12345"]')
         self.assertEqual(result["CODE_METHODOLOGY"], "Explicit")
 
         # Test unlisted service
@@ -437,11 +437,11 @@ class TestCodeFuncs(unittest.TestCase):
         # Test function
         result_df = code_funcs.code_breakout(test_df, self.constants)
 
-        # Verify results
+        # Verify results - code_breakout normalizes to JSON list format
         self.assertIsInstance(result_df, pd.DataFrame)
         self.assertEqual(len(result_df), 2)
-        self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], "12345")
+        self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], '["12345"]')
-        self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], "67890")
+        self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], '["67890"]')
 
         # Verify mock calls
         mock_fill_claim_type.assert_called_once()