Merged in bugfix/cpt-codes (pull request #812)

Bugfix/cpt codes * formatting fix for revenue_cd * format fix for procedure_codes * bugfix for revenue_codes * working version from code main * code formatting from main and code main * removed dubug prints * removed local file paths * standardize state fields * updated test scenerios * minor fix * Merge branch 'main' into bugfix/cpt-codes merging main into cpt code fix * checks for all fields * reused is_empty functionality * moved normalize_to_json_list to utils * Merge branch 'main' into bugfix/cpt-codes merging upto date main * minor touches * pytest fix Approved-by: Katon Minhas
2025-12-17 19:05:09 +00:00
parent f31c02517a
commit ca53ff8395
6 changed files with 197 additions and 64 deletions
@@ -1,3 +1,5 @@
+import ast
+import json
 import logging
 import os
 import re
@@ -29,12 +31,18 @@ def clean_service(service, constants: Constants) -> str:

    # Map common values
    for acronym, full_form in constants.SYNONYM_MAP.items():
+        if string_utils.is_empty(acronym) or string_utils.is_empty(full_form):
+            continue
        service = re.sub(rf"\b{re.escape(acronym)}\b", full_form, service)

    # Remove other values
    for term in constants.REMOVAL_LIST:
+        if term is None:
+            continue
        service = re.sub(rf"\b{re.escape(term)}\b", "", service).strip()
        service = re.sub(r"\s+", " ", service)
+    if service is None:
+        return ""
    service = service.replace(" - ", "-")

    if any([v in service for v in ["UNLISTED", "UNCATEGORIZED", "NON-LISTED"]]):
@@ -148,15 +156,8 @@ def code_category(service, proc_category, hcpcs_level2_mapping, filename):
            code = list(hcpcs_level2_mapping.keys())[
                list(hcpcs_level2_mapping.values()).index(answer)
            ]
-            code_answer_dict["PROCEDURE_CD"].append(code)
-            code_answer_dict["PROCEDURE_CD_DESC"].append(answer)
-
-    if len(code_answer_dict["PROCEDURE_CD"]) == 1:
-        code_answer_dict["PROCEDURE_CD"] = code_answer_dict["PROCEDURE_CD"][0]
-        code_answer_dict["PROCEDURE_CD_DESC"] = code_answer_dict["PROCEDURE_CD_DESC"][0]
-    else:
-        code_answer_dict["PROCEDURE_CD"] = str(code_answer_dict["PROCEDURE_CD"])
-        code_answer_dict["PROCEDURE_CD_DESC"] = str(code_answer_dict["PROCEDURE_CD_DESC"])
+            code_answer_dict["PROCEDURE_CD"].append(str(code))
+            code_answer_dict["PROCEDURE_CD_DESC"].append(str(answer))
    
    return code_answer_dict

@@ -181,13 +182,13 @@ def code_implicit_special(service, filename):
    )

    special_case_mapping = {
-        "Drugs": "J0000-J9999",
-        "Vaccines": "J0000-J9999|90471‑90474|90620‑90621|90633|90647‑90648|90651|90670|90672|90680‑90681|90686|90696|90698|90700|90707|90710|90713|90714",
-        "Surgery": "10004-69990",
-        "PT/OT/ST": "92507-92508|92526|97014|97110|97112|97116|97150|97161-97168|97530|97535",
-        "PT": "PT Codes TBD",
-        "OT": "OT Codes TBD",
-        "ST": "ST Codes TBD",
+        "Drugs": ["J0000-J9999"],
+        "Vaccines": ["J0000-J9999", "90471‑90474", "90620‑90621", "90633", "90647‑90648", "90651", "90670", "90672", "90680‑90681", "90686", "90696", "90698", "90700", "90707", "90710", "90713", "90714"],
+        "Surgery": ["10004-69990"],
+        "PT/OT/ST": ["92507-92508", "92526", "97014", "97110", "97112", "97116", "97150", "97161-97168", "97530", "97535"],
+        "PT": ["PT Codes TBD"],
+        "OT": ["OT Codes TBD"],
+        "ST": ["ST Codes TBD"],
    }
    code_answer_dict = {}
    if claude_answer_final in special_case_mapping:
@@ -343,33 +344,39 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants):
                    str(key) for key, val in cpt_mapping.items() if val == description
                ]
                if matching_codes:
-                    proc_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        proc_codes.extend(code.split("|"))
                    proc_descs.append(description)
            if description in hcpcs_mapping.values():
                matching_codes = [
                    str(key) for key, val in hcpcs_mapping.items() if val == description
                ]
                if matching_codes:
-                    proc_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        proc_codes.extend(code.split("|"))
                    proc_descs.append(description)
            if description in rev_mapping.values():
                matching_codes = [
                    str(key) for key, val in rev_mapping.items() if val == description
                ]
                if matching_codes:
-                    rev_codes.extend(matching_codes)
+                    # Split pipe-delimited codes into individual codes
+                    for code in matching_codes:
+                        rev_codes.extend(code.split("|"))
                    rev_descs.append(description)

        # Combine answers
        if proc_codes:
-            code_answer_dict["PROCEDURE_CD"] = proc_codes[0] if len(proc_codes) == 1 else str(proc_codes)
-            code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs[0] if len(proc_descs) == 1 else str(proc_descs)
+            code_answer_dict["PROCEDURE_CD"] = proc_codes
+            code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs
            code_answer_dict["CODE_METHODOLOGY"] = (
                f"Implicit - Level {level_dict['level_suffix']}"
            )
        if rev_codes:
-            code_answer_dict["REVENUE_CD"] = rev_codes[0] if len(rev_codes) == 1 else str(rev_codes)
-            code_answer_dict["REVENUE_CD_DESC"] = rev_descs[0] if len(rev_descs) == 1 else str(rev_descs)
+            code_answer_dict["REVENUE_CD"] = rev_codes
+            code_answer_dict["REVENUE_CD_DESC"] = rev_descs
            code_answer_dict["CODE_METHODOLOGY"] = (
                f"Implicit - Level {level_dict['level_suffix']}"
            )
@@ -521,6 +528,24 @@ def get_implicit_runs(answer_dict):
    return run_dict


+def normalize_answer_dict_codes(answer_dict):
+    """
+    Normalizes code fields in an answer dictionary to JSON list format.
+    
+    Args:
+        answer_dict (dict): Dictionary containing code fields to normalize.
+    Returns:
+        dict: Dictionary with normalized code fields.
+    """
+    fields_to_normalize = ["REVENUE_CD", "PROCEDURE_CD", "CPT4_PROC_CD"]
+    
+    for field in fields_to_normalize:
+        if field in answer_dict:
+            answer_dict[field] = string_utils.normalize_to_json_list(answer_dict[field])
+    
+    return answer_dict
+
+
 def extract_codes_from_service(answer_dict, constants: Constants):
    """
    Extracts codes from the service term in the answer dictionary.
@@ -548,8 +573,8 @@ def extract_codes_from_service(answer_dict, constants: Constants):
    service, bill_type = answer_dict.get("SERVICE_TERM"), answer_dict.get(
        "BILL_TYPE_CD_DESC"
    )
-    filename = answer_dict.get("FILENAME")
-    methodology = answer_dict.get("REIMB_TERM")
+    filename = answer_dict.get("FILENAME", "")
+    methodology = answer_dict.get("REIMB_TERM", "")

    if string_utils.is_empty(service):
        return {}
@@ -573,14 +598,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
    if service_clean == "UNLISTED":
        answer_dict["PROCEDURE_CD_DESC"] = "UNLISTED"
        answer_dict["CODE_METHODOLOGY"] = "UNLISTED"
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)

    # Exit point if N/A or in DO_NOT_RUN list
    if string_utils.is_empty(service_clean) or any(
        [v in service_clean for v in constants.DO_NOT_RUN]
    ):
        answer_dict["CODE_METHODOLOGY"] = "Generic - Before Prompts"
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)

    # Explicit Codes (run always)
    code_answer_dict = code_explicit(service_clean, methodology, filename)
@@ -595,7 +620,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
        else:
            code_answer_dict["CODE_METHODOLOGY"] = "Explicit"
        answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)

    # Implicit Codes: Code Categories
    if any(["Category:" in v for v in code_answer_dict.get("PROCEDURE_CD", "")]):
@@ -610,14 +635,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
        ):  # if any code value is not empty, return
            code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Letter Category"
            answer_dict.update(code_answer_dict)
-            return answer_dict
+            return normalize_answer_dict_codes(answer_dict)

    # Implicit Codes: Special Categories
    code_answer_dict = code_implicit_special(service_clean, "")
    if code_answer_dict:
        code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Special Case"
        answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)

    # Implicit Codes: Levels 1 and 2
    code_answer_dict = code_implicit_rag(
@@ -627,7 +652,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
        not string_utils.is_empty(value) for value in code_answer_dict.values()
    ):  # if any code value is not empty, return
        answer_dict.update(code_answer_dict)
-        return answer_dict
+        return normalize_answer_dict_codes(answer_dict)

    # No Match - Why? Generic or Specific
    last_check_answer = code_last_check(service_clean, "")
@@ -636,7 +661,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
    elif last_check_answer == "Specific":
        answer_dict["CODE_METHODOLOGY"] = "Specific - No Match"

-    return answer_dict
+    return normalize_answer_dict_codes(answer_dict)


 def fill_claim_type(answer_dicts):
@@ -689,8 +714,14 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
        for answer_dict in answer_dicts_with_code
    ]

-    return pd.DataFrame(answer_dicts_with_code)
+    df = pd.DataFrame(answer_dicts_with_code)

+    # Normalize code columns to JSON list format
+    for col in ["PROCEDURE_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]:
+        if col in df.columns:
+            df[col] = df[col].apply(string_utils.normalize_to_json_list)
+
+    return df

 def grouper_breakout(results_with_code: pd.DataFrame):
    """
@@ -729,4 +760,4 @@ def grouper_breakout(results_with_code: pd.DataFrame):
            answer_dict.update(grouper_answer)
        final_answer_dicts.append(answer_dict)

-    return pd.DataFrame(final_answer_dicts)
+    return pd.DataFrame(final_answer_dicts)
@@ -1,7 +1,10 @@
 import concurrent.futures
+import json

+import pandas as pd
 import src.codes.code_funcs as code_funcs
 import src.config as config
+import src.utils.string_utils as string_utils
 import src.utils.io_utils as io_utils
 from constants.constants import Constants
 from sentence_transformers import SentenceTransformer
@@ -9,11 +12,10 @@ from sentence_transformers import SentenceTransformer

 def main():
    # Constants
-    INPUT_FILE_PATH = "Codes-Test-Doc.xlsx"
-    OUTPUT_FILE_PATH = "Doczy-Codes-Test-9.csv"
+    INPUT_FILE_PATH = ""
+    OUTPUT_FILE_PATH = ""

    df = io_utils.read_local(INPUT_FILE_PATH)
-
    # Read Constants
    constants = Constants()
    model = SentenceTransformer("all-roberta-large-v1")
@@ -35,7 +37,7 @@ def main():
            }

            # Process the row
-            result_dict = code_funcs.extract_codes_from_service(answer_dict)
+            result_dict = code_funcs.extract_codes_from_service(answer_dict, constants)
            return idx, result_dict
        except Exception as e:
            print(f"Error processing row {idx}: {e}")
@@ -66,16 +68,16 @@ def main():
            if key not in df.columns:
                df[key] = None

-            # Ensure value is scalar
-            if isinstance(value, (list, dict)):
-                value = str(value)  # Convert lists/dicts to strings
-
            try:
-                # Update the specific cell
                df.at[idx, key] = value
            except Exception as e:
                print(f"Error updating DataFrame at row {idx}, key '{key}': {e}")

+    # NORMALIZE ALL CODE COLUMNS
+    for col in ["CPT4_PROC_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]:
+        if col in df.columns:
+            df[col] = df[col].apply(string_utils.normalize_to_json_list)
+
    # Save the updated DataFrame
    try:
        if OUTPUT_FILE_PATH:
@@ -85,7 +87,6 @@ def main():
            print("No output path specified. Results not saved.")
    except Exception as e:
        print(f"Error saving results: {e}")
-        # Try to save to a backup location
        backup_path = "backup_results.csv"
        try:
            df.to_csv(backup_path, index=False)
@@ -97,4 +98,4 @@ def main():


 if __name__ == "__main__":
-    main()
+    main()
@@ -318,7 +318,7 @@ def run_hybrid_smart_chunked_fields(
            )

        # Normalize PROVIDER_STATE  
-        answers_dict = string_utils.normalize_provider_state_field(answers_dict)
+        answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PROVIDER_STATE")

        # Normalize PAYER_STATE to standardized two-letter abbreviation
        for state_field in ["PAYER_STATE"]:
@@ -328,7 +328,8 @@ def run_hybrid_smart_chunked_fields(
                        answers_dict[state_field]
                    )
                )
-
+        # Normalize PAYER_STATE  
+        answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PAYER_STATE")
        return answers_dict

    except Exception as e:
@@ -258,11 +258,13 @@ def run_full_context_fields(
        full_context_answers_dict["PAYER_STATE"] = string_utils.normalize_state_to_abbreviation(
            full_context_answers_dict["PAYER_STATE"]
        )
+    full_context_answers_dict = string_utils.normalize_state_field(
+        full_context_answers_dict, field_name="PAYER_STATE")

    # normalize PROVIDER_STATE to two-letter abbreviation
    if "PROVIDER_STATE" in full_context_answers_dict:
-        full_context_answers_dict = string_utils.normalize_provider_state_field(
-            full_context_answers_dict
+        full_context_answers_dict = string_utils.normalize_state_field(
+            full_context_answers_dict, field_name="PROVIDER_STATE"
        )
    # Run Special Case Breakout on any breakout terms
    full_context_answers_dict = one_to_one_breakout(full_context_answers_dict, filename)
@@ -1,4 +1,5 @@
 import json
+import ast
 import logging
 import re
 import traceback
@@ -631,7 +632,7 @@ def parse_state_field_to_list(raw_value):
        logging.error(f"Could not convert value to string: {raw_value!r}")
        return []

-def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
+def normalize_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"):
    if field_name not in answers_dict or not answers_dict[field_name]:
        answers_dict[field_name] = []
        return answers_dict
@@ -639,7 +640,7 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID
    raw_value = answers_dict[field_name]

    # ---- Step 1: Parse into list ----
-    state_list = parse_state_field_to_list(raw_value)
+    state_list = parse_raw_state_value(raw_value)

    # ---- Step 2: Normalize each state ----
    normalized_states = []
@@ -652,3 +653,100 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID

    answers_dict[field_name] = normalized_states
    return answers_dict
+
+
+def parse_raw_state_value(raw_value):
+    """
+    Parses various input formats into a flat list of state strings.
+    Handles: strings, lists, nested lists, and various delimiters.
+    """
+    if not raw_value:
+        return []
+    
+    # If it's a string that looks like a list, try to parse it
+    if isinstance(raw_value, str):
+        stripped = raw_value.strip()
+        
+        # Try to parse as Python literal (handles "[['NC', 'WA'], 'CA|UT']")
+        if stripped.startswith('['):
+            try:
+                parsed = ast.literal_eval(stripped)
+                return parse_raw_state_value(parsed)  # Recursive call with parsed list
+            except (ValueError, SyntaxError):
+                pass  # Not valid Python literal, continue with string parsing
+        
+        # Regular string splitting on delimiters
+        states = re.split(r'[|,;\s]+', stripped)
+        return [s.strip() for s in states if s.strip()]
+    
+    if isinstance(raw_value, list):
+        result = []
+        for item in raw_value:
+            result.extend(parse_raw_state_value(item))  # Recursive call
+        return result
+    
+    return [str(raw_value)]
+
+
+def normalize_to_json_list(val):
+    """
+    Normalizes a value to a JSON list format.
+    Handles: pipe-delimited strings, string representations of Python lists,
+    actual lists with pipe-delimited items, and single values.
+    """    
+    # Handle None and empty values first
+    if is_empty(val):
+        return val
+    
+    # Handle pandas NA/NaN - check for scalar first
+    try:
+        if pd.isna(val):
+            return val
+    except (ValueError, TypeError):
+        # pd.isna fails on arrays/lists, which is fine - we'll handle them below
+        pass
+    
+    if isinstance(val, str):
+        # Check if it's a string representation of a Python list (e.g., "['item1', 'item2']")
+        if val.startswith("[") and val.endswith("]"):
+            try:
+                # Try JSON parse first
+                parsed = json.loads(val)
+            except json.JSONDecodeError:
+                # If JSON fails, try ast.literal_eval for Python list syntax
+                try:
+                    parsed = ast.literal_eval(val)
+                except (ValueError, SyntaxError):
+                    return val
+            
+            if isinstance(parsed, list):
+                expanded = []
+                for item in parsed:
+                    if isinstance(item, str) and "|" in item:
+                        expanded.extend(item.split("|"))
+                    else:
+                        expanded.append(item)
+                result = json.dumps(expanded)
+                return result
+            return val
+        
+        # Pipe-delimited format - convert to JSON list
+        if "|" in val:
+            result = json.dumps(val.split("|"))
+            return result
+        
+        # Single value - convert to JSON list
+        result = json.dumps([val])
+        return result
+    
+    if isinstance(val, list):
+        # Check if list contains pipe-delimited strings and expand them
+        expanded = []
+        for item in val:
+            if isinstance(item, str) and "|" in item:
+                expanded.extend(item.split("|"))
+            else:
+                expanded.append(item)
+        result = json.dumps(expanded)
+        return result
+    return val
@@ -108,9 +108,9 @@ class TestCodeFuncs(unittest.TestCase):
            "J CODES", proc_category, hcpcs_mapping, "test.pdf"
        )

-        # Verify the results
-        self.assertEqual(result["PROCEDURE_CD"], "['J0001', 'J0002']")
-        self.assertEqual(result["PROCEDURE_CD_DESC"], "['Drug A', 'Drug B']")
+        # Verify the results - code_category returns lists (JSON conversion happens in extract_codes_from_service)
+        self.assertEqual(result["PROCEDURE_CD"], ["J0001", "J0002"])
+        self.assertEqual(result["PROCEDURE_CD_DESC"], ["Drug A", "Drug B"])
        self.assertIn("J0001", result["PROCEDURE_CD"])
        self.assertIn("J0002", result["PROCEDURE_CD"])
        self.assertIn("Drug A", result["PROCEDURE_CD_DESC"])
@@ -124,9 +124,9 @@ class TestCodeFuncs(unittest.TestCase):
        mock_invoke_claude.return_value = "mock_response"
        mock_extract.return_value = "Drugs"

-        # Test drug category
+        # Test drug category - returns a list
        result = code_funcs.code_implicit_special("DRUG SERVICE", "test.pdf")
-        self.assertEqual(result["PROCEDURE_CD"], "J0000-J9999")
+        self.assertEqual(result["PROCEDURE_CD"], ["J0000-J9999"])
        self.assertEqual(result["PROCEDURE_CD_DESC"], "Drugs")

        # Test no match
@@ -221,9 +221,9 @@ class TestCodeFuncs(unittest.TestCase):
                "TEST SERVICE", implicit_run_dict, "test.pdf", self.constants
            )

-            # Verify results
-            self.assertEqual(result["PROCEDURE_CD"], "12345")
-            self.assertEqual(result["PROCEDURE_CD_DESC"], "Test Procedure")
+            # Verify results - code_implicit_rag returns lists
+            self.assertEqual(result["PROCEDURE_CD"], ["12345"])
+            self.assertEqual(result["PROCEDURE_CD_DESC"], ["Test Procedure"])
            self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Level 1")

        # Test exception handling
@@ -333,12 +333,12 @@ class TestCodeFuncs(unittest.TestCase):
        }
        mock_implicit_runs.return_value = {"PROCEDURE_CD": True, "REVENUE_CD": True}

-        # Test explicit code match
+        # Test explicit code match - extract_codes_from_service normalizes to JSON list format
        mock_explicit.return_value = {"PROCEDURE_CD": "12345"}
        result = code_funcs.extract_codes_from_service(
            {"SERVICE_TERM": "TEST SERVICE"}, self.constants
        )
-        self.assertEqual(result["PROCEDURE_CD"], "12345")
+        self.assertEqual(result["PROCEDURE_CD"], '["12345"]')
        self.assertEqual(result["CODE_METHODOLOGY"], "Explicit")

        # Test unlisted service
@@ -437,11 +437,11 @@ class TestCodeFuncs(unittest.TestCase):
        # Test function
        result_df = code_funcs.code_breakout(test_df, self.constants)

-        # Verify results
+        # Verify results - code_breakout normalizes to JSON list format
        self.assertIsInstance(result_df, pd.DataFrame)
        self.assertEqual(len(result_df), 2)
-        self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], "12345")
-        self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], "67890")
+        self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], '["12345"]')
+        self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], '["67890"]')

        # Verify mock calls
        mock_fill_claim_type.assert_called_once()