diff --git a/fieldExtraction/src/codes/code_funcs.py b/fieldExtraction/src/codes/code_funcs.py index fffd78f..1860296 100644 --- a/fieldExtraction/src/codes/code_funcs.py +++ b/fieldExtraction/src/codes/code_funcs.py @@ -1,3 +1,5 @@ +import ast +import json import logging import os import re @@ -29,12 +31,18 @@ def clean_service(service, constants: Constants) -> str: # Map common values for acronym, full_form in constants.SYNONYM_MAP.items(): + if string_utils.is_empty(acronym) or string_utils.is_empty(full_form): + continue service = re.sub(rf"\b{re.escape(acronym)}\b", full_form, service) # Remove other values for term in constants.REMOVAL_LIST: + if term is None: + continue service = re.sub(rf"\b{re.escape(term)}\b", "", service).strip() service = re.sub(r"\s+", " ", service) + if service is None: + return "" service = service.replace(" - ", "-") if any([v in service for v in ["UNLISTED", "UNCATEGORIZED", "NON-LISTED"]]): @@ -148,15 +156,8 @@ def code_category(service, proc_category, hcpcs_level2_mapping, filename): code = list(hcpcs_level2_mapping.keys())[ list(hcpcs_level2_mapping.values()).index(answer) ] - code_answer_dict["PROCEDURE_CD"].append(code) - code_answer_dict["PROCEDURE_CD_DESC"].append(answer) - - if len(code_answer_dict["PROCEDURE_CD"]) == 1: - code_answer_dict["PROCEDURE_CD"] = code_answer_dict["PROCEDURE_CD"][0] - code_answer_dict["PROCEDURE_CD_DESC"] = code_answer_dict["PROCEDURE_CD_DESC"][0] - else: - code_answer_dict["PROCEDURE_CD"] = str(code_answer_dict["PROCEDURE_CD"]) - code_answer_dict["PROCEDURE_CD_DESC"] = str(code_answer_dict["PROCEDURE_CD_DESC"]) + code_answer_dict["PROCEDURE_CD"].append(str(code)) + code_answer_dict["PROCEDURE_CD_DESC"].append(str(answer)) return code_answer_dict @@ -181,13 +182,13 @@ def code_implicit_special(service, filename): ) special_case_mapping = { - "Drugs": "J0000-J9999", - "Vaccines": "J0000-J9999|90471‑90474|90620‑90621|90633|90647‑90648|90651|90670|90672|90680‑90681|90686|90696|90698|90700|90707|90710|90713|90714", - "Surgery": "10004-69990", - "PT/OT/ST": "92507-92508|92526|97014|97110|97112|97116|97150|97161-97168|97530|97535", - "PT": "PT Codes TBD", - "OT": "OT Codes TBD", - "ST": "ST Codes TBD", + "Drugs": ["J0000-J9999"], + "Vaccines": ["J0000-J9999", "90471‑90474", "90620‑90621", "90633", "90647‑90648", "90651", "90670", "90672", "90680‑90681", "90686", "90696", "90698", "90700", "90707", "90710", "90713", "90714"], + "Surgery": ["10004-69990"], + "PT/OT/ST": ["92507-92508", "92526", "97014", "97110", "97112", "97116", "97150", "97161-97168", "97530", "97535"], + "PT": ["PT Codes TBD"], + "OT": ["OT Codes TBD"], + "ST": ["ST Codes TBD"], } code_answer_dict = {} if claude_answer_final in special_case_mapping: @@ -343,33 +344,39 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): str(key) for key, val in cpt_mapping.items() if val == description ] if matching_codes: - proc_codes.extend(matching_codes) + # Split pipe-delimited codes into individual codes + for code in matching_codes: + proc_codes.extend(code.split("|")) proc_descs.append(description) if description in hcpcs_mapping.values(): matching_codes = [ str(key) for key, val in hcpcs_mapping.items() if val == description ] if matching_codes: - proc_codes.extend(matching_codes) + # Split pipe-delimited codes into individual codes + for code in matching_codes: + proc_codes.extend(code.split("|")) proc_descs.append(description) if description in rev_mapping.values(): matching_codes = [ str(key) for key, val in rev_mapping.items() if val == description ] if matching_codes: - rev_codes.extend(matching_codes) + # Split pipe-delimited codes into individual codes + for code in matching_codes: + rev_codes.extend(code.split("|")) rev_descs.append(description) # Combine answers if proc_codes: - code_answer_dict["PROCEDURE_CD"] = proc_codes[0] if len(proc_codes) == 1 else str(proc_codes) - code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs[0] if len(proc_descs) == 1 else str(proc_descs) + code_answer_dict["PROCEDURE_CD"] = proc_codes + code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs code_answer_dict["CODE_METHODOLOGY"] = ( f"Implicit - Level {level_dict['level_suffix']}" ) if rev_codes: - code_answer_dict["REVENUE_CD"] = rev_codes[0] if len(rev_codes) == 1 else str(rev_codes) - code_answer_dict["REVENUE_CD_DESC"] = rev_descs[0] if len(rev_descs) == 1 else str(rev_descs) + code_answer_dict["REVENUE_CD"] = rev_codes + code_answer_dict["REVENUE_CD_DESC"] = rev_descs code_answer_dict["CODE_METHODOLOGY"] = ( f"Implicit - Level {level_dict['level_suffix']}" ) @@ -521,6 +528,24 @@ def get_implicit_runs(answer_dict): return run_dict +def normalize_answer_dict_codes(answer_dict): + """ + Normalizes code fields in an answer dictionary to JSON list format. + + Args: + answer_dict (dict): Dictionary containing code fields to normalize. + Returns: + dict: Dictionary with normalized code fields. + """ + fields_to_normalize = ["REVENUE_CD", "PROCEDURE_CD", "CPT4_PROC_CD"] + + for field in fields_to_normalize: + if field in answer_dict: + answer_dict[field] = string_utils.normalize_to_json_list(answer_dict[field]) + + return answer_dict + + def extract_codes_from_service(answer_dict, constants: Constants): """ Extracts codes from the service term in the answer dictionary. @@ -548,8 +573,8 @@ def extract_codes_from_service(answer_dict, constants: Constants): service, bill_type = answer_dict.get("SERVICE_TERM"), answer_dict.get( "BILL_TYPE_CD_DESC" ) - filename = answer_dict.get("FILENAME") - methodology = answer_dict.get("REIMB_TERM") + filename = answer_dict.get("FILENAME", "") + methodology = answer_dict.get("REIMB_TERM", "") if string_utils.is_empty(service): return {} @@ -573,14 +598,14 @@ def extract_codes_from_service(answer_dict, constants: Constants): if service_clean == "UNLISTED": answer_dict["PROCEDURE_CD_DESC"] = "UNLISTED" answer_dict["CODE_METHODOLOGY"] = "UNLISTED" - return answer_dict + return normalize_answer_dict_codes(answer_dict) # Exit point if N/A or in DO_NOT_RUN list if string_utils.is_empty(service_clean) or any( [v in service_clean for v in constants.DO_NOT_RUN] ): answer_dict["CODE_METHODOLOGY"] = "Generic - Before Prompts" - return answer_dict + return normalize_answer_dict_codes(answer_dict) # Explicit Codes (run always) code_answer_dict = code_explicit(service_clean, methodology, filename) @@ -595,7 +620,7 @@ def extract_codes_from_service(answer_dict, constants: Constants): else: code_answer_dict["CODE_METHODOLOGY"] = "Explicit" answer_dict.update(code_answer_dict) - return answer_dict + return normalize_answer_dict_codes(answer_dict) # Implicit Codes: Code Categories if any(["Category:" in v for v in code_answer_dict.get("PROCEDURE_CD", "")]): @@ -610,14 +635,14 @@ def extract_codes_from_service(answer_dict, constants: Constants): ): # if any code value is not empty, return code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Letter Category" answer_dict.update(code_answer_dict) - return answer_dict + return normalize_answer_dict_codes(answer_dict) # Implicit Codes: Special Categories code_answer_dict = code_implicit_special(service_clean, "") if code_answer_dict: code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Special Case" answer_dict.update(code_answer_dict) - return answer_dict + return normalize_answer_dict_codes(answer_dict) # Implicit Codes: Levels 1 and 2 code_answer_dict = code_implicit_rag( @@ -627,7 +652,7 @@ def extract_codes_from_service(answer_dict, constants: Constants): not string_utils.is_empty(value) for value in code_answer_dict.values() ): # if any code value is not empty, return answer_dict.update(code_answer_dict) - return answer_dict + return normalize_answer_dict_codes(answer_dict) # No Match - Why? Generic or Specific last_check_answer = code_last_check(service_clean, "") @@ -636,7 +661,7 @@ def extract_codes_from_service(answer_dict, constants: Constants): elif last_check_answer == "Specific": answer_dict["CODE_METHODOLOGY"] = "Specific - No Match" - return answer_dict + return normalize_answer_dict_codes(answer_dict) def fill_claim_type(answer_dicts): @@ -689,8 +714,14 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants): for answer_dict in answer_dicts_with_code ] - return pd.DataFrame(answer_dicts_with_code) + df = pd.DataFrame(answer_dicts_with_code) + # Normalize code columns to JSON list format + for col in ["PROCEDURE_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]: + if col in df.columns: + df[col] = df[col].apply(string_utils.normalize_to_json_list) + + return df def grouper_breakout(results_with_code: pd.DataFrame): """ @@ -729,4 +760,4 @@ def grouper_breakout(results_with_code: pd.DataFrame): answer_dict.update(grouper_answer) final_answer_dicts.append(answer_dict) - return pd.DataFrame(final_answer_dicts) + return pd.DataFrame(final_answer_dicts) \ No newline at end of file diff --git a/fieldExtraction/src/codes/main.py b/fieldExtraction/src/codes/main.py index 51569ea..19cdcf7 100644 --- a/fieldExtraction/src/codes/main.py +++ b/fieldExtraction/src/codes/main.py @@ -1,7 +1,10 @@ import concurrent.futures +import json +import pandas as pd import src.codes.code_funcs as code_funcs import src.config as config +import src.utils.string_utils as string_utils import src.utils.io_utils as io_utils from constants.constants import Constants from sentence_transformers import SentenceTransformer @@ -9,11 +12,10 @@ from sentence_transformers import SentenceTransformer def main(): # Constants - INPUT_FILE_PATH = "Codes-Test-Doc.xlsx" - OUTPUT_FILE_PATH = "Doczy-Codes-Test-9.csv" + INPUT_FILE_PATH = "" + OUTPUT_FILE_PATH = "" df = io_utils.read_local(INPUT_FILE_PATH) - # Read Constants constants = Constants() model = SentenceTransformer("all-roberta-large-v1") @@ -35,7 +37,7 @@ def main(): } # Process the row - result_dict = code_funcs.extract_codes_from_service(answer_dict) + result_dict = code_funcs.extract_codes_from_service(answer_dict, constants) return idx, result_dict except Exception as e: print(f"Error processing row {idx}: {e}") @@ -66,16 +68,16 @@ def main(): if key not in df.columns: df[key] = None - # Ensure value is scalar - if isinstance(value, (list, dict)): - value = str(value) # Convert lists/dicts to strings - try: - # Update the specific cell df.at[idx, key] = value except Exception as e: print(f"Error updating DataFrame at row {idx}, key '{key}': {e}") + # NORMALIZE ALL CODE COLUMNS + for col in ["CPT4_PROC_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]: + if col in df.columns: + df[col] = df[col].apply(string_utils.normalize_to_json_list) + # Save the updated DataFrame try: if OUTPUT_FILE_PATH: @@ -85,7 +87,6 @@ def main(): print("No output path specified. Results not saved.") except Exception as e: print(f"Error saving results: {e}") - # Try to save to a backup location backup_path = "backup_results.csv" try: df.to_csv(backup_path, index=False) @@ -97,4 +98,4 @@ def main(): if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py b/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py index e168340..6bf54d2 100644 --- a/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py +++ b/fieldExtraction/src/investment/hybrid_smart_chunking_funcs.py @@ -318,7 +318,7 @@ def run_hybrid_smart_chunked_fields( ) # Normalize PROVIDER_STATE - answers_dict = string_utils.normalize_provider_state_field(answers_dict) + answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PROVIDER_STATE") # Normalize PAYER_STATE to standardized two-letter abbreviation for state_field in ["PAYER_STATE"]: @@ -328,7 +328,8 @@ def run_hybrid_smart_chunked_fields( answers_dict[state_field] ) ) - + # Normalize PAYER_STATE + answers_dict = string_utils.normalize_state_field(answers_dict, field_name="PAYER_STATE") return answers_dict except Exception as e: diff --git a/fieldExtraction/src/investment/one_to_one_funcs.py b/fieldExtraction/src/investment/one_to_one_funcs.py index 753c8fe..dd9300b 100644 --- a/fieldExtraction/src/investment/one_to_one_funcs.py +++ b/fieldExtraction/src/investment/one_to_one_funcs.py @@ -258,11 +258,13 @@ def run_full_context_fields( full_context_answers_dict["PAYER_STATE"] = string_utils.normalize_state_to_abbreviation( full_context_answers_dict["PAYER_STATE"] ) + full_context_answers_dict = string_utils.normalize_state_field( + full_context_answers_dict, field_name="PAYER_STATE") # normalize PROVIDER_STATE to two-letter abbreviation if "PROVIDER_STATE" in full_context_answers_dict: - full_context_answers_dict = string_utils.normalize_provider_state_field( - full_context_answers_dict + full_context_answers_dict = string_utils.normalize_state_field( + full_context_answers_dict, field_name="PROVIDER_STATE" ) # Run Special Case Breakout on any breakout terms full_context_answers_dict = one_to_one_breakout(full_context_answers_dict, filename) diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index 755ecd8..d091e31 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -1,4 +1,5 @@ import json +import ast import logging import re import traceback @@ -631,7 +632,7 @@ def parse_state_field_to_list(raw_value): logging.error(f"Could not convert value to string: {raw_value!r}") return [] -def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"): +def normalize_state_field(answers_dict: dict, field_name: str = "PROVIDER_STATE"): if field_name not in answers_dict or not answers_dict[field_name]: answers_dict[field_name] = [] return answers_dict @@ -639,7 +640,7 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID raw_value = answers_dict[field_name] # ---- Step 1: Parse into list ---- - state_list = parse_state_field_to_list(raw_value) + state_list = parse_raw_state_value(raw_value) # ---- Step 2: Normalize each state ---- normalized_states = [] @@ -652,3 +653,100 @@ def normalize_provider_state_field(answers_dict: dict, field_name: str = "PROVID answers_dict[field_name] = normalized_states return answers_dict + + +def parse_raw_state_value(raw_value): + """ + Parses various input formats into a flat list of state strings. + Handles: strings, lists, nested lists, and various delimiters. + """ + if not raw_value: + return [] + + # If it's a string that looks like a list, try to parse it + if isinstance(raw_value, str): + stripped = raw_value.strip() + + # Try to parse as Python literal (handles "[['NC', 'WA'], 'CA|UT']") + if stripped.startswith('['): + try: + parsed = ast.literal_eval(stripped) + return parse_raw_state_value(parsed) # Recursive call with parsed list + except (ValueError, SyntaxError): + pass # Not valid Python literal, continue with string parsing + + # Regular string splitting on delimiters + states = re.split(r'[|,;\s]+', stripped) + return [s.strip() for s in states if s.strip()] + + if isinstance(raw_value, list): + result = [] + for item in raw_value: + result.extend(parse_raw_state_value(item)) # Recursive call + return result + + return [str(raw_value)] + + +def normalize_to_json_list(val): + """ + Normalizes a value to a JSON list format. + Handles: pipe-delimited strings, string representations of Python lists, + actual lists with pipe-delimited items, and single values. + """ + # Handle None and empty values first + if is_empty(val): + return val + + # Handle pandas NA/NaN - check for scalar first + try: + if pd.isna(val): + return val + except (ValueError, TypeError): + # pd.isna fails on arrays/lists, which is fine - we'll handle them below + pass + + if isinstance(val, str): + # Check if it's a string representation of a Python list (e.g., "['item1', 'item2']") + if val.startswith("[") and val.endswith("]"): + try: + # Try JSON parse first + parsed = json.loads(val) + except json.JSONDecodeError: + # If JSON fails, try ast.literal_eval for Python list syntax + try: + parsed = ast.literal_eval(val) + except (ValueError, SyntaxError): + return val + + if isinstance(parsed, list): + expanded = [] + for item in parsed: + if isinstance(item, str) and "|" in item: + expanded.extend(item.split("|")) + else: + expanded.append(item) + result = json.dumps(expanded) + return result + return val + + # Pipe-delimited format - convert to JSON list + if "|" in val: + result = json.dumps(val.split("|")) + return result + + # Single value - convert to JSON list + result = json.dumps([val]) + return result + + if isinstance(val, list): + # Check if list contains pipe-delimited strings and expand them + expanded = [] + for item in val: + if isinstance(item, str) and "|" in item: + expanded.extend(item.split("|")) + else: + expanded.append(item) + result = json.dumps(expanded) + return result + return val \ No newline at end of file diff --git a/fieldExtraction/tests/test_code_funcs.py b/fieldExtraction/tests/test_code_funcs.py index 342f1e1..343b7d6 100644 --- a/fieldExtraction/tests/test_code_funcs.py +++ b/fieldExtraction/tests/test_code_funcs.py @@ -108,9 +108,9 @@ class TestCodeFuncs(unittest.TestCase): "J CODES", proc_category, hcpcs_mapping, "test.pdf" ) - # Verify the results - self.assertEqual(result["PROCEDURE_CD"], "['J0001', 'J0002']") - self.assertEqual(result["PROCEDURE_CD_DESC"], "['Drug A', 'Drug B']") + # Verify the results - code_category returns lists (JSON conversion happens in extract_codes_from_service) + self.assertEqual(result["PROCEDURE_CD"], ["J0001", "J0002"]) + self.assertEqual(result["PROCEDURE_CD_DESC"], ["Drug A", "Drug B"]) self.assertIn("J0001", result["PROCEDURE_CD"]) self.assertIn("J0002", result["PROCEDURE_CD"]) self.assertIn("Drug A", result["PROCEDURE_CD_DESC"]) @@ -124,9 +124,9 @@ class TestCodeFuncs(unittest.TestCase): mock_invoke_claude.return_value = "mock_response" mock_extract.return_value = "Drugs" - # Test drug category + # Test drug category - returns a list result = code_funcs.code_implicit_special("DRUG SERVICE", "test.pdf") - self.assertEqual(result["PROCEDURE_CD"], "J0000-J9999") + self.assertEqual(result["PROCEDURE_CD"], ["J0000-J9999"]) self.assertEqual(result["PROCEDURE_CD_DESC"], "Drugs") # Test no match @@ -221,9 +221,9 @@ class TestCodeFuncs(unittest.TestCase): "TEST SERVICE", implicit_run_dict, "test.pdf", self.constants ) - # Verify results - self.assertEqual(result["PROCEDURE_CD"], "12345") - self.assertEqual(result["PROCEDURE_CD_DESC"], "Test Procedure") + # Verify results - code_implicit_rag returns lists + self.assertEqual(result["PROCEDURE_CD"], ["12345"]) + self.assertEqual(result["PROCEDURE_CD_DESC"], ["Test Procedure"]) self.assertEqual(result["CODE_METHODOLOGY"], "Implicit - Level 1") # Test exception handling @@ -333,12 +333,12 @@ class TestCodeFuncs(unittest.TestCase): } mock_implicit_runs.return_value = {"PROCEDURE_CD": True, "REVENUE_CD": True} - # Test explicit code match + # Test explicit code match - extract_codes_from_service normalizes to JSON list format mock_explicit.return_value = {"PROCEDURE_CD": "12345"} result = code_funcs.extract_codes_from_service( {"SERVICE_TERM": "TEST SERVICE"}, self.constants ) - self.assertEqual(result["PROCEDURE_CD"], "12345") + self.assertEqual(result["PROCEDURE_CD"], '["12345"]') self.assertEqual(result["CODE_METHODOLOGY"], "Explicit") # Test unlisted service @@ -437,11 +437,11 @@ class TestCodeFuncs(unittest.TestCase): # Test function result_df = code_funcs.code_breakout(test_df, self.constants) - # Verify results + # Verify results - code_breakout normalizes to JSON list format self.assertIsInstance(result_df, pd.DataFrame) self.assertEqual(len(result_df), 2) - self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], "12345") - self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], "67890") + self.assertEqual(result_df.iloc[0]["PROCEDURE_CD"], '["12345"]') + self.assertEqual(result_df.iloc[1]["PROCEDURE_CD"], '["67890"]') # Verify mock calls mock_fill_claim_type.assert_called_once()