jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/cpt-codes (pull request #812)

Browse Source 
Bugfix/cpt codes

* formatting fix for revenue_cd

* format fix for procedure_codes

* bugfix for revenue_codes

* working version from code main

* code formatting from main and code main

* removed dubug prints

* removed local file paths

* standardize state fields

* updated test scenerios

* minor fix

* Merge branch 'main' into bugfix/cpt-codes
merging main into cpt code fix

* checks for all fields

* reused is_empty functionality

* moved normalize_to_json_list to utils

* Merge branch 'main' into bugfix/cpt-codes
merging upto date main

* minor touches

* pytest fix


Approved-by: Katon Minhas
This commit is contained in:
Karan Desai
2025-12-17 19:05:09 +00:00
committed by Katon Minhas
parent f31c02517a
commit ca53ff8395
6 changed files with 197 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/codes/code_funcs.py
+65 -34
View File
@@ -1,3 +1,5 @@
import ast
import json
import logging
import os
import re
@@ -29,12 +31,18 @@ def clean_service(service, constants: Constants) -> str:
# Map common values
for acronym, full_form in constants.SYNONYM_MAP.items():
if string_utils.is_empty(acronym) or string_utils.is_empty(full_form):
continue
service = re.sub(rf"\b{re.escape(acronym)}\b", full_form, service)
# Remove other values
for term in constants.REMOVAL_LIST:
if term is None:
continue
service = re.sub(rf"\b{re.escape(term)}\b", "", service).strip()
service = re.sub(r"\s+", " ", service)
if service is None:
return ""
service = service.replace(" - ", "-")
if any([v in service for v in ["UNLISTED", "UNCATEGORIZED", "NON-LISTED"]]):
@@ -148,15 +156,8 @@ def code_category(service, proc_category, hcpcs_level2_mapping, filename):
code = list(hcpcs_level2_mapping.keys())[
list(hcpcs_level2_mapping.values()).index(answer)
]
code_answer_dict["PROCEDURE_CD"].append(code)
code_answer_dict["PROCEDURE_CD_DESC"].append(answer)
if len(code_answer_dict["PROCEDURE_CD"]) == 1:
code_answer_dict["PROCEDURE_CD"] = code_answer_dict["PROCEDURE_CD"][0]
code_answer_dict["PROCEDURE_CD_DESC"] = code_answer_dict["PROCEDURE_CD_DESC"][0]
else:
code_answer_dict["PROCEDURE_CD"] = str(code_answer_dict["PROCEDURE_CD"])
code_answer_dict["PROCEDURE_CD_DESC"] = str(code_answer_dict["PROCEDURE_CD_DESC"])
code_answer_dict["PROCEDURE_CD"].append(str(code))
code_answer_dict["PROCEDURE_CD_DESC"].append(str(answer))
return code_answer_dict
@@ -181,13 +182,13 @@ def code_implicit_special(service, filename):
)
special_case_mapping = {
"Drugs": "J0000-J9999",
"Vaccines": "J0000-J9999|90471‑90474|90620‑90621|90633|90647‑90648|90651|90670|90672|90680‑90681|90686|90696|90698|90700|90707|90710|90713|90714",
"Surgery": "10004-69990",
"PT/OT/ST": "92507-92508|92526|97014|97110|97112|97116|97150|97161-97168|97530|97535",
"PT": "PT Codes TBD",
"OT": "OT Codes TBD",
"ST": "ST Codes TBD",
"Drugs": ["J0000-J9999"],
"Vaccines": ["J0000-J9999", "90471‑90474", "90620‑90621", "90633", "90647‑90648", "90651", "90670", "90672", "90680‑90681", "90686", "90696", "90698", "90700", "90707", "90710", "90713", "90714"],
"Surgery": ["10004-69990"],
"PT/OT/ST": ["92507-92508", "92526", "97014", "97110", "97112", "97116", "97150", "97161-97168", "97530", "97535"],
"PT": ["PT Codes TBD"],
"OT": ["OT Codes TBD"],
"ST": ["ST Codes TBD"],
}
code_answer_dict = {}
if claude_answer_final in special_case_mapping:
@@ -343,33 +344,39 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants):
str(key) for key, val in cpt_mapping.items() if val == description
]
if matching_codes:
proc_codes.extend(matching_codes)
# Split pipe-delimited codes into individual codes
for code in matching_codes:
proc_codes.extend(code.split("|"))
proc_descs.append(description)
if description in hcpcs_mapping.values():
matching_codes = [
str(key) for key, val in hcpcs_mapping.items() if val == description
]
if matching_codes:
proc_codes.extend(matching_codes)
# Split pipe-delimited codes into individual codes
for code in matching_codes:
proc_codes.extend(code.split("|"))
proc_descs.append(description)
if description in rev_mapping.values():
matching_codes = [
str(key) for key, val in rev_mapping.items() if val == description
]
if matching_codes:
rev_codes.extend(matching_codes)
# Split pipe-delimited codes into individual codes
for code in matching_codes:
rev_codes.extend(code.split("|"))
rev_descs.append(description)
# Combine answers
if proc_codes:
code_answer_dict["PROCEDURE_CD"] = proc_codes[0] if len(proc_codes) == 1 else str(proc_codes)
code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs[0] if len(proc_descs) == 1 else str(proc_descs)
code_answer_dict["PROCEDURE_CD"] = proc_codes
code_answer_dict["PROCEDURE_CD_DESC"] = proc_descs
code_answer_dict["CODE_METHODOLOGY"] = (
f"Implicit - Level {level_dict['level_suffix']}"
)
if rev_codes:
code_answer_dict["REVENUE_CD"] = rev_codes[0] if len(rev_codes) == 1 else str(rev_codes)
code_answer_dict["REVENUE_CD_DESC"] = rev_descs[0] if len(rev_descs) == 1 else str(rev_descs)
code_answer_dict["REVENUE_CD"] = rev_codes
code_answer_dict["REVENUE_CD_DESC"] = rev_descs
code_answer_dict["CODE_METHODOLOGY"] = (
f"Implicit - Level {level_dict['level_suffix']}"
)
@@ -521,6 +528,24 @@ def get_implicit_runs(answer_dict):
return run_dict
def normalize_answer_dict_codes(answer_dict):
"""
Normalizes code fields in an answer dictionary to JSON list format.
Args:
answer_dict (dict): Dictionary containing code fields to normalize.
Returns:
dict: Dictionary with normalized code fields.
"""
fields_to_normalize = ["REVENUE_CD", "PROCEDURE_CD", "CPT4_PROC_CD"]
for field in fields_to_normalize:
if field in answer_dict:
answer_dict[field] = string_utils.normalize_to_json_list(answer_dict[field])
return answer_dict
def extract_codes_from_service(answer_dict, constants: Constants):
"""
Extracts codes from the service term in the answer dictionary.
@@ -548,8 +573,8 @@ def extract_codes_from_service(answer_dict, constants: Constants):
service, bill_type = answer_dict.get("SERVICE_TERM"), answer_dict.get(
"BILL_TYPE_CD_DESC"
)
filename = answer_dict.get("FILENAME")
methodology = answer_dict.get("REIMB_TERM")
filename = answer_dict.get("FILENAME", "")
methodology = answer_dict.get("REIMB_TERM", "")
if string_utils.is_empty(service):
return {}
@@ -573,14 +598,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
if service_clean == "UNLISTED":
answer_dict["PROCEDURE_CD_DESC"] = "UNLISTED"
answer_dict["CODE_METHODOLOGY"] = "UNLISTED"
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# Exit point if N/A or in DO_NOT_RUN list
if string_utils.is_empty(service_clean) or any(
[v in service_clean for v in constants.DO_NOT_RUN]
):
answer_dict["CODE_METHODOLOGY"] = "Generic - Before Prompts"
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# Explicit Codes (run always)
code_answer_dict = code_explicit(service_clean, methodology, filename)
@@ -595,7 +620,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
else:
code_answer_dict["CODE_METHODOLOGY"] = "Explicit"
answer_dict.update(code_answer_dict)
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# Implicit Codes: Code Categories
if any(["Category:" in v for v in code_answer_dict.get("PROCEDURE_CD", "")]):
@@ -610,14 +635,14 @@ def extract_codes_from_service(answer_dict, constants: Constants):
): # if any code value is not empty, return
code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Letter Category"
answer_dict.update(code_answer_dict)
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# Implicit Codes: Special Categories
code_answer_dict = code_implicit_special(service_clean, "")
if code_answer_dict:
code_answer_dict["CODE_METHODOLOGY"] = "Implicit - Special Case"
answer_dict.update(code_answer_dict)
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# Implicit Codes: Levels 1 and 2
code_answer_dict = code_implicit_rag(
@@ -627,7 +652,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
not string_utils.is_empty(value) for value in code_answer_dict.values()
): # if any code value is not empty, return
answer_dict.update(code_answer_dict)
return answer_dict
return normalize_answer_dict_codes(answer_dict)
# No Match - Why? Generic or Specific
last_check_answer = code_last_check(service_clean, "")
@@ -636,7 +661,7 @@ def extract_codes_from_service(answer_dict, constants: Constants):
elif last_check_answer == "Specific":
answer_dict["CODE_METHODOLOGY"] = "Specific - No Match"
return answer_dict
return normalize_answer_dict_codes(answer_dict)
def fill_claim_type(answer_dicts):
@@ -689,8 +714,14 @@ def code_breakout(merged_results: pd.DataFrame, constants: Constants):
for answer_dict in answer_dicts_with_code
]
return pd.DataFrame(answer_dicts_with_code)
df = pd.DataFrame(answer_dicts_with_code)
# Normalize code columns to JSON list format
for col in ["PROCEDURE_CD", "CPT4_PROC_MOD", "REVENUE_CD", "DIAG_CD", "GROUPER_CD", "NDC_CD", "CLAIM_ADMIT_TYPE_CD", "CLAIM_STATUS_CD"]:
if col in df.columns:
df[col] = df[col].apply(string_utils.normalize_to_json_list)
return df
def grouper_breakout(results_with_code: pd.DataFrame):
"""
@@ -729,4 +760,4 @@ def grouper_breakout(results_with_code: pd.DataFrame):
answer_dict.update(grouper_answer)
final_answer_dicts.append(answer_dict)
return pd.DataFrame(final_answer_dicts)
return pd.DataFrame(final_answer_dicts)