Merged in feature/apply-postprocessing-to-test-bed (pull request #526)
Refactor CPT postprocessing, create notebook to postprocess test bed for MCS team * Move comment * Implement testbed postprocessing functions * Refactor normalize CPT postprocessing * remove old notebook, add new postprocess notebook * Enhance normalize_cpt_fields to handle comma-separated values in lists * Update normalize_cpt_fields to return an empty string for None values * Add testbed postprocessing script, removed notebook Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
@@ -195,18 +195,34 @@ def normalize_cpt_fields(value):
|
||||
Returns:
|
||||
list: A normalized list of string codes.
|
||||
"""
|
||||
if value is None or (isinstance(value, float) and pd.isna(value)): # Handle NaN values
|
||||
result = []
|
||||
elif isinstance(value, (list, tuple)): # If already a list or tuple
|
||||
if not value or (isinstance(value, float) and pd.isna(value)): # Handle NaN values
|
||||
return "" # Empty string
|
||||
|
||||
# First normalize to a list
|
||||
if isinstance(value, (list, tuple)): # If already a list or tuple
|
||||
result = [str(v).strip() for v in value]
|
||||
elif isinstance(value, str): # If it's a string
|
||||
value = value.strip()
|
||||
if value.startswith("[") and value.endswith("]"): # String representation of a list
|
||||
try:
|
||||
result = [str(v).strip() for v in ast.literal_eval(value)]
|
||||
# Parse it first
|
||||
parsed_list = ast.literal_eval(value)
|
||||
result = []
|
||||
# Process each item in the parsed list
|
||||
for item in parsed_list:
|
||||
item_str = str(item).strip()
|
||||
# If the item contains commas, split it into multiple items
|
||||
if "," in item_str:
|
||||
result.extend([str(v).strip() for v in item_str.split(",")])
|
||||
else:
|
||||
result.append(item_str)
|
||||
except (ValueError, SyntaxError):
|
||||
result = [value] # If parsing fails, treat it as a single value
|
||||
elif "-" in value: # If it's a range
|
||||
elif "," in value: # If it's a comma-separated string
|
||||
# Split by comma and create a list
|
||||
result = [str(v).strip() for v in value.split(",")]
|
||||
elif "-" in value:
|
||||
# Range format
|
||||
result = [value]
|
||||
else: # Single value
|
||||
result = [value]
|
||||
@@ -215,7 +231,7 @@ def normalize_cpt_fields(value):
|
||||
else:
|
||||
result = [str(value).strip()] # Default case for other types
|
||||
|
||||
return result # Return the list directly
|
||||
return str(result) # Return the list directly
|
||||
|
||||
|
||||
def update_reimb_prov_name(df):
|
||||
|
||||
@@ -13,8 +13,8 @@ def postprocess(df):
|
||||
|
||||
df = investment_postprocessing_funcs.date_postprocess(df, config.FIELD_JSON_PATH)
|
||||
|
||||
# Format rate fields
|
||||
if "REIMB_FEE_RATE" in df.columns:
|
||||
# Format rate fields
|
||||
df['REIMB_FEE_RATE'] = df['REIMB_FEE_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas)
|
||||
if "REIMB_PCT_RATE" in df.columns:
|
||||
df['REIMB_PCT_RATE'] = df['REIMB_PCT_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas)
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
# This script postprocesses the testbed data from an Excel file. It uses a custom
|
||||
# function `testbed_postprocess` from the `testbed_utils` module (similar to
|
||||
# postprocess.postprocess() but skipping some steps)
|
||||
|
||||
# Run this from the fieldExtraction directory:
|
||||
# poetry run python -m src.testbed.testbed_postprocess
|
||||
|
||||
#### CONFIG ####
|
||||
testbed_filename = "src/testbed/v6_DoczyAI_all_test_bed_LIVE_5.6.25.xlsx"
|
||||
testbed_sheetname = 'all_test_bed_labels'
|
||||
output_filename = "src/testbed/v6_DoczyAI_all_test_bed_LIVE_5.6.25_POSTPROCESSED.xlsx"
|
||||
|
||||
#### IMPORTS ####
|
||||
import pandas as pd
|
||||
from src.testbed.testbed_utils import testbed_postprocess
|
||||
|
||||
#### EXECUTION ####
|
||||
# Load the testbed
|
||||
df = pd.read_excel(testbed_filename, sheet_name=testbed_sheetname)
|
||||
print(f"Loaded testbed from {testbed_filename} with {df.shape[0]} rows and {df.shape[1]} columns")
|
||||
|
||||
# Postprocess the testbed
|
||||
testbed_postprocess(df).to_excel(output_filename, index=False)
|
||||
|
||||
print(f"Postprocessed testbed saved to {output_filename}")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,10 @@ import itertools
|
||||
|
||||
from collections import Counter
|
||||
|
||||
# Imports for the testbed postprocessing functions
|
||||
from src.investment import investment_postprocessing_funcs
|
||||
from src.constants.investment_columns import COLUMN_ORDER
|
||||
|
||||
|
||||
def convert_to_float(value):
|
||||
try:
|
||||
@@ -29,7 +33,15 @@ def normalize_tin_npi(value):
|
||||
return value
|
||||
|
||||
def testbed_preprocess(df):
|
||||
df = df.applymap(convert_to_float)
|
||||
# Apply numeric conversion to non-CPT/diag/rev fields only
|
||||
for col in df.columns:
|
||||
if "_CD" not in col and "CPT" not in col and "DIAG" not in col and "REV" not in col:
|
||||
df[col] = df[col].apply(convert_to_float)
|
||||
else:
|
||||
# For CPT/diag/rev fields, apply string conversion AND convert | delimiters
|
||||
# to commas
|
||||
df[col] = df[col].astype(str).str.replace("|", ",", regex=False)
|
||||
|
||||
df = df.applymap(lambda x: str(x).strip().upper())
|
||||
df = df.replace("UNKNOWN", "").fillna("")
|
||||
df = df.replace("NAN", "")
|
||||
@@ -50,101 +62,6 @@ def testbed_preprocess(df):
|
||||
|
||||
return df
|
||||
|
||||
def group_providers_into_json_array(df):
|
||||
"""Groups provider information from a DataFrame into a JSON array format and appends it as a new column.
|
||||
|
||||
This function processes a DataFrame containing provider information, groups the data by the "FILE_NAME" column,
|
||||
and creates a JSON array for each group. The JSON array includes details about the group provider and other
|
||||
associated providers. The resulting JSON array is added as a new column, "PROV_INFO_JSON", to the DataFrame.
|
||||
|
||||
Args:
|
||||
df (pandas.DataFrame): The input DataFrame containing the following columns:
|
||||
- "FILE_NAME": Identifier for grouping rows.
|
||||
- "PROV_GROUP_TIN": Tax Identification Number (TIN) of the group provider.
|
||||
- "PROV_GROUP_NPI": National Provider Identifier (NPI) of the group provider.
|
||||
- "PROV_GROUP_NAME_FULL": Full name of the group provider.
|
||||
- "PROV_OTHER_TIN": Pipe-separated TINs of other providers.
|
||||
- "PROV_OTHER_NPI": Pipe-separated NPIs of other providers.
|
||||
- "PROV_OTHER_NAME_FULL": Pipe-separated full names of other providers.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: A modified DataFrame with an additional column "PROV_INFO_JSON",
|
||||
which contains a JSON array for each "FILE_NAME" group. Each JSON array includes:
|
||||
- "TIN": TIN of the provider (or "N/A" if not available).
|
||||
- "NPI": NPI of the provider (or "N/A" if not available).
|
||||
- "NAME": Full name of the provider (or "N/A" if not available).
|
||||
- "IS_GROUP": Boolean indicating whether the provider is the group provider (True)
|
||||
or an associated provider (False).
|
||||
|
||||
Notes:
|
||||
- Missing or NaN values in the input DataFrame are replaced with "N/A" in the JSON output.
|
||||
- If the lengths of "PROV_OTHER_TIN", "PROV_OTHER_NPI", and "PROV_OTHER_NAME_FULL" are not equal,
|
||||
missing values are filled with "N/A" using itertools.zip_longest.
|
||||
- If the OTHER columns are misaligned with one another, the function will NOT
|
||||
be able to correct them. It will simply fill in the gaps with "N/A".
|
||||
"""
|
||||
|
||||
df = df.copy()
|
||||
# Create a dictionary to hold provider data in JSON array format for each file
|
||||
group_providers_dict = {}
|
||||
|
||||
for group, grouped_df in df.groupby("FILE_NAME"):
|
||||
json_dict_list = []
|
||||
|
||||
# Get the first row of the grouped DataFrame
|
||||
first_row = grouped_df.iloc[0]
|
||||
|
||||
|
||||
# Extract TIN/NPI/NAME for group provider.
|
||||
group_tin = str(first_row["PROV_GROUP_TIN"]).replace("nan", "N/A")
|
||||
group_npi = str(first_row["PROV_GROUP_NPI"]).replace("nan", "N/A")
|
||||
group_name_full = str(first_row["PROV_GROUP_NAME_FULL"])
|
||||
|
||||
# Append the group provider's TIN/NPI/NAME to the list.
|
||||
json_dict_list.append({
|
||||
"TIN": group_tin,
|
||||
"NPI": group_npi,
|
||||
"NAME": group_name_full, "IS_GROUP": True})
|
||||
|
||||
# Create lists of TIN/NPI/NAME for other providers.
|
||||
# Split on pipe, then strip whitespace.
|
||||
other_tins = str(first_row["PROV_OTHER_TIN"]).split("|")
|
||||
other_npis = str(first_row["PROV_OTHER_NPI"]).split("|")
|
||||
other_names = str(first_row["PROV_OTHER_NAME_FULL"]).split("|")
|
||||
other_tins = [tin.strip().replace("nan", "N/A") for tin in other_tins]
|
||||
other_npis = [npi.strip().replace("nan", "N/A") for npi in other_npis]
|
||||
other_names = [name.strip().replace("nan", "N/A") for name in other_names]
|
||||
|
||||
# Append each other provider's TIN/NPI/NAME to the list.
|
||||
# Use zip to iterate over the three lists in parallel.
|
||||
# IF the lengths of the lists are not equal, use itertools.zip_longest to fill in the gaps with N/A.
|
||||
for tin, npi, name in itertools.zip_longest(other_tins, other_npis, other_names, fillvalue="N/A"):
|
||||
json_dict_list.append({
|
||||
"TIN": tin,
|
||||
"NPI": npi,
|
||||
"NAME": name,
|
||||
"IS_GROUP": False
|
||||
})
|
||||
|
||||
group_providers_dict[group] = json_dict_list
|
||||
|
||||
# Add JSON data back to original DataFrame
|
||||
df["PROV_INFO_JSON"] = df["FILE_NAME"].map(group_providers_dict)
|
||||
|
||||
|
||||
## Reorder columns to place PROV_INFO_JSON right after PROV_OTHER_NAME_FULL
|
||||
cols = list(df.columns)
|
||||
# Find the position of PROV_OTHER_NAME_FULL
|
||||
prov_other_name_full_pos = cols.index("PROV_OTHER_NAME_FULL")
|
||||
# Remove PROV_INFO_JSON from its current position
|
||||
cols.remove("PROV_INFO_JSON")
|
||||
# Insert it after PROV_OTHER_NAME_FULL
|
||||
cols.insert(prov_other_name_full_pos + 1, "PROV_INFO_JSON")
|
||||
# Reorder the DataFrame
|
||||
df = df[cols]
|
||||
|
||||
return df
|
||||
|
||||
def calculate_field_metrics(testbed_df, results_df, field):
|
||||
"""Calculate precision, recall, and accuracy for a single field.
|
||||
|
||||
@@ -557,4 +474,158 @@ def calculate_precision_recall(accuracies):
|
||||
precision_df = pd.DataFrame(precision_dicts)
|
||||
recall_df = pd.DataFrame(recall_dicts)
|
||||
|
||||
return precision_df, recall_df, overall_metrics_df
|
||||
return precision_df, recall_df, overall_metrics_df
|
||||
|
||||
|
||||
|
||||
########## TESTBED POSTPROCESSING CODE ##########
|
||||
# These functions are to take MCS's test bed that they provided and apply
|
||||
# (most of) the same postprocessing steps that we apply to the results of the model.
|
||||
|
||||
|
||||
|
||||
def group_providers_into_json_array(df):
|
||||
"""Groups provider information from a DataFrame into a JSON array format and appends it as a new column.
|
||||
|
||||
This function processes a DataFrame containing provider information, groups the data by the "FILE_NAME" column,
|
||||
and creates a JSON array for each group. The JSON array includes details about the group provider and other
|
||||
associated providers. The resulting JSON array is added as a new column, "PROV_INFO_JSON", to the DataFrame.
|
||||
|
||||
Args:
|
||||
df (pandas.DataFrame): The input DataFrame containing the following columns:
|
||||
- "FILE_NAME": Identifier for grouping rows.
|
||||
- "PROV_GROUP_TIN": Tax Identification Number (TIN) of the group provider.
|
||||
- "PROV_GROUP_NPI": National Provider Identifier (NPI) of the group provider.
|
||||
- "PROV_GROUP_NAME_FULL": Full name of the group provider.
|
||||
- "PROV_OTHER_TIN": Pipe-separated TINs of other providers.
|
||||
- "PROV_OTHER_NPI": Pipe-separated NPIs of other providers.
|
||||
- "PROV_OTHER_NAME_FULL": Pipe-separated full names of other providers.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: A modified DataFrame with an additional column "PROV_INFO_JSON",
|
||||
which contains a JSON array for each "FILE_NAME" group. Each JSON array includes:
|
||||
- "TIN": TIN of the provider (or "N/A" if not available).
|
||||
- "NPI": NPI of the provider (or "N/A" if not available).
|
||||
- "NAME": Full name of the provider (or "N/A" if not available).
|
||||
- "IS_GROUP": Boolean indicating whether the provider is the group provider (True)
|
||||
or an associated provider (False).
|
||||
|
||||
Notes:
|
||||
- Missing or NaN values in the input DataFrame are replaced with "N/A" in the JSON output.
|
||||
- If the lengths of "PROV_OTHER_TIN", "PROV_OTHER_NPI", and "PROV_OTHER_NAME_FULL" are not equal,
|
||||
missing values are filled with "N/A" using itertools.zip_longest.
|
||||
- If the OTHER columns are misaligned with one another, the function will NOT
|
||||
be able to correct them. It will simply fill in the gaps with "N/A".
|
||||
"""
|
||||
|
||||
df = df.copy()
|
||||
# Create a dictionary to hold provider data in JSON array format for each file
|
||||
group_providers_dict = {}
|
||||
|
||||
for group, grouped_df in df.groupby("FILE_NAME"):
|
||||
json_dict_list = []
|
||||
|
||||
# Get the first row of the grouped DataFrame
|
||||
first_row = grouped_df.iloc[0]
|
||||
|
||||
|
||||
# Extract TIN/NPI/NAME for group provider.
|
||||
group_tin = str(first_row["PROV_GROUP_TIN"]).replace("nan", "N/A")
|
||||
group_npi = str(first_row["PROV_GROUP_NPI"]).replace("nan", "N/A")
|
||||
group_name_full = str(first_row["PROV_GROUP_NAME_FULL"])
|
||||
|
||||
# Append the group provider's TIN/NPI/NAME to the list.
|
||||
json_dict_list.append({
|
||||
"TIN": group_tin,
|
||||
"NPI": group_npi,
|
||||
"NAME": group_name_full, "IS_GROUP": True})
|
||||
|
||||
# Create lists of TIN/NPI/NAME for other providers.
|
||||
# Split on pipe, then strip whitespace.
|
||||
other_tins = str(first_row["PROV_OTHER_TIN"]).split("|")
|
||||
other_npis = str(first_row["PROV_OTHER_NPI"]).split("|")
|
||||
other_names = str(first_row["PROV_OTHER_NAME_FULL"]).split("|")
|
||||
other_tins = [tin.strip().replace("nan", "N/A") for tin in other_tins]
|
||||
other_npis = [npi.strip().replace("nan", "N/A") for npi in other_npis]
|
||||
other_names = [name.strip().replace("nan", "N/A") for name in other_names]
|
||||
|
||||
# Append each other provider's TIN/NPI/NAME to the list.
|
||||
# Use zip to iterate over the three lists in parallel.
|
||||
# IF the lengths of the lists are not equal, use itertools.zip_longest to fill in the gaps with N/A.
|
||||
for tin, npi, name in itertools.zip_longest(other_tins, other_npis, other_names, fillvalue="N/A"):
|
||||
json_dict_list.append({
|
||||
"TIN": tin,
|
||||
"NPI": npi,
|
||||
"NAME": name,
|
||||
"IS_GROUP": False
|
||||
})
|
||||
|
||||
group_providers_dict[group] = json_dict_list
|
||||
|
||||
# Add JSON data back to original DataFrame
|
||||
df["PROV_INFO_JSON"] = df["FILE_NAME"].map(group_providers_dict)
|
||||
|
||||
|
||||
## Reorder columns to place PROV_INFO_JSON right after PROV_OTHER_NAME_FULL
|
||||
cols = list(df.columns)
|
||||
# Find the position of PROV_OTHER_NAME_FULL
|
||||
prov_other_name_full_pos = cols.index("PROV_OTHER_NAME_FULL")
|
||||
# Remove PROV_INFO_JSON from its current position
|
||||
cols.remove("PROV_INFO_JSON")
|
||||
# Insert it after PROV_OTHER_NAME_FULL
|
||||
cols.insert(prov_other_name_full_pos + 1, "PROV_INFO_JSON")
|
||||
# Reorder the DataFrame
|
||||
df = df[cols]
|
||||
|
||||
return df
|
||||
|
||||
def testbed_postprocess(df):
|
||||
"""Postprocess the testbed DataFrame.
|
||||
|
||||
Args:
|
||||
df (pandas.DataFrame): The input DataFrame containing the testbed data.
|
||||
|
||||
Returns:
|
||||
pandas.DataFrame: The postprocessed DataFrame.
|
||||
"""
|
||||
# Apply preprocessing steps
|
||||
df = testbed_preprocess(df)
|
||||
|
||||
# Group providers into JSON array format
|
||||
df = group_providers_into_json_array(df)
|
||||
|
||||
# Add CLIENT_NAME column
|
||||
df['CLIENT_NAME'] = "AARETE_TESTBED"
|
||||
|
||||
# Reformat rate fields
|
||||
if "REIMB_FEE_RATE" in df.columns:
|
||||
# Format rate fields
|
||||
df['REIMB_FEE_RATE'] = df['REIMB_FEE_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas)
|
||||
if "REIMB_PCT_RATE" in df.columns:
|
||||
df['REIMB_PCT_RATE'] = df['REIMB_PCT_RATE'].apply(investment_postprocessing_funcs.format_rate_fields_with_commas)
|
||||
|
||||
for col in df.columns:
|
||||
if "_IND" in col:
|
||||
df[col] = df[col].apply(investment_postprocessing_funcs.normalize_indicator_field)
|
||||
# Normalize _IND fields and clean up TIN/NPI fields
|
||||
if "TIN" in col or "NPI" in col:
|
||||
df[col] = df[col].apply(investment_postprocessing_funcs.remove_hyphens)
|
||||
# Apply the flatten_singleton_string_list function
|
||||
if "_CD" in col and "CPT" not in col:
|
||||
df[col] = df[col].apply(investment_postprocessing_funcs.flatten_singleton_string_list)
|
||||
# Convert DATE to YYYY/MM/DD format
|
||||
if "_DT" in col or "DATE" in col:
|
||||
df[col] = df[col].apply(investment_postprocessing_funcs.validate_and_reformat_date)
|
||||
# Normalize CPT fields
|
||||
if "CPT" in col:
|
||||
df[col] = df[col].apply(investment_postprocessing_funcs.normalize_cpt_fields)
|
||||
|
||||
df = investment_postprocessing_funcs.update_reimb_prov_name(df)
|
||||
|
||||
# Add reimb_id
|
||||
df = investment_postprocessing_funcs.generate_reimb_ids(df)
|
||||
|
||||
# Standardize output column order - this should ALWAYS be the final postprocessing step
|
||||
df = investment_postprocessing_funcs.reorder_columns(df, COLUMN_ORDER)
|
||||
|
||||
return df
|
||||
@@ -100,12 +100,13 @@ class TestPostprocessFunctions(unittest.TestCase):
|
||||
self.assertEqual(normalize_auto_renewal_term(None), "")
|
||||
|
||||
def test_normalize_cpt_fields(self):
|
||||
self.assertEqual(normalize_cpt_fields("[123, 456]"), ["123", "456"])
|
||||
self.assertEqual(normalize_cpt_fields("123-456"), ["123-456"])
|
||||
self.assertEqual(normalize_cpt_fields("123"), ["123"])
|
||||
self.assertEqual(normalize_cpt_fields(None), [])
|
||||
self.assertEqual(normalize_cpt_fields(123), ["123"])
|
||||
|
||||
self.assertEqual(normalize_cpt_fields("[123, 456]"), "['123', '456']")
|
||||
self.assertEqual(normalize_cpt_fields("123-456"), "['123-456']")
|
||||
self.assertEqual(normalize_cpt_fields("['T0000-T9999, S0000-S9999']"), "['T0000-T9999', 'S0000-S9999']")
|
||||
self.assertEqual(normalize_cpt_fields("123"), "['123']")
|
||||
self.assertEqual(normalize_cpt_fields(None), "")
|
||||
self.assertEqual(normalize_cpt_fields(123), "['123']")
|
||||
|
||||
def test_process_patient_age_range(self):
|
||||
"""Tests the process_patient_age_range function with various age range formats.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user