Katon Minhas
f8b8a580ff
fields/additional initializations * Set up test file * New fields + new field names * Add initial prompts to json * Column renaming - remove CONTRACT_ prefix * update tests * testing * Fix regex pattern in json.loads * Merge branch 'hotfix/file-errors' into refactor/new-field-names * add UNIT_OF_MEASURE * initialize new values * Add rate change interval * Merge branch 'main' into fields/additional-initializations * remove test * replace field names * update REIMB_ID * update date columns * update reimbursement effective dt * Revert adhoc scripts * Revert client files * Update postprocessing_funcs * revert regex_utils * Fix valid issue * UNIT_OF_MEASURE * REIMB_TERM * Revert postprocessing funcs tests Approved-by: Alex Galarce
376 lines
14 KiB
Plaintext
376 lines
14 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# !pip install rapidfuzz"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from one_to_n_comparison import evaluate_one_to_n_fields_fuzzy_matching_field_by_field\n",
|
|
"import pandas as pd\n",
|
|
"import os"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# import sys\n",
|
|
"# sys.path.append('../../') # Update this path to the correct path where the 'src' module is located\n",
|
|
"\n",
|
|
"os.chdir('../../')\n",
|
|
"print(os.getcwd())"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from src import config\n",
|
|
"\n",
|
|
"config.RUN_MODE = \"local\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"MATCH_THRESHOLD = 90\n",
|
|
"# This is the labeled data from sharepoint. It can be found at\n",
|
|
"# OneDrive - AArete, LLC\\investment_samples\\Test Bed Files\n",
|
|
"# You'll need access to the `investment_samples` sharepoint folder\n",
|
|
"file_path = \"/root/doczy.ai/fieldExtraction/src/testbed/Reimbursement Field Files/\"\n",
|
|
"file_path = \"Reimbursement Field Files/\"\n",
|
|
"labels_files = os.listdir(file_path)\n",
|
|
"\n",
|
|
"# fields = [\"SERVICE_TERM\", \"REIMB_TERM\"] # comparison fields\n",
|
|
"fields = [\"AARETE_DERIVED_REIMB_METHOD\", \n",
|
|
" \"REIMB_FEE_RATE\", \n",
|
|
" # \"REIMB_PCT_RATE\",\n",
|
|
" \"FEE_SCHEDULE\",\n",
|
|
" \"AARETE_DERIVED_FEE_SCHEDULE\",\n",
|
|
" \"AARETE_DERIVED_FEE_SCHEDULE_VERSION\"\n",
|
|
" ]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"\n",
|
|
"# make list of dataframes from labels_files\n",
|
|
"# Track where labels are coming from\n",
|
|
"labels_dfs = []\n",
|
|
"for filename in labels_files:\n",
|
|
" print(f\"Reading {filename}\")\n",
|
|
" if filename == \"DocziAI_2-12.xlsx\":\n",
|
|
" df_extract = pd.read_excel(file_path + filename, sheet_name=\"doczy-ai-test-20240211\", skiprows=3)\n",
|
|
" df_extract['data_source'] = filename\n",
|
|
" labels_dfs.append(df_extract)\n",
|
|
" elif filename.endswith(\".xlsx\"):\n",
|
|
" df_extract = pd.read_excel(file_path + filename)\n",
|
|
" df_extract['data_source'] = filename\n",
|
|
" labels_dfs.append(df_extract)\n",
|
|
" elif filename.endswith(\".csv\"):\n",
|
|
" df_extract = pd.read_csv(file_path + filename)\n",
|
|
" df_extract['data_source'] = filename\n",
|
|
" labels_dfs.append(df_extract)\n",
|
|
"\n",
|
|
"print(f\"Extracted {len(labels_dfs)} dataframes.\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"labels_df = pd.concat(labels_dfs, ignore_index=True)\n",
|
|
"# This is any consolidated output from the model; they can be found in S3\n",
|
|
"# predictions_df = pd.read_csv('inv-test-0305-PM-RESULTS.csv')\n",
|
|
"\n",
|
|
"# For all float and int columns, standardize to Int64 and handle NaN values (optional)\n",
|
|
"# labels_df = labels_df.astype({col: 'Int64' for col in labels_df.select_dtypes(include=['float', 'int']).columns})\n",
|
|
"# predictions_df = predictions_df.astype({col: 'Int64' for col in predictions_df.select_dtypes(include=['float', 'int']).columns})\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"labels_df[labels_df['data_source'] == 'DocziAI_2-12.xlsx']\n",
|
|
"\n",
|
|
"for fn in labels_df['CONTRACT_FILE_NAME'].unique():\n",
|
|
" n_sources = len(labels_df[labels_df['CONTRACT_FILE_NAME'] == fn]['data_source'].unique())\n",
|
|
" if n_sources != 1:\n",
|
|
" # Remove rows from docziAI that have a separate data source\n",
|
|
" labels_df = labels_df[~((labels_df['CONTRACT_FILE_NAME'] == fn) & (labels_df['data_source'] == 'DocziAI_2-12.xlsx'))]\n",
|
|
" print(fn, n_sources)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"unique_contracts_labels = labels_df['CONTRACT_FILE_NAME'].nunique()\n",
|
|
"print(f\"There are {unique_contracts_labels} unique contract file names in the labels dataframe.\")\n",
|
|
"\n",
|
|
"labels_df.to_csv('labels_df.csv', index=False)\n",
|
|
"\n",
|
|
"# unique_contracts_predictions = predictions_df['CONTRACT_FILE_NAME'].nunique()\n",
|
|
"# print(f\"There are {unique_contracts_predictions} unique contract file names in the predictions dataframe.\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"labels_df = pd.read_csv('labels_df.csv')\n",
|
|
"\n",
|
|
"labels_df = labels_df[['CONTRACT_FILE_NAME',\n",
|
|
" 'SERVICE_TERM',\n",
|
|
" 'REIMB_TERM',\n",
|
|
" 'CARVEOUT_IND',\n",
|
|
" 'LESSER_OF_IND',\n",
|
|
" 'GREATER_OF_IND',\n",
|
|
" 'AARETE_DERIVED_REIMB_METHOD',\n",
|
|
" 'REIMB_FEE_RATE',\n",
|
|
" 'REIMB_PCT_RATE',\n",
|
|
" 'NOT_TO_EXCEED_IND',\n",
|
|
" 'DEFAULT_IND',\n",
|
|
" 'FEE_SCHEDULE',\n",
|
|
" 'AARETE_DERIVED_FEE_SCHEDULE',\n",
|
|
" 'AARETE_DERIVED_FEE_SCHEDULE_VERSION',\n",
|
|
"# 'FEE_SCHEDULE_VERSION'\n",
|
|
" ]]\n",
|
|
"labels_df = labels_df.drop_duplicates()\n",
|
|
"labels_df = labels_df.replace('N/A', '').fillna('')\n",
|
|
"labels_df['REIMB_PCT_RATE'] = labels_df['REIMB_PCT_RATE'].replace('', 100)\n",
|
|
"# labels_df[\"CONTRACT_FILE_NAME\"].unique()\n",
|
|
"labels_df = labels_df[labels_df[\n",
|
|
" 'CONTRACT_FILE_NAME'].isin([\"95-3372911; Multiple-Davita Healthcare Partners, Inc.-ICMProviderAgreement_138658.txt\",\n",
|
|
" \"Custom_2018-09-15 COMM Agmt FE - Bandon Community Health Center dba Coast Community Health Center.txt\",\n",
|
|
" \"Progressive Women's Health, PLLC_Physician Agreement MU.txt\",\n",
|
|
" \"Akron General Health System_Eleventh Amendment_20171001.txt\",\n",
|
|
" \"2021-07-15 COMM Agmt FE - Samaritan Health Services MU.txt\",\n",
|
|
" \"2019 01 01 Gulf Coast Division (Agreement_STAR, CHIP, CHIP P, STAR_PLUS) MU.txt\",\n",
|
|
" \"2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof.txt\"])]\n",
|
|
"print(labels_df.shape)\n",
|
|
"labels_df.to_csv('labels_to_compare.csv', index=False)\n",
|
|
"labels_df = pd.read_csv('labels_to_compare.csv')\n",
|
|
"\n",
|
|
"labels_df_for_prediction = labels_df[[\"CONTRACT_FILE_NAME\", \"SERVICE_TERM\", \"CARVEOUT_IND\", \"REIMB_TERM\"]]\n",
|
|
"labels_df_unique_rows = labels_df_for_prediction.drop_duplicates()\n",
|
|
"labels_df_unique_rows.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from src.investment.one_to_n_funcs import get_methodology_breakout\n",
|
|
"\n",
|
|
"reimbursement_primary_answers = labels_df_unique_rows[[\"CONTRACT_FILE_NAME\", \"SERVICE_TERM\", \"CARVEOUT_IND\", \"REIMB_TERM\"]].to_dict(orient='records')\n",
|
|
"predictions_list = []\n",
|
|
"\n",
|
|
"for reimbursement_primary in reimbursement_primary_answers:\n",
|
|
" filename = reimbursement_primary[\"CONTRACT_FILE_NAME\"]\n",
|
|
" methodology_breakout_answers = get_methodology_breakout([reimbursement_primary], filename)\n",
|
|
" print(reimbursement_primary)\n",
|
|
" print(methodology_breakout_answers)\n",
|
|
" predictions_list.extend(methodology_breakout_answers)\n",
|
|
"\n",
|
|
"predictions_df = pd.DataFrame(predictions_list)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Remove rows where 'AARETE_DERIVED_REIMB_METHOD' value is \"Medicare Member Cost Share\"\n",
|
|
"predictions_df = predictions_df[predictions_df['AARETE_DERIVED_REIMB_METHOD'] != \"Medicare Member Cost Share\"]\n",
|
|
"\n",
|
|
"# Standardize 'AARETE_DERIVED_REIMB_METHOD' values\n",
|
|
"# Copy AWP, ASP, WAC to AARETE_DERIVED_FEE_SCHEDULE column for the corresponding row\n",
|
|
"medrx_fee_schedule_values = ['AWP', 'ASP', 'WAC']\n",
|
|
"predictions_df.loc[predictions_df['AARETE_DERIVED_REIMB_METHOD'].isin(medrx_fee_schedule_values), 'AARETE_DERIVED_FEE_SCHEDULE'] = predictions_df['AARETE_DERIVED_REIMB_METHOD']\n",
|
|
"\n",
|
|
"# Convert AARETE_DERIVED_REIMB_METHOD values [AWP, ASP, WAC] to 'MedRx'\n",
|
|
"predictions_df['AARETE_DERIVED_REIMB_METHOD'] = predictions_df['AARETE_DERIVED_REIMB_METHOD'].replace({'AWP': 'MedRx', 'ASP': 'MedRx', 'WAC': 'MedRx'})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"predictions_df = predictions_df.replace('N/A', '').fillna('')\n",
|
|
"predictions_df.to_csv('predictions.csv', index=False)\n",
|
|
"predictions_df = pd.read_csv('predictions.csv')\n",
|
|
"predictions_df.drop(\"FEE_SCHEDULE_VERSION\", axis=1, inplace=True)\n",
|
|
"\n",
|
|
"predictions_df.shape"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"precisions = []\n",
|
|
"recalls = []\n",
|
|
"accuracies = []\n",
|
|
"\n",
|
|
"comparisons = {}\n",
|
|
"\n",
|
|
"total_true_positives = 0\n",
|
|
"total_false_positives = 0\n",
|
|
"total_false_negatives = 0\n",
|
|
"\n",
|
|
"\n",
|
|
"for contract_file_name in labels_df[\"CONTRACT_FILE_NAME\"].unique():\n",
|
|
" print(f\"Processing {contract_file_name}\")\n",
|
|
" labels_df_contract = labels_df[\n",
|
|
" labels_df[\"CONTRACT_FILE_NAME\"] == contract_file_name\n",
|
|
" ]\n",
|
|
" predictions_df_contract = predictions_df[\n",
|
|
" predictions_df[\"CONTRACT_FILE_NAME\"] == contract_file_name\n",
|
|
" ]\n",
|
|
" if len(labels_df_contract) == 0:\n",
|
|
" print(f\"Contract {contract_file_name} not found in labels.\\n\")\n",
|
|
" continue\n",
|
|
" elif len(predictions_df_contract) == 0:\n",
|
|
" print(f\"Contract {contract_file_name} not found in predictions.\\n\")\n",
|
|
" continue\n",
|
|
"\n",
|
|
" # Full reimbursement primary\n",
|
|
" comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(\n",
|
|
" labels_df_contract,\n",
|
|
" predictions_df_contract,\n",
|
|
" \"CONTRACT_FILE_NAME\",\n",
|
|
" fields,\n",
|
|
" thresholds=MATCH_THRESHOLD\n",
|
|
" )\n",
|
|
" comparisons[contract_file_name] = comparison\n",
|
|
"\n",
|
|
" # comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(labels_df_contract, predictions_df_contract, \"CONTRACT_FILE_NAME\", ['REIMB_TERM', 'FEE_SCHEDULE'])\n",
|
|
"\n",
|
|
" precisions.append(comparison[\"precision\"])\n",
|
|
" recalls.append(comparison[\"recall\"])\n",
|
|
" accuracies.append(comparison[\"accuracy\"])\n",
|
|
"\n",
|
|
"\n",
|
|
" total_true_positives += comparison[\"true_positives\"]\n",
|
|
" total_false_positives += comparison[\"false_positives\"]\n",
|
|
" total_false_negatives += comparison[\"false_negatives\"]\n",
|
|
"\n",
|
|
" # contract file name is already printed above\n",
|
|
" print(\n",
|
|
" f\"false_negatives: {comparison['false_negatives']} | false_positives: {comparison['false_positives']} | true_positives: {comparison['true_positives']} | accuracy: {comparison['accuracy']} | precision: {comparison['precision']} | recall: {comparison['recall']} | f1_score: {comparison['f1_score']}\\n\"\n",
|
|
" )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from statistics import median, mean\n",
|
|
"\n",
|
|
"print(f\"Median accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {median(accuracies):.4f}\")\n",
|
|
"print(f\"Median precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {median(precisions):.4f}\")\n",
|
|
"print(f\"Median recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {median(recalls):.4f}\")\n",
|
|
"print(f\"Mean accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {mean(accuracies):.4f}\")\n",
|
|
"print(f\"Mean precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {mean(precisions):.4f}\")\n",
|
|
"print(f\"Mean recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {mean(recalls):.4f}\")\n",
|
|
"print(f\"Grand accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {(total_true_positives) / (total_true_positives + total_false_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives}, total false negatives: {total_false_negatives})\")\n",
|
|
"print(f\"Grand precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_positives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives})\")\n",
|
|
"print(f\"Grand recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false negatives: {total_false_negatives})\")\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# comparisons['2001-11-01 California Emergency Physicians PSA_MU.txt']"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# comparisons['02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt']"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.3"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|