From 04cb261831dcee59a14d06d5fdceddc84d4735bd Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Mon, 7 Apr 2025 20:43:17 +0000 Subject: [PATCH] Merged in feature/rework_reimb_id (pull request #467) Rework reimbursement ID to associate parts of "lesser of" (or other such coupled statement) with one another * refactor reimbursement ID generation with hashed service terms * reduce hash length for unique reimbursement ID generation * update reimbursement ID format to include 'exh_pg' for clarity * clean up testing.ipynb * one more formatting fix to testing.ipynb * Move reimb_id generation to postprocessing * move reimb_id function to postprocessing_funcs.py * refactor generate_reimb_ids to group by FILE_NAME and EXHIBIT_PAGE for ID generation * clean testing.ipynb further Approved-by: Katon Minhas --- .../investment_field_testing/testing.ipynb | 284 +----------------- .../src/investment/one_to_n_funcs.py | 16 +- fieldExtraction/src/investment/postprocess.py | 5 +- fieldExtraction/src/postprocessing_funcs.py | 60 +++- 4 files changed, 80 insertions(+), 285 deletions(-) diff --git a/fieldExtraction/scripts/investment_field_testing/testing.ipynb b/fieldExtraction/scripts/investment_field_testing/testing.ipynb index 7202f07..1c03f1e 100644 --- a/fieldExtraction/scripts/investment_field_testing/testing.ipynb +++ b/fieldExtraction/scripts/investment_field_testing/testing.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "# Manual Test Bed\n", - "Runs contracts through from `input_data`, but cannot yet automatically check them against the answer key (that workbook is unreliable in its answers)\n", + "Runs contracts through from `input_data`\n", "\n", "To make this work: in `config.py`, change READ_MODE and RUN_MODE to 'local'.\n", "Create an .env in the base directory of your project according to these instructions:\n", @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -26,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -34,194 +34,38 @@ "TEST_DATA_INPUT_DIR = \"scripts/investment_field_testing/input_data\"" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "For now, we are no longer doing anything with the test bed" - ] - }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# # Load target data from `scripts/investment_field_testing/input_data` directory\n", - "\n", - "# df_targets = pd.read_excel(TEST_DATA_INPUT_DIR + \"/test_bed_investment.xlsx\", sheet_name=\"test_bed_investment\")\n", - "\n", - "# print(df_targets.shape)\n", - "\n", - "# df_targets.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Also change config.RUN_MODE and config.READ_MODE to 'local'\n", "test_params = {}\n", - "# test_params['local_input_dir'] = TEST_DATA_INPUT_DIR + \"/investment_228\"\n", - "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR + \"/mix_all_358\"\n", + "test_params['local_input_dir'] = TEST_DATA_INPUT_DIR\n", "test_params['max_workers'] = 1\n", - "test_params['input_files'] = [\"2001-11-01 California Emergency Physicians PSA_MU.txt\"] # you can specify specific files to run here" + "test_params['input_files'] = [ # you can specify specific files to run here\n", + " \"00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\",\n", + " # \"20-2401676-Community Hospital of LaGrange County, Inc dba Parkview LaGrange Hospital-ICMProviderAgreement_89356_1.txt\",\n", + " ]" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "doczy-output exists\n" - ] - } - ], + "outputs": [], "source": [ "from src.investment.main import main as investment_main" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Total Input Files : 1\n", - "Processing 2001-11-01 California Emergency Physicians PSA_MU.txt...\n", - "Preprocessing Complete - 2001-11-01 California Emergency Physicians PSA_MU.txt\n", - "Running regex fields...\n", - "Running smart chunked fields...\n", - "\n", - "fields: [, , , , ]\n", - "field_group_answer_raw: I'll analyze the contract and answer each question, then provide the final answers in JSON format.\n", - "\n", - "CONTRACT_AUTO_RENEWAL_TERM:\n", - "The contract states in section 5.1: \"this agreement shall automatically renew for successive one year periods, unless terminated as set forth herein.\" Therefore, the auto-renewal term is one year.\n", - "\n", - "CONTRACT_EFFECTIVE_DT:\n", - "The effective date is found on the signature page, which states: \"11/10/ 10-30-01\". This appears to be two dates, but following the instructions to preserve the exact format as it appears in the document, I'll use this as is.\n", - "\n", - "CONTRACT_TITLE:\n", - "The title of the contract is found at the beginning of the document: \"CALIFORNIA ER PHYSICIANS PHYSICIAN GROUP SERVICES AGREEMENT\".\n", - "\n", - "CONTRACT_TERMINATION_DT:\n", - "The contract does not specify a specific termination date. Section 5.1 states that the initial term is 12 months after the effective date, and it automatically renews for successive one-year periods unless terminated. Therefore, it's effectively \"year to year\".\n", - "\n", - "CONTRACT_AUTO_RENEWAL_IND:\n", - "The contract does automatically renew according to section 5.1, so the answer is Y.\n", - "\n", - "Here's the final answer in JSON format:\n", - "\n", - "{\n", - " \"CONTRACT_AUTO_RENEWAL_TERM\": \"one year\",\n", - " \"CONTRACT_EFFECTIVE_DT\": \"11/10/ 10-30-01\",\n", - " \"CONTRACT_TITLE\": \"CALIFORNIA ER PHYSICIANS PHYSICIAN GROUP SERVICES AGREEMENT\",\n", - " \"CONTRACT_TERMINATION_DT\": \"year to year\",\n", - " \"CONTRACT_AUTO_RENEWAL_IND\": \"Y\"\n", - "}\n", - "Running full context fields...\n", - "One to One Complete - 2001-11-01 California Emergency Physicians PSA_MU.txt\n", - "Postprocessing Complete - 2001-11-01 California Emergency Physicians PSA_MU.txt\n" - ] - } - ], + "outputs": [], "source": [ - "df_out = investment_main(testing=True, test_params=test_params)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
CONTRACT_AUTO_RENEWAL_TERMCONTRACT_EFFECTIVE_DTCONTRACT_TITLECONTRACT_TERMINATION_DTCONTRACT_AUTO_RENEWAL_INDCONTRACT_FILE_NAMECLIENT_NAMEAARETE_DERIVED_TERMINATION_DATE
0one year10/30/2001CALIFORNIA ER PHYSICIANS PHYSICIAN GROUP SERVI...year to yearY2001-11-01 California Emergency Physicians PSA...Test-ClientN/A
\n", - "
" - ], - "text/plain": [ - " CONTRACT_AUTO_RENEWAL_TERM CONTRACT_EFFECTIVE_DT \\\n", - "0 one year 10/30/2001 \n", - "\n", - " CONTRACT_TITLE CONTRACT_TERMINATION_DT \\\n", - "0 CALIFORNIA ER PHYSICIANS PHYSICIAN GROUP SERVI... year to year \n", - "\n", - " CONTRACT_AUTO_RENEWAL_IND \\\n", - "0 Y \n", - "\n", - " CONTRACT_FILE_NAME CLIENT_NAME \\\n", - "0 2001-11-01 California Emergency Physicians PSA... Test-Client \n", - "\n", - " AARETE_DERIVED_TERMINATION_DATE \n", - "0 N/A " - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_out.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# The following is not functional because we first need to clean the test bed" + "df_out, df_error = investment_main(testing=True, test_params=test_params)" ] }, { @@ -230,109 +74,13 @@ "metadata": {}, "outputs": [], "source": [ - "# one_to_one_fields = []\n", - "# for col in df_out.columns:\n", - "# if df_out[col].nunique() == 1:\n", - "# # print(col, df_out[col].nunique())\n", - "# one_to_one_fields.append(col)\n", - "\n", - "# exclude_list = ['CARVEOUT_IND', # 1:n\n", - "# \"GREATER_OF_IND\", # 1:n\n", - "# \"REIMB_FEE_RATE\", # 1:n\n", - "# \"NETWORK\", # dynamic\n", - "# \"CLAIM_TYPE_CD\", # not found in the source spreadsheet\n", - "# \"CONTRACT_AMENDMENT_NUM\", # not found in the source spreadsheet\n", - "# \"PROV_GROUP_NAME_FULL\", # not found in the source spreadsheet\n", - "# \"PROV_OTHER_NAME_FULL\", # This is called PROV_GROUP_NAME in the source spreadsheet\n", - "# \"CONTRACT_FILE_NAME\", # not found in the source spreadsheet\n", - "# \"CARVEOUT_CD\", # 1:n, also not found in the source spreadsheet\n", - "# 'PAYER_NAME',\n", - "# 'CLIENT_NAME',\n", - " \n", - "# ]\n", - "\n", - "# one_to_one_fields = [item for item in one_to_one_fields \n", - "# if item not in exclude_list]\n", - "\n", - "# one_to_one_fields" + "df_out" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# mapping_main_to_testbed = {\"AUTO_RENEWAL_TERM\" : \"AUTO_RENEWAL_TERM_LENGTH\", \"AARETE_DERIVED_TERMINATION_DT\" : \"TERMINATION_DT\"}\n", - "\n", - "# def apply_name_mapping(col: str):\n", - "# if col in mapping_main_to_testbed:\n", - "# return mapping_main_to_testbed[col]\n", - "# return col" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# exclude_list = [\"CLAIM_TYPE_CD\"]\n", - "\n", - "# for field in one_to_one_fields:\n", - "# field = apply_name_mapping(field)\n", - "# # if field == \"AUTO_RENEWAL_TERM\":\n", - "# # field = \"AUTO_RENEWAL_TERM_LENGTH\"\n", - "# # elif field == \"AARETE_DERIVED_TERMINATION_DT\":\n", - "# # field = \"TERMINATION_DT\"\n", - "# # elif field == \"CLAIM_TYPE_CD\":\n", - "# print(df_targets[field].head())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# row = df_out[one_to_one_fields].iloc[0]\n", - "# row" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# filter_target = df_targets[df_targets['Contract Name'] == 'Adena Health System National Template Agreement C19460229AA.txt']\n", - "\n", - "# for field, answer in row.items():\n", - "# print(field, f\"LLM answer: {answer}\")\n", - "# print(filter_target[apply_name_mapping(field)].iloc[0],\"\\n\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# filter_target" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "doczy-smart-chunking-blZ6pB_m-py3.12", + "display_name": "doczy-smart-chunking-U_35LsVn-py3.12", "language": "python", "name": "python3" }, diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 8077782..12a437f 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -1,4 +1,5 @@ import re +import hashlib from typing import Callable import src.constants.investment_values as investment_values @@ -458,21 +459,6 @@ def reimbursement_level( ) # print(f"{datetime_str()} code_breakout_answers complete for exhibit page {exhibit_page} of {filename}...") - # Add unique identifiers to each row composed of filename, exhibit page, service code, and index - for idx, row in enumerate(code_breakout_answers): - # Clean filename for identifier (remove path and extension) - clean_filename = filename.split("/")[-1].replace(".", "_") - - # Get service code (or description) and shorten it for the identifier - service_identifier = "" - if "SERVICE_TERM" in row: # This should never be empty but just in case - service_identifier = row["SERVICE_TERM"].replace(" ", "_")[:10] - - # Create unique ID - row["REIMB_ID"] = ( - f"{clean_filename}_{exhibit_page}_{service_identifier}_{idx+1:03d}" - ) - return code_breakout_answers else: return [] diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index d60f0b9..c8cd4f9 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -55,7 +55,10 @@ def postprocess(df): # Convert AARETE_DERIVED_REIMB_METHOD values [AWP, ASP, WAC] to 'MedRx' df['AARETE_DERIVED_REIMB_METHOD'] = df['AARETE_DERIVED_REIMB_METHOD'].replace({'AWP': 'MedRx', 'ASP': 'MedRx', 'WAC': 'MedRx'}) - + + # Add reimb_id + df = postprocessing_funcs.generate_reimb_ids(df) + # Standardize output column order - this should ALWAYS be the final postprocessing step df = postprocessing_funcs.reorder_columns(df, COLUMN_ORDER) diff --git a/fieldExtraction/src/postprocessing_funcs.py b/fieldExtraction/src/postprocessing_funcs.py index f62ec53..eec4f0a 100644 --- a/fieldExtraction/src/postprocessing_funcs.py +++ b/fieldExtraction/src/postprocessing_funcs.py @@ -1,6 +1,7 @@ import logging import re +import hashlib import numpy as np import pandas as pd @@ -1176,4 +1177,61 @@ def date_postprocessing(date: str) -> str: prompt = investment_prompts.date_fix_prompt(date) response = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, "date_fix") - return extract_text_from_delimiters(response, Delimiter.PIPE) \ No newline at end of file + return extract_text_from_delimiters(response, Delimiter.PIPE) + +def generate_reimb_ids(df: pd.DataFrame) -> pd.DataFrame: + """Generates REIMB_ID for each row in the DataFrame based on the FILE_NAME and other fields. + The REIMB_ID is constructed using the following format: + {FILE_NAME}_exh_pg_{EXHIBIT_PAGE}_{index}_{hash_value} + + Where: + - FILE_NAME is the name of the file (without extension). + - EXHIBIT_PAGE is the page number of the exhibit. + - index is the index of the row in the group. + - hash_value is the first 8 characters of the MD5 hash of a concatenated string of relevant fields. + The relevant fields are SERVICE_TERM, REIMB_TERM, PROGRAM, PRODUCT, NETWORK, and LOB. + + Args: + df (pd.DataFrame): Input DataFrame containing the columns FILE_NAME, EXHIBIT_PAGE, SERVICE_TERM, REIMB_TERM, PROGRAM, PRODUCT, NETWORK, and LOB. + The FILE_NAME column should contain the name of the file (with or without extension). + The EXHIBIT_PAGE column should contain the page number of the exhibit. + + Returns: + pd.DataFrame: DataFrame with generated REIMB_IDs for each row. + """ + # First sort to ensure consistent ordering + df = df.sort_values(by=['FILE_NAME', 'EXHIBIT_PAGE']) if "EXHIBIT_PAGE" in df.columns else df.sort_values(by=['FILE_NAME']) + + # Create a temporary copy to avoid SettingWithCopyWarning + df_temp = df.copy() + + # Process each file+exhibit_page group to reset counter for each exhibit page + group_cols = ['FILE_NAME', 'EXHIBIT_PAGE'] if "EXHIBIT_PAGE" in df.columns else ['FILE_NAME'] + for group_key, indices in df.groupby(group_cols).groups.items(): + if isinstance(group_key, tuple): + filename, exhibit_page = group_key + exhibit_page = str(exhibit_page).zfill(3) + else: + filename = group_key + exhibit_page = '000' + clean_filename = str(filename).split("/")[-1].replace(".", "_") + + for i, idx in enumerate(indices): + row = df.loc[idx] + + # Get fields to include in hash + service_term = str(row.get('SERVICE_TERM', '')) + reimb_term = str(row.get('REIMB_TERM', '')) + program = str(row.get('PROGRAM', '')) + product = str(row.get('PRODUCT', '')) + network = str(row.get('NETWORK', '')) + lob = str(row.get('LOB', '')) + + # Create hash string + hash_string = f"{clean_filename}_{service_term}_{reimb_term}_{program}_{product}_{network}_{lob}" + hash_value = hashlib.md5(hash_string.encode()).hexdigest()[:8] + + # Generate REIMB_ID + df_temp.at[idx, 'REIMB_ID'] = f"{clean_filename}_exh_pg_{exhibit_page}_{i+1:03d}_{hash_value}" + + return df_temp \ No newline at end of file