From baf5002b2ce3abdaae4551b3021c3a1d22fe7fc4 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 10 Jun 2025 19:50:15 +0000 Subject: [PATCH] Merged in feature/exhibit-linking (pull request #564) Feature/exhibit linking * Initialize new exhibit header prompt * Save * Merge branch 'main' into feature/exhibit-linking * update prompt * cleanup return type * remove test * Test fixes * Fix test * Add tests * Merge branch 'main' into feature/exhibit-linking * Merge branch 'main' into feature/exhibit-linking * Remove test notebooks * Delete test notebook * Save * Merge branch 'main' into feature/exhibit-linking * try except for exhbit page finding * Prep for merge * update test * Remove test * Merge branch 'main' into feature/exhibit-linking * update unit test * Fix unit test * replace try except * Update page sort * Add docstring Approved-by: Alex Galarce --- .../src/investment/file_processing.py | 8 +- .../src/investment/one_to_n_funcs.py | 9 - fieldExtraction/src/investment/preprocess.py | 4 +- fieldExtraction/src/preprocessing_funcs.py | 121 +- .../src/prompts/investment_prompts.py | 81 +- .../methodology_breakout_issue_fixes.ipynb | 373 ------ .../methodology_primary_row_comparison.ipynb | 70 -- .../testbed/test_methodology_breakout.ipynb | 375 ------ .../src/testbed/test_one_to_n.ipynb | 1051 ----------------- fieldExtraction/tests/one_to_n_test.py | 45 - .../tests/preprocessing_funcs_test.py | 378 +----- 11 files changed, 179 insertions(+), 2336 deletions(-) delete mode 100644 fieldExtraction/src/testbed/methodology_breakout_issue_fixes.ipynb delete mode 100644 fieldExtraction/src/testbed/methodology_primary_row_comparison.ipynb delete mode 100644 fieldExtraction/src/testbed/test_methodology_breakout.ipynb delete mode 100644 fieldExtraction/src/testbed/test_one_to_n.ipynb diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 9fd7099..7934ae1 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -121,8 +121,14 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase for exhibit_page in exhibit_dict.keys(): exhibit_text = exhibit_dict[exhibit_page] - exhibit_header = all_exhibit_headers[exhibit_page] + if exhibit_page in all_exhibit_headers.keys(): + exhibit_header = all_exhibit_headers[exhibit_page] + elif ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys(): + exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])] + else: + exhibit_header = "No Header Found" + ################## INITIALIZE FIELDS ################## reimbursement_level_fields = FieldSet(relationship="one_to_n", field_type="reimbursement_level", file_path=FIELD_JSON_PATH) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 0a1d659..84f906d 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -463,18 +463,9 @@ def reimbursement_level( if reimbursement_primary_answers: # If any reimbursements found carveout_answers = get_special_cases(reimbursement_primary_answers, filename) - - - # # Later, we will only run methodology breakout for those without certain carveout indicators. We will run carveout-specific breakouts for the others - # # not required now as get_special_cases will already have run the methodology breakout for all cases - # methodology_breakout_answers = get_methodology_breakout( - # carveout_answers, filename - # ) - code_breakout_answers = code_funcs.get_code_breakout( carveout_answers, filename, all_dataset ) - # print(f"{datetime_str()} code_breakout_answers complete for exhibit page {exhibit_page} of {filename}...") return code_breakout_answers else: diff --git a/fieldExtraction/src/investment/preprocess.py b/fieldExtraction/src/investment/preprocess.py index bdcc622..fbf170d 100644 --- a/fieldExtraction/src/investment/preprocess.py +++ b/fieldExtraction/src/investment/preprocess.py @@ -60,10 +60,10 @@ def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]: - dict: A dictionary where keys are exhibit page numbers and values are the exhibit headers. """ - exhibit_pages = preprocessing_funcs.get_exhibit_pages(text_dict, filename) + exhibit_pages, all_exhibit_headers = preprocessing_funcs.get_exhibit_pages(text_dict, filename) + exhibit_pages, all_exhibit_headers = preprocessing_funcs.link_exhibit_pages(all_exhibit_headers, filename) exhibit_chunk_mapping = preprocessing_funcs.chunk_by_exhibit(text_dict, exhibit_pages) exhibit_dict = preprocessing_funcs.get_exhibit_dict(text_dict, exhibit_chunk_mapping) - all_exhibit_headers = preprocessing_funcs.get_exhibit_headers(exhibit_dict=exhibit_dict, filename=filename) return exhibit_dict, all_exhibit_headers def one_to_one_smart_chunking(text_dict, contract_text, one_to_one_fields): diff --git a/fieldExtraction/src/preprocessing_funcs.py b/fieldExtraction/src/preprocessing_funcs.py index 89ac431..4c1178b 100644 --- a/fieldExtraction/src/preprocessing_funcs.py +++ b/fieldExtraction/src/preprocessing_funcs.py @@ -211,67 +211,108 @@ def filter_quick_review(text_dict): or "COVER SHEET" in page_text.upper() } -def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> list[str]: - """Extract beginning-of-exhibit pages from a contract using LLM-based detection. - - This function identifies pages that start new exhibits by checking each page (or - subpage) for exhibit headers using an LLM prompt. It handles both regular pages - (e.g., "5") and subpages created by table splitting (e.g., "5.1", "5.2"). - - Key behaviors: - - The first page of the contract is always included in the exhibit pages. - - For base pages with subpages, if ANY subpage contains an exhibit header, - the FIRST subpage is added to the exhibit list - - Each base page number is only processed once to avoid duplicates. - - Args: - text_dict (dict[str, str]): Dictionary keyed by string-formatted page numbers and valued by page text - Page numbers may include subpages (e.g., "5.1", "5.2") - filename (str): Filename of the contract (for tracking purposes) - - Returns: - list[str]: Ordered list of page numbers that start new exhibits. May include - subpage numbers (e.g., ["1", "5.1", "5.2"]) depending on table splitting. - - Example: - Given pages ["1", "2", "5.0", "5.1", "5.2", "7"] where "5.1" contains an exhibit header: - Returns ["1", "5.1", "7"] (assuming pages 1 and 7 also have exhibit headers) +def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> tuple[list[str], dict[str, str]]: """ + Identify exhibit pages from the text dictionary. + This function processes the text dictionary to identify pages that are considered exhibits. + It uses the first page as an exhibit and checks subsequent pages for exhibit headers. + If a page is identified as an exhibit, it is added to the exhibit pages list and its header is stored in the exhibit header dictionary. + If a page does not have a header or is not an exhibit, it is skipped. + Args: + text_dict (dict[str, str]): A dictionary where keys are page numbers (as strings) and values are the text of those pages. + filename (str): The name of the file being processed, used for logging and LLM invocation. + Returns: + tuple[list[str], dict[str, str]]: A tuple containing: + - A list of page numbers (as strings) that are identified as exhibit pages. + - A dictionary where keys are page numbers and values are the headers of those exhibits. + """ + sorted_pages = sorted(text_dict.keys(), key=string_utils.page_key_sort) exhibit_pages = [] + exhibit_header_dict = {} first_page = True - # Sort pages to handle subpages properly (5.0, 5.1, 5.2, etc.) - # TODO: augment `string_utils.page_key_sort` to handle subpages - sorted_pages = sorted(text_dict.keys(), key=lambda x: ( - int(x.split('.')[0]), - float(x.split('.')[1]) if '.' in x else 0) - ) - for page_num in sorted_pages: - + page_content = text_dict[page_num] # Always include the first page as an exhibit if first_page: exhibit_pages.append(page_num) + exhibit_header_dict[page_num] = "" is_exhibit = True first_page = False else: if "." not in page_num or ".0" in page_num: - page_content = text_dict[page_num] - prompt = investment_prompts.EXHIBIT_CHECK(page_content[0:200]) + prompt = investment_prompts.EXHIBIT_HEADER(page_content[0:400]) claude_answer_raw = llm_utils.invoke_claude( prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=150 ) claude_answer_extracted = string_utils.extract_text_from_delimiters( claude_answer_raw, Delimiter.PIPE ) - if "Y" in claude_answer_extracted: - exhibit_pages.append(page_num) - is_exhibit = True - else: + if "N/A" in claude_answer_extracted: is_exhibit = False + else: + exhibit_pages.append(page_num) + exhibit_header_dict[page_num] = claude_answer_extracted + is_exhibit = True elif is_exhibit==True: exhibit_pages.append(page_num) - return exhibit_pages + return exhibit_pages, exhibit_header_dict + + +def link_exhibit_pages(all_exhibit_headers, filename): + """ + Filters exhibit pages to include only those with meaningfully different headers. + This function processes exhibit headers sequentially and uses literal comparison and + LLM-based semantic comparison to determine when exhibits represent continuation pages + vs. distinct exhibits. Continuation pages are filtered out of the headers list and + mapping, and when used downstream are included in their parent exhibit's text chunk. + Links exhibit pages based on their headers. If the header of a page is identical (either literally or semantically) to the previous page's header, it is not added to the final list of exhibit pages. + This function will link contiguous header pages together, so that the final list of exhibit pages will only contain pages that are meaningfully different from the previous page. E.g. "1" with header {"Exhibit 1"} and "2" with header {"Continued exhibit 1"} will be linked together, but "3" with header {"Exhibit 2"} will not be linked to either of them. + Args: + all_exhibit_headers (dict[str, str]): A dictionary mapping page numbers (as strings) to their exhibit headers. Expected to be in chronological order + filename (str): The name of the file being processed, used for logging and LLM invocation. + Returns: + tuple[list[str], dict[str, str]]: A tuple containing: + - A list of page numbers (as strings) that are identified as exhibit pages. + - A dictionary where keys are page numbers and values are the headers of those exhibits. + Example: + Input: {"1": "Exhibit A", "2": "Exhibit A (continued)", "3": "Exhibit B"} + Output (["1", "3"], {"1": "Exhibit A", "3": "Exhibit B"}) + Notes: + - First exhibit page is always included. + - Literal header comparison is performed first to optimize LLM usage and latency. + - Uses `EXHIBIT_LINKAGE` prompt for semantic similarity assessment + - Expects LLM to return "Y" or "N" enclosed in pipe delimiters + - Pages with non-distinct headers are excluded from the final results. + """ + first_exhibit = True + final_exhibit_pages = [] + final_exhibit_headers = {} + for page_num, page_header in all_exhibit_headers.items(): + if first_exhibit: + exhibit_is_different = "N" + previous_header = page_header + final_exhibit_pages.append(page_num) + final_exhibit_headers[page_num] = page_header + first_exhibit = False + else: + # If literally identical, skip prompt + if page_header.upper().strip() == previous_header.upper().strip(): + exhibit_is_different = "N" + else: + prompt = investment_prompts.EXHIBIT_LINKAGE(page_header, previous_header) + claude_answer_raw = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename) + exhibit_is_different = string_utils.extract_text_from_delimiters(claude_answer_raw, Delimiter.PIPE) + + # If the prompt has identified that the current exhibit is meaningfully different than the previous + if "Y" in exhibit_is_different: + final_exhibit_pages.append(page_num) + final_exhibit_headers[page_num] = page_header + + previous_header = page_header + + return final_exhibit_pages, final_exhibit_headers + def chunk_by_exhibit(text_dict: dict, exhibit_pages: list diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 3a308ef..2875862 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -256,55 +256,78 @@ class FieldSet: ################################## PROMPT-TEMPLATES ################################# ##################################################################################### -def EXHIBIT_CHECK(page_start): - return f"""Analyze the following contract excerpt and determine if it contains an exhibit/section header. - -Look for these patterns at the START of the text: -- EXHIBIT [letter/number] -- ATTACHMENT [letter/number] -- SCHEDULE [letter/number] -- ARTICLE [letter/number] -- AMENDMENT [letter/number] -- ADDENDUM [letter/number] - -The header should be the FIRST substantive content, regardless of what follows. - -IGNORE: -- References within sentences (e.g., "see Exhibit A") -- Legal citations (e.g., "Section 85.113") -- Numbered clauses (e.g., "1.20", "7.1") - -Return |Y| if there's a header at the start, |N| if not. - -Text to analyze: -{page_start} - -Answer: |Your decision|""" - def EXHIBIT_HEADER(context): return f"""Analyze the following contract excerpt and extract the full header found. -You must capture the complete hierarchy of document identifiers present in the text. Include all of the following when present: +If an Exhibit Header is present, you must capture the complete hierarchy of document identifiers present in the text. Include all of the following when present: * Full Attachment names and numbers (e.g., "Attachment C: Commercial-Exchange") * Complete Exhibit numbers/letters with full titles (e.g., "Exhibit 1 - Medicare") -* Any other relevant subtitles and section identifiers, including but not limited to 'Article', 'Amendment', 'Schedule', 'Addendum', or similar. +* Any other relevant subtitles and section identifiers, including but not limited to 'Article', 'Amendment', 'Schedule', 'Addendum', 'Appendix', or similar. * Any words in the header that may specify what type of information is contained, such as "Compensation Schedule" or "Definitions". * Provider/Entity names when listed with exhibit information +If there is no Exhibit Header present, return |N/A|. Do not fill in values from the examples above, these are only examples. + - Rules for inclusion: * Do NOT abbreviate or summarize any part of the exhibit names * Do NOT include page numbers * Do NOT cherry-pick only certain parts - capture the complete exhibit hierarchy * Include ALL text that appears to be part of the exhibit/attachment header * Keep exact capitalization and formatting as shown in document -- Write 'N/A' only if NO exhibit, attachment, amendment, or other identifiers are found + * Note that docusign IDs and other metadata do not constitute a header and should be excluded Here is the text to analyze: {context.replace('"', "'")} Before returning any output, ensure that all instructions for inclusion were followed. -Enclose your final answer in |pipes|. +Briefly explain your answer, then enclose your final answer in |pipes|. +""" + +def EXHIBIT_LINKAGE(header1, header2): + return f""" + +[INSTRUCTIONS] +* Determine whether the two Headers are meaningfully different from each other. +* The headers may refer to Attachment, Exhibit, Schedule, Addendum, or similar. If the highest-level number is identical, then the Headers ARE meaningfully different. For example, "Exhibit B-1" and "Exhibit B-2" are meaningfully different because they are both Exhibit B but have different sub-exhibits. +* If the headers have the same numbers, but different descriptors, they are NOT meaningfully different. For example, "Attachment A" and "Attachment A: Professional" are NOT meaningfully different because they are both attachment A. +* If one header is a continuation of the other (e.g. "Continued", "CONT'D"), they are NOT meaningfully different. +* Return |Y| if the two Headers are meaningfully different. Return |N| if they are not. + +[CONTEXT] +Header 1: +"{header1}" + +Header 2: +"{header2}" + +Briefly explain your answer, then enclose your final answer in |pipes|. +""" + + +def IDENTIFY_REIMBURSEMENT_EXHIBITS_PROMPT(exhibit_headers: dict, yaml_output: bool = False): + formatted_headers = [f"Page {page}: {header}" for page, header in exhibit_headers.items()] + return f"""Review these exhibit headers and identify which ones are likely to contain reimbursement methodologies, payment terms, or compensation schedules. + +Consider both explicit and implicit indicators of reimbursement information: + +EXPLICIT INDICATORS: +- Exhibits that have "Compensation Schedule", "Reimbursement Schedule", "Rates", or similar in the title. The title must somehow explicitly indicate that the Exhibit contains payment terms. + +IMPLICIT INDICATORS: +- Sections titled "COVERED SERVICES" for specific insurance products typically include payment terms because they define what services are reimbursable and how +- Attachments specifying plan types (Medicare, Medicaid, Commercial, Marketplace) usually contain reimbursement details +- References to specific programs (Medicare Advantage, CHIP) that have distinct payment methodologies +- Attachments that appear to define specific services or service categories + +For each header, return 'EXPLICIT' if there is an EXPLICIT INDICATOR, 'IMPLICIT' if there is an IMPLICIT INDICATOR, or 'NO' if the header indicates that the exhibit is unlikely to contain reimbursement information. + +Headers to review: +{"\n".join(formatted_headers)} + +Return your answer as a {"JSON object" if not yaml_output else "YAML list"} where keys are PAGE NUMBERS and values are 'EXPLICIT', 'IMPLICIT', or 'NO'. +Feel free to explain your reasoning, but place your final {"JSON" if not yaml_output else "YAML"} output between triple backticks. +If you're uncertain about a header, err on the side of inclusion ('IMPLICIT'). """ diff --git a/fieldExtraction/src/testbed/methodology_breakout_issue_fixes.ipynb b/fieldExtraction/src/testbed/methodology_breakout_issue_fixes.ipynb deleted file mode 100644 index e5d2494..0000000 --- a/fieldExtraction/src/testbed/methodology_breakout_issue_fixes.ipynb +++ /dev/null @@ -1,373 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/home/maamseek/doczy.ai/fieldExtraction\n" - ] - } - ], - "source": [ - "import pandas as pd\n", - "import os\n", - "# import sys\n", - "# sys.path.append('../../') # Update this path to the correct path where the 'src' module is located\n", - "\n", - "os.chdir('../../')\n", - "print(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "doczy-output exists\n" - ] - } - ], - "source": [ - "from src import config\n", - "\n", - "config.RUN_MODE = \"local\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3180, 146)\n", - "(1, 146)\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
FILE_NAMESERVICE_TERMREIMB_TERM
2107AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txtSpecial Care Broward Health North, Broward Hea...$ 1,875 Per Diem
\n", - "
" - ], - "text/plain": [ - " FILE_NAME \\\n", - "2107 AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt \n", - "\n", - " SERVICE_TERM REIMB_TERM \n", - "2107 Special Care Broward Health North, Broward Hea... $ 1,875 Per Diem " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "labels_df = pd.read_csv('src/testbed/v9_DoczyAI_all_test_bed_LIVE_5.23.csv',encoding='utf-8')\n", - "print(labels_df.shape)\n", - "# labels_df = labels_df[labels_df[\n", - "# 'FILE_NAME'].isin([\n", - "# # \"00-125434-CENTRAL MS DIAGNOSTIC, LLC-ICMPROVIDERAGREEMENT_78285.TXT\",\n", - "# \"02-0677066-EYEMASTERS-ICMPROVIDERAGREEMENT_64220_2.TXT\",\n", - "# # \"050540697 DEEPAK NANDA 2017 PHSP MU.TXT\",\n", - "# # \"06-1798267-ANTHONY W. LE, A PROFESSIONAL CORP-ICMPROVIDERAGREEMENT_316844.TXT\",\n", - "# # \"133923495 CAIPA 2015 PHSP.TXT\",\n", - "# ])]\n", - "# print(labels_df.shape)\n", - "\n", - "labels_df = labels_df[labels_df[\n", - " 'REIMB_TERM'].isin([\n", - " # \"THE LESSER OF BILLED OR ALLOWABLE. WILL BE PAID WITH A CONVERSION FACTOR OF $30.72.\",\n", - " # \"THE LESSER OF BILLED OR ALLOWABLE. THE PROFESSIONAL SERVICES FEE SCHEDULE FOR THE MEDICAID LINE OF BUSINESS IS BASED PRIMARILY ON THE CMS TRANSITIONAL FACILITY/NON-FACILITY RELATIVE VALUE UNIT SCALE (\\\"RVU\\\") AND METHODOLOGY AS DEFINED AND INSTRUCTED BY THE FEDERAL REGISTER IN PRINT AS OF DECEMBER 31, 2013. COVERED SERVICES WILL BE CALCULATED USING THE CONVERSION FACTORS BELOW. ANCILLARY PROVIDER SERVICES $26.00 CONVERSION FACTOR*. ANESTHESIA $21.20 CONVERSION FACTOR**.\",\n", - " \"$ 1,875 Per Diem\",\n", - " # \"Physician Group shall be compensated for anesthesiology services which are Covered Services at the lesser of (a) $39.00 per unit value in accordance with the American Society of Anesthesiology (ASA) unit scale, or (b) 75% of the Physician Group's usual billed charges.\",\n", - " # \"80% of the AHCCCS Physician Fee For Service rates prevailing as of the date of service Contrast material/isotopes for PET scans $200 per PET scan\",\n", - " # \"PHYSICIAN GROUP SHALL BE COMPENSATED FOR ANESTHESIOLOGY SERVICES WHICH ARE COVERED SERVICES AT THE LESSER OF (A) $39.00 PER UNIT VALUE IN ACCORDANCE WITH THE AMERICAN SOCIETY OF ANESTHESIOLOGY (ASA) UNIT SCALE, OR (B) 75% OF THE PHYSICIAN GROUP'S USUAL BILLED CHARGES.\",\n", - " # , \"Health Net will offer $1,000,000 incremental reimbursement for the Medi-Cal line of business each year of the two year renewal effective 5/1/18 and 5/1/19, resulting in an additional $1M reimbursement from 5/1/18 through 4/30/19 and an additional $2M reimbursement from 5/1/19 through 4/30/20.\"\n", - " ])]\n", - "print(labels_df.shape)\n", - "\n", - "labels_df= labels_df.rename(columns = {\"CONTRACT_SERVICE_CD_OR_DESC\": \"SERVICE_TERM\",\n", - " # \"CONTRACT_CARVEOUT_IND\": \"CARVEOUT_IND\",\n", - " \"CONTRACT_REIMBURSEMENT_METHOD\": \"REIMB_TERM\"})\n", - "\n", - "labels_df_for_prediction = labels_df[[\"FILE_NAME\", \"SERVICE_TERM\"\n", - " # , \"CARVEOUT_IND\"\n", - " , \"REIMB_TERM\"]]\n", - "labels_df_unique_rows = labels_df_for_prediction.drop_duplicates()\n", - "labels_df_unique_rows" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'FILE_NAME': 'AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt', 'SERVICE_TERM': 'Special Care Broward Health North, Broward Health Medical Center | 169', 'REIMB_TERM': '$ 1,875 Per Diem', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N'}\n", - "[{'FILE_NAME': 'AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt', 'SERVICE_TERM': 'Special Care Broward Health North, Broward Health Medical Center | 169', 'REIMB_TERM': '$ 1,875 Per Diem', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'N', 'GREATER_OF_IND': 'N', 'AARETE_DERIVED_REIMB_METHOD': 'Flat Rate', 'UNIT_OF_MEASURE': 'Per Day', 'REIMB_FEE_RATE': 1875, 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'NOT_TO_EXCEED_IND': 'N', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}]\n" - ] - } - ], - "source": [ - "# from src.investment.one_to_n_funcs import get_methodology_breakout\n", - "\n", - "# reimbursement_primary_answers = labels_df_unique_rows[[\"FILE_NAME\", \"SERVICE_TERM\", \"CARVEOUT_IND\", \"REIMB_TERM\"]].to_dict(orient='records')\n", - "# predictions_list = []\n", - "\n", - "# for reimbursement_primary in reimbursement_primary_answers:\n", - "# filename = reimbursement_primary[\"FILE_NAME\"]\n", - "# methodology_breakout_answers = get_methodology_breakout([reimbursement_primary], filename)\n", - "# print(reimbursement_primary)\n", - "# print(methodology_breakout_answers)\n", - "# predictions_list.extend(methodology_breakout_answers)\n", - "\n", - "# predictions_df = pd.DataFrame(predictions_list)\n", - "\n", - "from src.investment.one_to_n_funcs import get_special_cases\n", - "\n", - "reimbursement_primary_answers = labels_df_unique_rows[[\"FILE_NAME\", \"SERVICE_TERM\"\n", - " # , \"CARVEOUT_IND\"\n", - " , \"REIMB_TERM\"]].to_dict(orient='records')\n", - "predictions_list = []\n", - "\n", - "for reimbursement_primary in reimbursement_primary_answers:\n", - " filename = reimbursement_primary[\"FILE_NAME\"]\n", - " carveout_answers = get_special_cases([reimbursement_primary], filename)\n", - " print(reimbursement_primary)\n", - " print(carveout_answers)\n", - " predictions_list.extend(carveout_answers)\n", - "\n", - "predictions_df = pd.DataFrame(predictions_list)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "import src.investment.investment_postprocessing_funcs\n", - "\n", - "predictions_df1 = src.investment.investment_postprocessing_funcs.standardize_reimb_method_and_fee_schedule(predictions_df)\n", - "predictions_df1 = src.investment.investment_postprocessing_funcs.remove_update_reimbursement(predictions_df1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(1, 17)" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "predictions_df1 = predictions_df1.replace('N/A', '').fillna('')\n", - "predictions_df1.to_csv('predictions.csv', index=False)\n", - "predictions_df1.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
FILE_NAMESERVICE_TERMREIMB_TERMCARVEOUT_CDCARVEOUT_INDDEFAULT_INDLESSER_OF_INDGREATER_OF_INDAARETE_DERIVED_REIMB_METHODUNIT_OF_MEASUREREIMB_FEE_RATEREIMB_PCT_RATEREIMB_CONVERSION_FACTORNOT_TO_EXCEED_INDFEE_SCHEDULEAARETE_DERIVED_FEE_SCHEDULEAARETE_DERIVED_FEE_SCHEDULE_VERSION
0AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txtSpecial Care Broward Health North, Broward Hea...$ 1,875 Per DiemPAYMENT_CARVEOUTYNNNFlat RatePer Day1875N/AN/ANN/AN/AN/A
\n", - "
" - ], - "text/plain": [ - " FILE_NAME \\\n", - "0 AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt \n", - "\n", - " SERVICE_TERM REIMB_TERM \\\n", - "0 Special Care Broward Health North, Broward Hea... $ 1,875 Per Diem \n", - "\n", - " CARVEOUT_CD CARVEOUT_IND DEFAULT_IND LESSER_OF_IND GREATER_OF_IND \\\n", - "0 PAYMENT_CARVEOUT Y N N N \n", - "\n", - " AARETE_DERIVED_REIMB_METHOD UNIT_OF_MEASURE REIMB_FEE_RATE REIMB_PCT_RATE \\\n", - "0 Flat Rate Per Day 1875 N/A \n", - "\n", - " REIMB_CONVERSION_FACTOR NOT_TO_EXCEED_IND FEE_SCHEDULE \\\n", - "0 N/A N N/A \n", - "\n", - " AARETE_DERIVED_FEE_SCHEDULE AARETE_DERIVED_FEE_SCHEDULE_VERSION \n", - "0 N/A N/A " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "predictions_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fieldExtraction/src/testbed/methodology_primary_row_comparison.ipynb b/fieldExtraction/src/testbed/methodology_primary_row_comparison.ipynb deleted file mode 100644 index b8f82f2..0000000 --- a/fieldExtraction/src/testbed/methodology_primary_row_comparison.ipynb +++ /dev/null @@ -1,70 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "9d212511", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import os\n", - "os.chdir(\"../..\")\n", - "from src.testbed.testbed_utils import compare_term_extraction_results\n", - "\n", - "predictions_df = pd.read_csv('src/testbed/inv-test-0513-new-reimb-primary-prompt-RESULTS.csv')\n", - "gt_df = pd.read_excel('src/testbed/v6_DoczyAI_all_test_bed_LIVE_5.6.25.xlsx', sheet_name='all_test_bed_labels')\n", - "\n", - "common_files = set(predictions_df['FILE_NAME']).intersection(set(gt_df['FILE_NAME']))\n", - "print(f\"Number of common files: {len(common_files)}\")\n", - "\n", - "# Compare rows for each common file\n", - "for file_name in common_files:\n", - " result = compare_term_extraction_results(file_name, predictions_df, gt_df)\n", - " if \"error\" in result:\n", - " print(f\"Error for file {file_name}: {result['error']}\")\n", - " else:\n", - " print(f\"File: {result['file_name']}\")\n", - " print(f\"Unique pairs in predictions: {result['pred_count']}\")\n", - " print(f\"Unique pairs in ground truth: {result['testbed_count']}\")\n", - " print(f\"Common pairs: {len(result['common_pairs'])}\")\n", - " print(f\"Missing in predictions (false negatives): {len(result['false_negatives'])}\")\n", - " print(f\"Missing in ground truth (false positives): {len(result['false_positives'])}\")\n", - " print(f\"False Negatives:\")\n", - " for pair in result['false_negatives']:\n", - " print(f\" {pair}\")\n", - " print(f\"False Positives:\")\n", - " for pair in result['false_positives']:\n", - " print(f\" {pair}\")\n", - " print(\"-\" * 50)" - ] - }, - { - "cell_type": "markdown", - "id": "2e0453f9", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.7" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/fieldExtraction/src/testbed/test_methodology_breakout.ipynb b/fieldExtraction/src/testbed/test_methodology_breakout.ipynb deleted file mode 100644 index b709bc8..0000000 --- a/fieldExtraction/src/testbed/test_methodology_breakout.ipynb +++ /dev/null @@ -1,375 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# !pip install rapidfuzz" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from one_to_n_comparison import evaluate_one_to_n_fields_fuzzy_matching_field_by_field\n", - "import pandas as pd\n", - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# import sys\n", - "# sys.path.append('../../') # Update this path to the correct path where the 'src' module is located\n", - "\n", - "os.chdir('../../')\n", - "print(os.getcwd())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from src import config\n", - "\n", - "config.RUN_MODE = \"local\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "MATCH_THRESHOLD = 90\n", - "# This is the labeled data from sharepoint. It can be found at\n", - "# OneDrive - AArete, LLC\\investment_samples\\Test Bed Files\n", - "# You'll need access to the `investment_samples` sharepoint folder\n", - "file_path = \"/root/doczy.ai/fieldExtraction/src/testbed/Reimbursement Field Files/\"\n", - "file_path = \"Reimbursement Field Files/\"\n", - "labels_files = os.listdir(file_path)\n", - "\n", - "# fields = [\"SERVICE_TERM\", \"REIMB_TERM\"] # comparison fields\n", - "fields = [\"AARETE_DERIVED_REIMB_METHOD\", \n", - " \"REIMB_FEE_RATE\", \n", - " # \"REIMB_PCT_RATE\",\n", - " \"FEE_SCHEDULE\",\n", - " \"AARETE_DERIVED_FEE_SCHEDULE\",\n", - " \"AARETE_DERIVED_FEE_SCHEDULE_VERSION\"\n", - " ]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# make list of dataframes from labels_files\n", - "# Track where labels are coming from\n", - "labels_dfs = []\n", - "for filename in labels_files:\n", - " print(f\"Reading {filename}\")\n", - " if filename == \"DocziAI_2-12.xlsx\":\n", - " df_extract = pd.read_excel(file_path + filename, sheet_name=\"doczy-ai-test-20240211\", skiprows=3)\n", - " df_extract['data_source'] = filename\n", - " labels_dfs.append(df_extract)\n", - " elif filename.endswith(\".xlsx\"):\n", - " df_extract = pd.read_excel(file_path + filename)\n", - " df_extract['data_source'] = filename\n", - " labels_dfs.append(df_extract)\n", - " elif filename.endswith(\".csv\"):\n", - " df_extract = pd.read_csv(file_path + filename)\n", - " df_extract['data_source'] = filename\n", - " labels_dfs.append(df_extract)\n", - "\n", - "print(f\"Extracted {len(labels_dfs)} dataframes.\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "labels_df = pd.concat(labels_dfs, ignore_index=True)\n", - "# This is any consolidated output from the model; they can be found in S3\n", - "# predictions_df = pd.read_csv('inv-test-0305-PM-RESULTS.csv')\n", - "\n", - "# For all float and int columns, standardize to Int64 and handle NaN values (optional)\n", - "# labels_df = labels_df.astype({col: 'Int64' for col in labels_df.select_dtypes(include=['float', 'int']).columns})\n", - "# predictions_df = predictions_df.astype({col: 'Int64' for col in predictions_df.select_dtypes(include=['float', 'int']).columns})\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "labels_df[labels_df['data_source'] == 'DocziAI_2-12.xlsx']\n", - "\n", - "for fn in labels_df['CONTRACT_FILE_NAME'].unique():\n", - " n_sources = len(labels_df[labels_df['CONTRACT_FILE_NAME'] == fn]['data_source'].unique())\n", - " if n_sources != 1:\n", - " # Remove rows from docziAI that have a separate data source\n", - " labels_df = labels_df[~((labels_df['CONTRACT_FILE_NAME'] == fn) & (labels_df['data_source'] == 'DocziAI_2-12.xlsx'))]\n", - " print(fn, n_sources)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "unique_contracts_labels = labels_df['CONTRACT_FILE_NAME'].nunique()\n", - "print(f\"There are {unique_contracts_labels} unique contract file names in the labels dataframe.\")\n", - "\n", - "labels_df.to_csv('labels_df.csv', index=False)\n", - "\n", - "# unique_contracts_predictions = predictions_df['CONTRACT_FILE_NAME'].nunique()\n", - "# print(f\"There are {unique_contracts_predictions} unique contract file names in the predictions dataframe.\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "labels_df = pd.read_csv('labels_df.csv')\n", - "\n", - "labels_df = labels_df[['CONTRACT_FILE_NAME',\n", - " 'SERVICE_TERM',\n", - " 'REIMB_TERM',\n", - " 'CARVEOUT_IND',\n", - " 'LESSER_OF_IND',\n", - " 'GREATER_OF_IND',\n", - " 'AARETE_DERIVED_REIMB_METHOD',\n", - " 'REIMB_FEE_RATE',\n", - " 'REIMB_PCT_RATE',\n", - " 'NOT_TO_EXCEED_IND',\n", - " 'DEFAULT_IND',\n", - " 'FEE_SCHEDULE',\n", - " 'AARETE_DERIVED_FEE_SCHEDULE',\n", - " 'AARETE_DERIVED_FEE_SCHEDULE_VERSION',\n", - "# 'FEE_SCHEDULE_VERSION'\n", - " ]]\n", - "labels_df = labels_df.drop_duplicates()\n", - "labels_df = labels_df.replace('N/A', '').fillna('')\n", - "labels_df['REIMB_PCT_RATE'] = labels_df['REIMB_PCT_RATE'].replace('', 100)\n", - "# labels_df[\"CONTRACT_FILE_NAME\"].unique()\n", - "labels_df = labels_df[labels_df[\n", - " 'CONTRACT_FILE_NAME'].isin([\"95-3372911; Multiple-Davita Healthcare Partners, Inc.-ICMProviderAgreement_138658.txt\",\n", - " \"Custom_2018-09-15 COMM Agmt FE - Bandon Community Health Center dba Coast Community Health Center.txt\",\n", - " \"Progressive Women's Health, PLLC_Physician Agreement MU.txt\",\n", - " \"Akron General Health System_Eleventh Amendment_20171001.txt\",\n", - " \"2021-07-15 COMM Agmt FE - Samaritan Health Services MU.txt\",\n", - " \"2019 01 01 Gulf Coast Division (Agreement_STAR, CHIP, CHIP P, STAR_PLUS) MU.txt\",\n", - " \"2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof.txt\"])]\n", - "print(labels_df.shape)\n", - "labels_df.to_csv('labels_to_compare.csv', index=False)\n", - "labels_df = pd.read_csv('labels_to_compare.csv')\n", - "\n", - "labels_df_for_prediction = labels_df[[\"CONTRACT_FILE_NAME\", \"SERVICE_TERM\", \"CARVEOUT_IND\", \"REIMB_TERM\"]]\n", - "labels_df_unique_rows = labels_df_for_prediction.drop_duplicates()\n", - "labels_df_unique_rows.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from src.investment.one_to_n_funcs import get_methodology_breakout\n", - "\n", - "reimbursement_primary_answers = labels_df_unique_rows[[\"CONTRACT_FILE_NAME\", \"SERVICE_TERM\", \"CARVEOUT_IND\", \"REIMB_TERM\"]].to_dict(orient='records')\n", - "predictions_list = []\n", - "\n", - "for reimbursement_primary in reimbursement_primary_answers:\n", - " filename = reimbursement_primary[\"CONTRACT_FILE_NAME\"]\n", - " methodology_breakout_answers = get_methodology_breakout([reimbursement_primary], filename)\n", - " print(reimbursement_primary)\n", - " print(methodology_breakout_answers)\n", - " predictions_list.extend(methodology_breakout_answers)\n", - "\n", - "predictions_df = pd.DataFrame(predictions_list)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Remove rows where 'AARETE_DERIVED_REIMB_METHOD' value is \"Medicare Member Cost Share\"\n", - "predictions_df = predictions_df[predictions_df['AARETE_DERIVED_REIMB_METHOD'] != \"Medicare Member Cost Share\"]\n", - "\n", - "# Standardize 'AARETE_DERIVED_REIMB_METHOD' values\n", - "# Copy AWP, ASP, WAC to AARETE_DERIVED_FEE_SCHEDULE column for the corresponding row\n", - "medrx_fee_schedule_values = ['AWP', 'ASP', 'WAC']\n", - "predictions_df.loc[predictions_df['AARETE_DERIVED_REIMB_METHOD'].isin(medrx_fee_schedule_values), 'AARETE_DERIVED_FEE_SCHEDULE'] = predictions_df['AARETE_DERIVED_REIMB_METHOD']\n", - "\n", - "# Convert AARETE_DERIVED_REIMB_METHOD values [AWP, ASP, WAC] to 'MedRx'\n", - "predictions_df['AARETE_DERIVED_REIMB_METHOD'] = predictions_df['AARETE_DERIVED_REIMB_METHOD'].replace({'AWP': 'MedRx', 'ASP': 'MedRx', 'WAC': 'MedRx'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predictions_df = predictions_df.replace('N/A', '').fillna('')\n", - "predictions_df.to_csv('predictions.csv', index=False)\n", - "predictions_df = pd.read_csv('predictions.csv')\n", - "predictions_df.drop(\"FEE_SCHEDULE_VERSION\", axis=1, inplace=True)\n", - "\n", - "predictions_df.shape" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "precisions = []\n", - "recalls = []\n", - "accuracies = []\n", - "\n", - "comparisons = {}\n", - "\n", - "total_true_positives = 0\n", - "total_false_positives = 0\n", - "total_false_negatives = 0\n", - "\n", - "\n", - "for contract_file_name in labels_df[\"CONTRACT_FILE_NAME\"].unique():\n", - " print(f\"Processing {contract_file_name}\")\n", - " labels_df_contract = labels_df[\n", - " labels_df[\"CONTRACT_FILE_NAME\"] == contract_file_name\n", - " ]\n", - " predictions_df_contract = predictions_df[\n", - " predictions_df[\"CONTRACT_FILE_NAME\"] == contract_file_name\n", - " ]\n", - " if len(labels_df_contract) == 0:\n", - " print(f\"Contract {contract_file_name} not found in labels.\\n\")\n", - " continue\n", - " elif len(predictions_df_contract) == 0:\n", - " print(f\"Contract {contract_file_name} not found in predictions.\\n\")\n", - " continue\n", - "\n", - " # Full reimbursement primary\n", - " comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(\n", - " labels_df_contract,\n", - " predictions_df_contract,\n", - " \"CONTRACT_FILE_NAME\",\n", - " fields,\n", - " thresholds=MATCH_THRESHOLD\n", - " )\n", - " comparisons[contract_file_name] = comparison\n", - "\n", - " # comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(labels_df_contract, predictions_df_contract, \"CONTRACT_FILE_NAME\", ['REIMB_TERM', 'FEE_SCHEDULE'])\n", - "\n", - " precisions.append(comparison[\"precision\"])\n", - " recalls.append(comparison[\"recall\"])\n", - " accuracies.append(comparison[\"accuracy\"])\n", - "\n", - "\n", - " total_true_positives += comparison[\"true_positives\"]\n", - " total_false_positives += comparison[\"false_positives\"]\n", - " total_false_negatives += comparison[\"false_negatives\"]\n", - "\n", - " # contract file name is already printed above\n", - " print(\n", - " f\"false_negatives: {comparison['false_negatives']} | false_positives: {comparison['false_positives']} | true_positives: {comparison['true_positives']} | accuracy: {comparison['accuracy']} | precision: {comparison['precision']} | recall: {comparison['recall']} | f1_score: {comparison['f1_score']}\\n\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from statistics import median, mean\n", - "\n", - "print(f\"Median accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {median(accuracies):.4f}\")\n", - "print(f\"Median precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {median(precisions):.4f}\")\n", - "print(f\"Median recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {median(recalls):.4f}\")\n", - "print(f\"Mean accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {mean(accuracies):.4f}\")\n", - "print(f\"Mean precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {mean(precisions):.4f}\")\n", - "print(f\"Mean recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {mean(recalls):.4f}\")\n", - "print(f\"Grand accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {(total_true_positives) / (total_true_positives + total_false_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives}, total false negatives: {total_false_negatives})\")\n", - "print(f\"Grand precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_positives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives})\")\n", - "print(f\"Grand recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false negatives: {total_false_negatives})\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# comparisons['2001-11-01 California Emergency Physicians PSA_MU.txt']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# comparisons['02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fieldExtraction/src/testbed/test_one_to_n.ipynb b/fieldExtraction/src/testbed/test_one_to_n.ipynb deleted file mode 100644 index 62e834e..0000000 --- a/fieldExtraction/src/testbed/test_one_to_n.ipynb +++ /dev/null @@ -1,1051 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['/usr/lib/python312.zip', '/usr/lib/python3.12', '/usr/lib/python3.12/lib-dynload', '', '/home/kminhas/.cache/pypoetry/virtualenvs/doczy-smart-chunking-BxQLr-8H-py3.12/lib/python3.12/site-packages']\n" - ] - } - ], - "source": [ - "from one_to_n_comparison import evaluate_one_to_n_fields_fuzzy_matching_field_by_field\n", - "import pandas as pd\n", - "import os\n", - "import sys\n", - "\n", - "sys.path.append('/home/kminhas/doczy.ai/fieldExtraction/')\n", - "import src.utils.string_utils as string_utils" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "MATCH_THRESHOLD = 90\n", - "\n", - "fields = [\n", - " \"AARETE_DERIVED_PRODUCT\",\n", - " \"AARETE_DERIVED_LOB\",\n", - " \"AARETE_DERIVED_PROGRAM\",\n", - " \"AARETE_DERIVED_NETWORK\",\n", - " \"LESSER_OF_IND\",\n", - " \"GREATER_OF_IND\",\n", - " \"AARETE_DERIVED_REIMB_METHOD\",\n", - " \"UNIT_OF_MEASURE\",\n", - " \"REIMB_PCT_RATE\",\n", - " \"REIMB_FEE_RATE\",\n", - " \"NOT_TO_EXCEED_IND\",\n", - " \"AARETE_DERIVED_FEE_SCHEDULE\",\n", - " \"AARETE_DERIVED_FEE_SCHEDULE_VERSION\"] # comparison fields" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(3198, 165) (4513, 141)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/kminhas/.cache/pypoetry/virtualenvs/doczy-smart-chunking-BxQLr-8H-py3.12/lib/python3.12/site-packages/openpyxl/worksheet/_reader.py:329: UserWarning: Data Validation extension is not supported and will be removed\n", - " warn(msg)\n", - "/tmp/ipykernel_594582/483789969.py:2: DtypeWarning: Columns (5,9,11,12,15,18,28,108,109,113,114,116,127,135,140) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " predictions_df = pd.read_csv('../../inv-test-0416-RESULTS.csv')\n" - ] - } - ], - "source": [ - "\n", - "labels_df = pd.read_excel(\"../../Doczy-Test-Bed.xlsx\")\n", - "predictions_df = pd.read_csv('../../inv-test-0416-RESULTS.csv')\n", - "\n", - "# Remove any empty rows from both dfs\n", - "labels_df = labels_df.dropna(how='all')\n", - "predictions_df = predictions_df.dropna(how='all')\n", - "\n", - "print(labels_df.shape, predictions_df.shape)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "There are 70 unique contract file names in the labels dataframe.\n", - "There are 96 unique contract file names in the predictions dataframe.\n" - ] - } - ], - "source": [ - "unique_contracts_labels = labels_df['FILE_NAME'].nunique()\n", - "print(f\"There are {unique_contracts_labels} unique contract file names in the labels dataframe.\")\n", - "\n", - "unique_contracts_predictions = predictions_df['FILE_NAME'].nunique()\n", - "print(f\"There are {unique_contracts_predictions} unique contract file names in the predictions dataframe.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def match_rows(labels_df, predictions_df, N):\n", - " confusion_matrix = {field_name: {\"tp\": 0, \"fp\": 0} for field_name in fields}\n", - " \n", - " # Track which label rows have been matched\n", - " matched_label_indices = set()\n", - " available_predictions = set(predictions_df.index)\n", - "\n", - " for i in range(N):\n", - " num_columns_to_match = N - i\n", - "\n", - " # Only consider unmatched label rows\n", - " for label_idx, label_row in labels_df.iterrows():\n", - " if label_idx in matched_label_indices:\n", - " continue # Skip already matched label rows\n", - "\n", - " best_match_idx = None\n", - " max_matches = 0\n", - " best_matches = {}\n", - "\n", - " # Find the best match among available predictions\n", - " for prediction_idx in available_predictions:\n", - " prediction_row = predictions_df.loc[prediction_idx]\n", - " \n", - " # Count matching fields for this pair\n", - " field_counter_dict = {field_name: {\"tp\": 0} for field_name in fields}\n", - " num_matches = 0\n", - " \n", - " for field in fields:\n", - " if label_row[field] == prediction_row[field] or (string_utils.is_empty(label_row[field]) and string_utils.is_empty(prediction_row[field])):\n", - " num_matches += 1\n", - " field_counter_dict[field][\"tp\"] = 1\n", - " \n", - " if num_matches > max_matches:\n", - " max_matches = num_matches\n", - " best_match_idx = prediction_idx\n", - " best_matches = field_counter_dict\n", - " \n", - " # If we found a good enough match\n", - " if max_matches >= num_columns_to_match and best_match_idx is not None:\n", - " # Mark this label row as matched\n", - " matched_label_indices.add(label_idx)\n", - " # Remove the matched prediction row from available ones\n", - " available_predictions.remove(best_match_idx)\n", - " # Update the confusion matrix with the best match\n", - " for field in fields:\n", - " confusion_matrix[field][\"tp\"] += best_matches[field][\"tp\"]\n", - " \n", - " # Calculate FP and FN based on the number of matched fields\n", - " for field in fields:\n", - " # FN = number of label rows where this field wasn't matched\n", - " confusion_matrix[field]['fn'] = len(labels_df) - confusion_matrix[field]['tp']\n", - " # FP = number of prediction rows where this field wasn't matched\n", - " confusion_matrix[field]['fp'] = len(predictions_df) - confusion_matrix[field]['tp']\n", - " \n", - " return confusion_matrix" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\n", - "02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\n", - "No rows in either labels or predictions for file: 02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\n", - "050540697 Deepak Nanda 2017 PHSP MU.txt\n", - "06-1798267-Anthony W. Le, A Professional Corp-ICMProviderAgreement_316844.txt\n", - "133923495 CAIPA 2015 PHSP.txt\n", - "2001-11-01 California Emergency Physicians PSA_MU.txt\n", - "2013-09-01 Fresno Dental Surgery Center PPA_MU.txt\n", - "2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof MU.txt\n", - "2014-01-01 United Health Centers San Joaquin AMD MU.txt\n", - "20-1668371-Midwest Medical Equipment Solutions, Inc.-ICMProviderAgreementAmendment_166876.txt\n", - "2018 09 01 UTMB (Amendment 10 change in reimbursement) MU.txt\n", - "2019 01 01 Gulf Coast Division (Agreement_STAR, CHIP, CHIP P, STAR_PLUS) MU.txt\n", - "2019 06 01 Dr Raul Rivera & Associates (Amend 1)_MU.txt\n", - "2021-07-15 COMM Agmt FE - Samaritan Health Services MU.txt\n", - "2022 01 20 West Park Primary Care_Master (Master) PHY Executed.pdf.filepart MU.txt\n", - "33-0057155-Apria Healthcare, Inc.-ICMProviderAgreementAmendment_26762.txt\n", - "36-2902782-Circle Family HealthCare Network-ICMProviderAgreement_59631.txt\n", - "39-0806250-Divine Saviour Hospital-ICMProviderAgreement_97009.txt\n", - "429670586-Melissa Ann Clark-ICMProviderAgreement_128379_1.txt\n", - "46-0979580-Harris and Theodore Anesthesia Services, LLC-ICMProviderAgreement_313205.txt\n", - "5.2000_Rehab Designs of America_VendorAgreement MU.txt\n", - "6.2000_Omni Transport Systems_ServicesAgreement MU.txt\n", - "61-1400871-West Kentucky Dermatology PSC-ICMProviderAgreement_104221.txt\n", - "84-5179194-Heart and Soul Hospice LLC-ICMProviderAgreement_273737.txt\n", - "86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661.txt\n", - "91-1885077-COASTAL ANESTHESIOLOGY MEDICAL-ICMProviderAgreement_242069.txt\n", - "92-2764433-MAGNOLIA MOBILE HOME CARE LLC-ICMProviderAgreement_322270.txt\n", - "95-2648499-Mission Pathology Medical Associates-ICMProviderAgreement_254534.txt\n", - "95-3372911; Multiple-Davita Healthcare Partners, Inc.-ICMProviderAgreement_138658.txt\n", - "Adena Health System National Template Agreement C19460229AA.txt\n", - "AGMT - HSA - TEXAS HEALTH RESOURCES - 75-6001743 MU.txt\n", - "Akron General Health System_Eleventh Amendment_20171001.txt\n", - "AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt\n", - "Anc2_23-1943113_BayadaHHC_Amendment1_553403.txt\n", - "Anc8_Hanger Pros & Ortho West Contract.txt\n", - "Boilerplate_2020-01-01 COMM Amend FE (Beacon) - Portland Adventist Medical Center MU.txt\n", - "Boilerplate_AMD - MKPL - NBHD dba Broward Health - Copy_MU.txt\n", - "Boilerplate_Benjamin Bieber MD - Amendment 5.30.16_MU.txt\n", - "Boilerplate_Fac_Bronx Lebanon Hospital Center - Amendment 1.1.14 MU.txt\n", - "Boilerplate_PSA_Houston Eye Associates_76-0512625_05012007 MU.txt\n", - "BoilerplateAnc_Bentley Medical PLLC - Amendment 8.1.14_MU.txt\n", - "BSW _ Original Contract _ 1.1.2000 MU.txt\n", - "Cabell Huntington_OH MCD Compensation MU.txt\n", - "Community Health Network_Base Contract MU.txt\n", - "Custom_060121 HonorHealth and its Affiliates Care 1st Physician Agreement.txt\n", - "Custom_2009-08-10 Adventist - University Community Hopstial LTAC MU.txt\n", - "Custom_2012-10-01 AdvenHealth Amendment.txt\n", - "Custom_2018-09-15 COMM Agmt FE - Bandon Community Health Center dba Coast Community Health Center.txt\n", - "Custom_2018-09-15 COMM Amend FE Plaza Ambulatory Surgery Center, LLC MU.txt\n", - "Custom_HSA_EFF07012010_Driscoll Children's Hospital MU.txt\n", - "Custom_Parkview Health Systems-OH MP, IN MP-ID C15656222AA MU.txt\n", - "Custom_Prof_Anna Suponya MD PC - Agreement Provider Signed_MU.txt\n", - "Custom_PSA - Advanced Gastroenterology of Texas PLLC -Executed 47-4543923 MU.txt\n", - "Custom_PSA - TX - Fresenius BIOMEDICAL APPLICATIONS OF TEXAS INC - 11-2226275 - 06012010 (1) (2) MU.txt\n", - "Custom_SOUTHWEST BEHAVIORAL & HEALTH SERVICES INC.txt\n", - "CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\n", - "CustomFac_Benedictine Hospital - Amendment 1.1.16 (HBX,HARP,EPP) (REVISED).txt\n", - "Fac1_Bullhead City_Western Arizona Med Center - 860982071 MU.txt\n", - "Genesis_ICMProviderAgreement_ICMProviderAgreement_153537_5.txt\n", - "Hunt Regional Medical Center Executed Contract.txt\n", - "ICMProviderAgreement_AlisonUnitisLPCMH_227710_5.txt\n", - "ICMProviderAgreement_AzarEyeSurgeryCenter_232594_6.txt\n", - "ICMProviderAgreement_BanyanDelaware_210080_5.txt\n", - "ICMProviderAgreement_CorasWellnessAndBehavioralHealth_212482_8.txt\n", - "ICMProviderAgreement_ICMProviderAgreement_143342_10_Landmark of Midwest City Rehab and Nursing Center.txt\n", - "ICMProviderAgreement_ICMProviderAgreement_295973_15.txt\n", - "ICMProviderAgreement_NewCastleHealthandRehabilitationCenter_212261_13 MU.txt\n", - "Progressive Women's Health, PLLC_Physician Agreement MU.txt\n", - "Southeast Georgia Health System Inc_20170517_Dually Executed MU.txt\n", - "Tampa General Rehabilitation Hospital Agreement MU.txt\n" - ] - } - ], - "source": [ - "N = len(fields)\n", - "accuracies = []\n", - "\n", - "for file in labels_df['FILE_NAME'].unique():\n", - " print(file)\n", - " labels_df_file = labels_df[labels_df['FILE_NAME'] == file]\n", - " predictions_df_file = predictions_df[predictions_df['FILE_NAME'] == file]\n", - "\n", - " # Ensure there are rows in both dataframes\n", - " if labels_df_file.shape[0] == 0 or predictions_df_file.shape[0] == 0:\n", - " print(f\"No rows in either labels or predictions for file: {file}\")\n", - " continue\n", - " \n", - " confusion_matrix = match_rows(labels_df_file, predictions_df_file, N)\n", - "\n", - " confusion_matrix[\"Filename\"] = file\n", - " accuracies.append(confusion_matrix)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "def calculate_precision_recall(accuracies):\n", - " \"\"\"\n", - " Calculate precision and recall for each file and overall metrics for all fields.\n", - "\n", - " Args:\n", - " accuracies (list): A list of confusion matrices for each file.\n", - "\n", - " Returns:\n", - " tuple: (precision_df, recall_df, overall_metrics_df)\n", - " - precision_df: Precision for each field per file.\n", - " - recall_df: Recall for each field per file.\n", - " - overall_metrics_df: Overall precision, recall, and F1 score for all fields.\n", - " \"\"\"\n", - " precision_dicts, recall_dicts = [], []\n", - " overall_metrics = {field: {\"tp\": 0, \"fp\": 0, \"fn\": 0} for field in accuracies[0] if field != \"Filename\"}\n", - "\n", - " for confusion_matrix in accuracies:\n", - " filename = confusion_matrix[\"Filename\"]\n", - " precision_dict, recall_dict = {\"Filename\": filename}, {\"Filename\": filename}\n", - "\n", - " for field in confusion_matrix:\n", - " if field == \"Filename\":\n", - " continue\n", - "\n", - " tp = confusion_matrix[field][\"tp\"]\n", - " fp = confusion_matrix[field][\"fp\"]\n", - " fn = confusion_matrix[field][\"fn\"]\n", - "\n", - " # Update overall metrics\n", - " overall_metrics[field][\"tp\"] += tp\n", - " overall_metrics[field][\"fp\"] += fp\n", - " overall_metrics[field][\"fn\"] += fn\n", - "\n", - " # Calculate precision and recall\n", - " precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", - " recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", - "\n", - " precision_dict[field] = precision\n", - " recall_dict[field] = recall\n", - "\n", - " precision_dicts.append(precision_dict)\n", - " recall_dicts.append(recall_dict)\n", - "\n", - " # Calculate overall precision, recall, and F1 score for all fields\n", - " overall_metrics_list = []\n", - " for field, metrics in overall_metrics.items():\n", - " tp = metrics[\"tp\"]\n", - " fp = metrics[\"fp\"]\n", - " fn = metrics[\"fn\"]\n", - "\n", - " precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", - " recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", - " f1 = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n", - "\n", - " overall_metrics_list.append({\n", - " \"Field\": field,\n", - " \"tp\": tp,\n", - " \"fp\": fp,\n", - " \"fn\": fn,\n", - " \"prec\": precision,\n", - " \"rec\": recall,\n", - " \"f1\": f1\n", - " })\n", - "\n", - " # Convert overall metrics to a DataFrame\n", - " overall_metrics_df = pd.DataFrame(overall_metrics_list).set_index(\"Field\")\n", - "\n", - " # Convert precision and recall dictionaries to DataFrames\n", - " precision_df = pd.DataFrame(precision_dicts)\n", - " recall_df = pd.DataFrame(recall_dicts)\n", - "\n", - " return precision_df, recall_df, overall_metrics_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
tpfpfnprecrecf1
Field
AARETE_DERIVED_PRODUCT232310587600.6870750.7534870.718750
AARETE_DERIVED_LOB1737164413460.5137530.5634120.537438
AARETE_DERIVED_PROGRAM1333204817500.3942620.4323710.412438
AARETE_DERIVED_NETWORK972240921110.2874890.3152770.300743
LESSER_OF_IND25778045060.7622010.8358740.797339
GREATER_OF_IND28884931950.8541850.9367500.893564
AARETE_DERIVED_REIMB_METHOD1817156412660.5374150.5893610.562191
UNIT_OF_MEASURE1796158512870.5312040.5825490.555693
REIMB_PCT_RATE1444193716390.4270930.4683750.446782
REIMB_FEE_RATE19336230640.0056200.0061630.005879
NOT_TO_EXCEED_IND26956863880.7971010.8741490.833849
AARETE_DERIVED_FEE_SCHEDULE219211898910.6483290.7109960.678218
AARETE_DERIVED_FEE_SCHEDULE_VERSION1227215418560.3629100.3979890.379641
\n", - "
" - ], - "text/plain": [ - " tp fp fn prec rec \\\n", - "Field \n", - "AARETE_DERIVED_PRODUCT 2323 1058 760 0.687075 0.753487 \n", - "AARETE_DERIVED_LOB 1737 1644 1346 0.513753 0.563412 \n", - "AARETE_DERIVED_PROGRAM 1333 2048 1750 0.394262 0.432371 \n", - "AARETE_DERIVED_NETWORK 972 2409 2111 0.287489 0.315277 \n", - "LESSER_OF_IND 2577 804 506 0.762201 0.835874 \n", - "GREATER_OF_IND 2888 493 195 0.854185 0.936750 \n", - "AARETE_DERIVED_REIMB_METHOD 1817 1564 1266 0.537415 0.589361 \n", - "UNIT_OF_MEASURE 1796 1585 1287 0.531204 0.582549 \n", - "REIMB_PCT_RATE 1444 1937 1639 0.427093 0.468375 \n", - "REIMB_FEE_RATE 19 3362 3064 0.005620 0.006163 \n", - "NOT_TO_EXCEED_IND 2695 686 388 0.797101 0.874149 \n", - "AARETE_DERIVED_FEE_SCHEDULE 2192 1189 891 0.648329 0.710996 \n", - "AARETE_DERIVED_FEE_SCHEDULE_VERSION 1227 2154 1856 0.362910 0.397989 \n", - "\n", - " f1 \n", - "Field \n", - "AARETE_DERIVED_PRODUCT 0.718750 \n", - "AARETE_DERIVED_LOB 0.537438 \n", - "AARETE_DERIVED_PROGRAM 0.412438 \n", - "AARETE_DERIVED_NETWORK 0.300743 \n", - "LESSER_OF_IND 0.797339 \n", - "GREATER_OF_IND 0.893564 \n", - "AARETE_DERIVED_REIMB_METHOD 0.562191 \n", - "UNIT_OF_MEASURE 0.555693 \n", - "REIMB_PCT_RATE 0.446782 \n", - "REIMB_FEE_RATE 0.005879 \n", - "NOT_TO_EXCEED_IND 0.833849 \n", - "AARETE_DERIVED_FEE_SCHEDULE 0.678218 \n", - "AARETE_DERIVED_FEE_SCHEDULE_VERSION 0.379641 " - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "precision_df, recall_df, overall = calculate_precision_recall(accuracies)\n", - "overall.to_csv(\"test-bed-metrics-20250417.csv\", index=False)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
FilenameAARETE_DERIVED_PRODUCTAARETE_DERIVED_LOBAARETE_DERIVED_PROGRAMAARETE_DERIVED_NETWORKLESSER_OF_INDGREATER_OF_INDAARETE_DERIVED_REIMB_METHODUNIT_OF_MEASUREREIMB_PCT_RATEREIMB_FEE_RATENOT_TO_EXCEED_INDAARETE_DERIVED_FEE_SCHEDULEAARETE_DERIVED_FEE_SCHEDULE_VERSION
000-125434-Central MS Diagnostic, LLC-ICMProvid...0.00.3333330.0000000.01.0000001.0000000.8333330.01.0000000.00.7500000.0000000.416667
1050540697 Deepak Nanda 2017 PHSP MU.txt0.00.0000000.0000000.01.0000001.0000000.5000000.01.0000000.01.0000000.0000000.000000
206-1798267-Anthony W. Le, A Professional Corp-...0.00.6428570.0714290.00.7142860.7142860.2678570.00.5714290.00.7142860.0357140.142857
3133923495 CAIPA 2015 PHSP.txt0.00.0000000.0000000.00.7972030.7972030.1958040.00.0839160.00.7972030.0000000.111888
\n", - "
" - ], - "text/plain": [ - " Filename AARETE_DERIVED_PRODUCT \\\n", - "0 00-125434-Central MS Diagnostic, LLC-ICMProvid... 0.0 \n", - "1 050540697 Deepak Nanda 2017 PHSP MU.txt 0.0 \n", - "2 06-1798267-Anthony W. Le, A Professional Corp-... 0.0 \n", - "3 133923495 CAIPA 2015 PHSP.txt 0.0 \n", - "\n", - " AARETE_DERIVED_LOB AARETE_DERIVED_PROGRAM AARETE_DERIVED_NETWORK \\\n", - "0 0.333333 0.000000 0.0 \n", - "1 0.000000 0.000000 0.0 \n", - "2 0.642857 0.071429 0.0 \n", - "3 0.000000 0.000000 0.0 \n", - "\n", - " LESSER_OF_IND GREATER_OF_IND AARETE_DERIVED_REIMB_METHOD \\\n", - "0 1.000000 1.000000 0.833333 \n", - "1 1.000000 1.000000 0.500000 \n", - "2 0.714286 0.714286 0.267857 \n", - "3 0.797203 0.797203 0.195804 \n", - "\n", - " UNIT_OF_MEASURE REIMB_PCT_RATE REIMB_FEE_RATE NOT_TO_EXCEED_IND \\\n", - "0 0.0 1.000000 0.0 0.750000 \n", - "1 0.0 1.000000 0.0 1.000000 \n", - "2 0.0 0.571429 0.0 0.714286 \n", - "3 0.0 0.083916 0.0 0.797203 \n", - "\n", - " AARETE_DERIVED_FEE_SCHEDULE AARETE_DERIVED_FEE_SCHEDULE_VERSION \n", - "0 0.000000 0.416667 \n", - "1 0.000000 0.000000 \n", - "2 0.035714 0.142857 \n", - "3 0.000000 0.111888 " - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "precision_df" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Processing 00-125434-Central MS Diagnostic, LLC-ICMProviderAgreement_78285.txt\n", - "false_negatives: 12 | false_positives: 12 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt\n", - "Contract 02-0677066-Eyemasters-ICMProviderAgreement_64220_2.txt not found in predictions.\n", - "\n", - "Processing 050540697 Deepak Nanda 2017 PHSP MU.txt\n", - "false_negatives: 2 | false_positives: 2 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 06-1798267-Anthony W. Le, A Professional Corp-ICMProviderAgreement_316844.txt\n", - "false_negatives: 40 | false_positives: 56 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 133923495 CAIPA 2015 PHSP.txt\n", - "false_negatives: 114 | false_positives: 143 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2001-11-01 California Emergency Physicians PSA_MU.txt\n", - "false_negatives: 50 | false_positives: 47 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2013-09-01 Fresno Dental Surgery Center PPA_MU.txt\n", - "false_negatives: 41 | false_positives: 45 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof MU.txt\n", - "false_negatives: 350 | false_positives: 350 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2014-01-01 United Health Centers San Joaquin AMD MU.txt\n", - "false_negatives: 30 | false_positives: 30 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 20-1668371-Midwest Medical Equipment Solutions, Inc.-ICMProviderAgreementAmendment_166876.txt\n", - "false_negatives: 120 | false_positives: 56 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2018 09 01 UTMB (Amendment 10 change in reimbursement) MU.txt\n", - "false_negatives: 47 | false_positives: 36 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2019 01 01 Gulf Coast Division (Agreement_STAR, CHIP, CHIP P, STAR_PLUS) MU.txt\n", - "false_negatives: 36 | false_positives: 68 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2019 06 01 Dr Raul Rivera & Associates (Amend 1)_MU.txt\n", - "false_negatives: 2 | false_positives: 2 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2021-07-15 COMM Agmt FE - Samaritan Health Services MU.txt\n", - "false_negatives: 266 | false_positives: 282 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 2022 01 20 West Park Primary Care_Master (Master) PHY Executed.pdf.filepart MU.txt\n", - "false_negatives: 40 | false_positives: 50 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 33-0057155-Apria Healthcare, Inc.-ICMProviderAgreementAmendment_26762.txt\n", - "false_negatives: 533 | false_positives: 544 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 36-2902782-Circle Family HealthCare Network-ICMProviderAgreement_59631.txt\n", - "false_negatives: 33 | false_positives: 113 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 39-0806250-Divine Saviour Hospital-ICMProviderAgreement_97009.txt\n", - "false_negatives: 2 | false_positives: 2 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 429670586-Melissa Ann Clark-ICMProviderAgreement_128379_1.txt\n", - "false_negatives: 8 | false_positives: 10 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 46-0979580-Harris and Theodore Anesthesia Services, LLC-ICMProviderAgreement_313205.txt\n", - "false_negatives: 25 | false_positives: 35 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 5.2000_Rehab Designs of America_VendorAgreement MU.txt\n", - "false_negatives: 4 | false_positives: 5 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 6.2000_Omni Transport Systems_ServicesAgreement MU.txt\n", - "false_negatives: 4 | false_positives: 6 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 61-1400871-West Kentucky Dermatology PSC-ICMProviderAgreement_104221.txt\n", - "false_negatives: 2 | false_positives: 4 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 84-5179194-Heart and Soul Hospice LLC-ICMProviderAgreement_273737.txt\n", - "false_negatives: 4 | false_positives: 4 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 86-0898663-Founder Health, LLC dba Preferred Homecare-ICMProviderAgreement_120661.txt\n", - "false_negatives: 88 | false_positives: 103 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 91-1885077-COASTAL ANESTHESIOLOGY MEDICAL-ICMProviderAgreement_242069.txt\n", - "false_negatives: 47 | false_positives: 47 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 92-2764433-MAGNOLIA MOBILE HOME CARE LLC-ICMProviderAgreement_322270.txt\n", - "false_negatives: 13 | false_positives: 15 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 95-2648499-Mission Pathology Medical Associates-ICMProviderAgreement_254534.txt\n", - "false_negatives: 76 | false_positives: 66 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing 95-3372911; Multiple-Davita Healthcare Partners, Inc.-ICMProviderAgreement_138658.txt\n", - "false_negatives: 52 | false_positives: 50 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Adena Health System National Template Agreement C19460229AA.txt\n", - "false_negatives: 50 | false_positives: 54 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing AGMT - HSA - TEXAS HEALTH RESOURCES - 75-6001743 MU.txt\n", - "false_negatives: 2 | false_positives: 3 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Akron General Health System_Eleventh Amendment_20171001.txt\n", - "false_negatives: 26 | false_positives: 42 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing AMD - IV - HSA eff 2017 (amd 4) - Broward MU.txt\n", - "false_negatives: 52 | false_positives: 67 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Anc2_23-1943113_BayadaHHC_Amendment1_553403.txt\n", - "false_negatives: 60 | false_positives: 58 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Anc8_Hanger Pros & Ortho West Contract.txt\n", - "false_negatives: 86 | false_positives: 122 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Boilerplate_2020-01-01 COMM Amend FE (Beacon) - Portland Adventist Medical Center MU.txt\n", - "false_negatives: 7 | false_positives: 13 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Boilerplate_AMD - MKPL - NBHD dba Broward Health - Copy_MU.txt\n", - "false_negatives: 12 | false_positives: 14 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Boilerplate_Benjamin Bieber MD - Amendment 5.30.16_MU.txt\n", - "false_negatives: 8 | false_positives: 9 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Boilerplate_Fac_Bronx Lebanon Hospital Center - Amendment 1.1.14 MU.txt\n", - "false_negatives: 8 | false_positives: 8 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Boilerplate_PSA_Houston Eye Associates_76-0512625_05012007 MU.txt\n", - "false_negatives: 2 | false_positives: 3 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing BoilerplateAnc_Bentley Medical PLLC - Amendment 8.1.14_MU.txt\n", - "false_negatives: 2 | false_positives: 2 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing BSW _ Original Contract _ 1.1.2000 MU.txt\n", - "false_negatives: 22 | false_positives: 6 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Cabell Huntington_OH MCD Compensation MU.txt\n", - "false_negatives: 15 | false_positives: 17 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Community Health Network_Base Contract MU.txt\n", - "false_negatives: 8 | false_positives: 6 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_060121 HonorHealth and its Affiliates Care 1st Physician Agreement.txt\n", - "false_negatives: 21 | false_positives: 25 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_2009-08-10 Adventist - University Community Hopstial LTAC MU.txt\n", - "false_negatives: 9 | false_positives: 9 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_2012-10-01 AdvenHealth Amendment.txt\n", - "false_negatives: 105 | false_positives: 205 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_2018-09-15 COMM Agmt FE - Bandon Community Health Center dba Coast Community Health Center.txt\n", - "false_negatives: 176 | false_positives: 129 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_2018-09-15 COMM Amend FE Plaza Ambulatory Surgery Center, LLC MU.txt\n", - "false_negatives: 16 | false_positives: 20 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_HSA_EFF07012010_Driscoll Children's Hospital MU.txt\n", - "false_negatives: 4 | false_positives: 3 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_Parkview Health Systems-OH MP, IN MP-ID C15656222AA MU.txt\n", - "false_negatives: 14 | false_positives: 28 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_Prof_Anna Suponya MD PC - Agreement Provider Signed_MU.txt\n", - "false_negatives: 14 | false_positives: 11 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_PSA - Advanced Gastroenterology of Texas PLLC -Executed 47-4543923 MU.txt\n", - "false_negatives: 14 | false_positives: 18 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_PSA - TX - Fresenius BIOMEDICAL APPLICATIONS OF TEXAS INC - 11-2226275 - 06012010 (1) (2) MU.txt\n", - "false_negatives: 24 | false_positives: 24 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Custom_SOUTHWEST BEHAVIORAL & HEALTH SERVICES INC.txt\n", - "false_negatives: 69 | false_positives: 44 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt\n", - "false_negatives: 9 | false_positives: 14 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing CustomFac_Benedictine Hospital - Amendment 1.1.16 (HBX,HARP,EPP) (REVISED).txt\n", - "false_negatives: 30 | false_positives: 31 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Fac1_Bullhead City_Western Arizona Med Center - 860982071 MU.txt\n", - "false_negatives: 9 | false_positives: 5 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Genesis_ICMProviderAgreement_ICMProviderAgreement_153537_5.txt\n", - "false_negatives: 9 | false_positives: 9 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Hunt Regional Medical Center Executed Contract.txt\n", - "false_negatives: 2 | false_positives: 2 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_AlisonUnitisLPCMH_227710_5.txt\n", - "false_negatives: 9 | false_positives: 9 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_AzarEyeSurgeryCenter_232594_6.txt\n", - "false_negatives: 29 | false_positives: 30 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_BanyanDelaware_210080_5.txt\n", - "false_negatives: 16 | false_positives: 15 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_CorasWellnessAndBehavioralHealth_212482_8.txt\n", - "false_negatives: 17 | false_positives: 18 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_ICMProviderAgreement_143342_10_Landmark of Midwest City Rehab and Nursing Center.txt\n", - "false_negatives: 6 | false_positives: 24 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_ICMProviderAgreement_295973_15.txt\n", - "false_negatives: 5 | false_positives: 11 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing ICMProviderAgreement_NewCastleHealthandRehabilitationCenter_212261_13 MU.txt\n", - "false_negatives: 18 | false_positives: 18 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Progressive Women's Health, PLLC_Physician Agreement MU.txt\n", - "false_negatives: 36 | false_positives: 49 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Southeast Georgia Health System Inc_20170517_Dually Executed MU.txt\n", - "false_negatives: 12 | false_positives: 18 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n", - "Processing Tampa General Rehabilitation Hospital Agreement MU.txt\n", - "false_negatives: 9 | false_positives: 5 | true_positives: 0 | accuracy: 0.0 | precision: 0.0 | recall: 0.0 | f1_score: 0\n", - "\n" - ] - } - ], - "source": [ - "precisions, recalls, accuracies = [], [], []\n", - "comparisons = {}\n", - "\n", - "total_true_positives, total_false_positives, total_false_negatives = 0, 0, 0\n", - "\n", - "for contract_file_name in labels_df[\"FILE_NAME\"].unique():\n", - " print(f\"Processing {contract_file_name}\")\n", - " labels_df_contract = labels_df[\n", - " labels_df[\"FILE_NAME\"] == contract_file_name\n", - " ]\n", - " predictions_df_contract = predictions_df[\n", - " predictions_df[\"FILE_NAME\"] == contract_file_name\n", - " ]\n", - " if len(labels_df_contract) == 0:\n", - " print(f\"Contract {contract_file_name} not found in labels.\\n\")\n", - " continue\n", - " elif len(predictions_df_contract) == 0:\n", - " print(f\"Contract {contract_file_name} not found in predictions.\\n\")\n", - " continue\n", - "\n", - " # Full reimbursement primary\n", - " comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(\n", - " labels_df_contract,\n", - " predictions_df_contract,\n", - " \"FILE_NAME\",\n", - " fields,\n", - " thresholds=MATCH_THRESHOLD\n", - " )\n", - " comparisons[contract_file_name] = comparison\n", - "\n", - " # comparison = evaluate_one_to_n_fields_fuzzy_matching_field_by_field(labels_df_contract, predictions_df_contract, \"FILE_NAME\", ['REIMB_TERM', 'FEE_SCHEDULE'])\n", - "\n", - " precisions.append(comparison[\"precision\"])\n", - " recalls.append(comparison[\"recall\"])\n", - " accuracies.append(comparison[\"accuracy\"])\n", - "\n", - " total_true_positives += comparison[\"true_positives\"]\n", - " total_false_positives += comparison[\"false_positives\"]\n", - " total_false_negatives += comparison[\"false_negatives\"]\n", - "\n", - " # contract file name is already printed above\n", - " print(\n", - " f\"false_negatives: {comparison['false_negatives']} | false_positives: {comparison['false_positives']} | true_positives: {comparison['true_positives']} | accuracy: {comparison['accuracy']} | precision: {comparison['precision']} | recall: {comparison['recall']} | f1_score: {comparison['f1_score']}\\n\"\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Median accuracy across 69 contracts at match threshold 90: 0.0000\n", - "Median precision across 69 contracts at match threshold 90: 0.0000\n", - "Median recall across 69 contracts at match threshold 90: 0.0000\n", - "Mean accuracy across 69 contracts at match threshold 90: 0.0000\n", - "Mean precision across 69 contracts at match threshold 90: 0.0000\n", - "Mean recall across 69 contracts at match threshold 90: 0.0000\n", - "Grand accuracy across 69 contracts at match threshold 90: 0.0000 (total true positives: 0, total false positives: 3381, total false negatives: 3083)\n", - "Grand precision across 69 contracts at match threshold 90: 0.0000 (total true positives: 0, total false positives: 3381)\n", - "Grand recall across 69 contracts at match threshold 90: 0.0000 (total true positives: 0, total false negatives: 3083)\n" - ] - } - ], - "source": [ - "from statistics import median, mean\n", - "\n", - "print(f\"Median accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {median(accuracies):.4f}\")\n", - "print(f\"Median precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {median(precisions):.4f}\")\n", - "print(f\"Median recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {median(recalls):.4f}\")\n", - "print(f\"Mean accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {mean(accuracies):.4f}\")\n", - "print(f\"Mean precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {mean(precisions):.4f}\")\n", - "print(f\"Mean recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {mean(recalls):.4f}\")\n", - "print(f\"Grand accuracy across {len(accuracies)} contracts at match threshold {MATCH_THRESHOLD}: {(total_true_positives) / (total_true_positives + total_false_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives}, total false negatives: {total_false_negatives})\")\n", - "print(f\"Grand precision across {len(precisions)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_positives):.4f} (total true positives: {total_true_positives}, total false positives: {total_false_positives})\")\n", - "print(f\"Grand recall across {len(recalls)} contracts at match threshold {MATCH_THRESHOLD}: {total_true_positives / (total_true_positives + total_false_negatives):.4f} (total true positives: {total_true_positives}, total false negatives: {total_false_negatives})\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "doczy-smart-chunking-BxQLr-8H-py3.12", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/fieldExtraction/tests/one_to_n_test.py b/fieldExtraction/tests/one_to_n_test.py index b38654a..cc7a36e 100644 --- a/fieldExtraction/tests/one_to_n_test.py +++ b/fieldExtraction/tests/one_to_n_test.py @@ -12,51 +12,6 @@ class TestOneToN(unittest.TestCase): self.exhibit_text = "Sample exhibit text with some content" self.filename = "test_file.pdf" - @patch('src.utils.llm_utils.invoke_claude') - @patch('src.utils.string_utils.extract_text_from_delimiters') - def test_get_exhibit_headers(self, mock_extract_text, mock_invoke_claude): - """Test exhibit header extraction with both base pages and decimal pages""" - # Setup - exhibit_dict = { - "1.0": "First exhibit content", - "2.0": "Second exhibit content", - "2.1": "More second exhibit content", - "2.2": "Even more second exhibit content", - "3.0": "Third exhibit content" - } - - # Configure mocks - mock_invoke_claude.return_value = "raw_llm_response" - mock_extract_text.side_effect = [ - "Exhibit A - Services", # For page 1.0 - "Exhibit B - Payment", # For page 2.0 - "Exhibit C - Terms" # For page 3.0 - ] - - # Execute - result = preprocessing_funcs.get_exhibit_headers(exhibit_dict, self.filename) - - # Assert - expected_result = { - "1.0": "Exhibit A - Services", - "2.0": "Exhibit B - Payment", - "2.1": "Exhibit B - Payment", # Should use cached result from 2.0 - "2.2": "Exhibit B - Payment", # Should use cached result from 2.0 - "3.0": "Exhibit C - Terms" - } - - self.assertEqual(result, expected_result) - - # Verify LLM was only called for base pages (3 times, not 5) - self.assertEqual(mock_invoke_claude.call_count, 3) - - # Verify prompt content - mock_invoke_claude.assert_any_call( - investment_prompts.EXHIBIT_HEADER("First exhibit content"[0:400]), - config.MODEL_ID_CLAUDE35_SONNET, - self.filename - ) - @patch('src.utils.llm_utils.invoke_claude') def test_get_reimbursement_primary(self, mock_invoke_claude): # Setup diff --git a/fieldExtraction/tests/preprocessing_funcs_test.py b/fieldExtraction/tests/preprocessing_funcs_test.py index 9f63baa..64e4c8a 100644 --- a/fieldExtraction/tests/preprocessing_funcs_test.py +++ b/fieldExtraction/tests/preprocessing_funcs_test.py @@ -1,14 +1,16 @@ import pytest -from src.preprocessing_funcs import split_text from src.preprocessing_funcs import ( remove_page_indicators, - split_text, clean_law_symbols, filter_quick_review, get_exhibit_pages, ) +import src.prompts.investment_prompts as investment_prompts +import src.config as config from src.client.smart_chunking_funcs import smart_chunk_ac +from unittest.mock import patch + class TestPreprocessingFuncs: # Test cases for clean_newlines @@ -109,352 +111,46 @@ class TestPreprocessingFuncs: assert contract == expected_contract assert quick_review == expected_quick_review - # Test cases for get_exhibit_pages - @pytest.mark.parametrize("mock_response, expected_pages, text_dict", [ - # First page included by default, second page not an exhibit - (["|N|"], ["1"], {"1": "Exhibit content", "2": "Non-exhibit content"}), - # Multiple pages with mixed exhibit status - (["|N|", "|Y|"], ["1", "3"], {"1": "First page", "2": "Regular content", "3": "Exhibit B"}), - # No exhibit page detected - ([], [], {}), - # Multiple exhibit pages - (["|Y|","|Y|"], ["1", "2"], {"1": "Exhibit content", "2": "Attachment"}), - # Invalid response; still include the first page - (["Invalid"], ["1"], {"1": "Non-exhibit content"}), - ]) - def test_get_exhibit_pages(self, mocker, mock_response, expected_pages, text_dict): - # Mock the LLM response - mocker_invoke_claude = mocker.patch("src.utils.llm_utils.invoke_claude", side_effect=mock_response) - filename = "test.pdf" - assert get_exhibit_pages(text_dict, filename) == expected_pages - - # Verify calls made - should be number of pages minus 1 (first page), but never negative - expected_calls = max(0, len(text_dict) - 1) - assert mocker_invoke_claude.call_count == expected_calls - -from src.preprocessing_funcs import get_exhibit_dict - -class TestGetExhibitDict: - - def test_case_1_short_table_first_page(self): - """Case 1: Short Table on First Page""" + @patch('src.utils.llm_utils.invoke_claude') + @patch('src.utils.string_utils.extract_text_from_delimiters') + def test_get_exhibit_pages(self, mock_extract_text, mock_invoke_claude): + """Test exhibit page detection with both base and decimal pages""" + # Setup test data text_dict = { - '5': 'Exhibit 5 header text', - '5.0': 'Short table content' - } - exhibit_chunk_mapping = { - '5': '5', - '5.0': '5' + "1": "First page content", + "2.0": "Second page content with exhibit header", + "2.1": "Continuation of second page", + "2.2": "More continuation of second page", + "3.0": "Third page content with exhibit header", + "4": "Regular page content" } - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nShort table content' - } - assert result == expected - - def test_case_2_continued_short_table(self): - """Case 2: Continued Short Table on Subsequent Pages""" - text_dict = { - '5': 'Exhibit 5 header text', - '5.0': 'Table start content', - '5.1': 'Table continuation content' - } - exhibit_chunk_mapping = { - '5': '5', - '5.0': '5', - '5.1': '5' - } + filename = "test_contract.pdf" - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nTable start content', - '5.1': 'Exhibit 5 header text\nTable continuation content' - } - assert result == expected - - def test_case_3_long_table_first_page(self): - """Case 3: Long Table on First Page (split by row limit)""" - text_dict = { - '5': 'Exhibit 5 header text', - '5.0': 'Long table part 1', - '5.1': 'Long table part 2', - '5.2': 'Long table part 3' - } - exhibit_chunk_mapping = { - '5': '5', - '5.0': '5', - '5.1': '5', - '5.2': '5' - } + # Configure mocks + mock_invoke_claude.return_value = "raw_llm_response" + mock_extract_text.side_effect = [ + "Exhibit A", # For page 2.0 + "N/A", # For page 4 + "Exhibit B" # For page 3.0 + ] - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nLong table part 1', - '5.1': 'Exhibit 5 header text\nLong table part 2', - '5.2': 'Exhibit 5 header text\nLong table part 3' - } - assert result == expected - - def test_case_4_continued_long_table(self): - """Case 4: Continued Long Table on Subsequent Pages""" - text_dict = { - '5': 'Exhibit 5 header text', - '6': 'Page 6 text', - '6.0': 'Continued table part 1', - '6.1': 'Continued table part 2', - '7': 'Page 7 text' - } - exhibit_chunk_mapping = { - '5': '5', - '6': '5', - '6.0': '5', - '6.1': '5', - '7': '5' - } + # Execute + exhibit_pages, exhibit_headers = get_exhibit_pages(text_dict, filename) - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nPage 6 text\nContinued table part 1\nPage 7 text', - '5.1': 'Exhibit 5 header text\nPage 6 text\nContinued table part 2\nPage 7 text', - '5.2': 'Exhibit 5 header text\nPage 6 text\nPage 7 text' - } - assert result == expected - - def test_case_5_multiple_tables_first_page(self): - """Case 5: Multiple Tables on First page only""" - text_dict = { - '5': 'Exhibit 5 header text', - '5.0': 'First table content', - '5.1': 'Second table content', - '5.2': 'Third table content' - } - exhibit_chunk_mapping = { - '5': '5', - '5.0': '5', - '5.1': '5', - '5.2': '5' - } + # Assert + expected_exhibit_pages = ["1", "2.0", "2.1", "2.2", '4'] - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nFirst table content', - '5.1': 'Exhibit 5 header text\nSecond table content', - '5.2': 'Exhibit 5 header text\nThird table content' - } - assert result == expected - - def test_case_6_multiple_tables_with_continuation(self): - """Case 6: Multiple Tables on Subsequent Page, with first being continuation""" - text_dict = { - '5': 'Exhibit 5 header text', - '6': 'Page 6 text', - '6.0': 'Continuation of previous table', - '6.1': 'New table on same page', - '6.2': 'Another new table on same page' - } - exhibit_chunk_mapping = { - '5': '5', - '6': '5', - '6.0': '5', - '6.1': '5', - '6.2': '5' - } + assert exhibit_pages == expected_exhibit_pages - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nPage 6 text\nContinuation of previous table', - '5.1': 'Exhibit 5 header text\nPage 6 text\nNew table on same page', - '5.2': 'Exhibit 5 header text\nPage 6 text\nAnother new table on same page' - } - assert result == expected - - def test_case_7_multiple_tables_across_pages(self): - """Case 7: Multiple Tables on First and Subsequent Pages""" - text_dict = { - '5': 'Exhibit 5 header text', - '5.0': 'First table on page 5', - '5.1': 'Second table on page 5', - '6': 'Page 6 text', - '6.0': 'First table on page 6', - '6.1': 'Second table on page 6', - '7': 'Page 7 text' - } - exhibit_chunk_mapping = { - '5': '5', - '5.0': '5', - '5.1': '5', - '6': '5', - '6.0': '5', - '6.1': '5', - '7': '5' - } + # Verify LLM was only called for base pages + assert mock_invoke_claude.call_count == 3 - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nFirst table on page 5\nPage 6 text\nPage 7 text', - '5.1': 'Exhibit 5 header text\nSecond table on page 5\nPage 6 text\nPage 7 text', - '5.2': 'Exhibit 5 header text\nPage 6 text\nFirst table on page 6\nPage 7 text', - '5.3': 'Exhibit 5 header text\nPage 6 text\nSecond table on page 6\nPage 7 text', - '5.4': 'Exhibit 5 header text\nPage 6 text\nPage 7 text' - } - assert result == expected - - def test_edge_case_exhibit_is_decimal_page(self): - """Edge Case: Exhibit itself is a decimal page""" - text_dict = { - '5.1': 'Exhibit starting at decimal page' - } - exhibit_chunk_mapping = { - '5.1': '5.1' - } - - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.1': 'Exhibit starting at decimal page' - } - assert result == expected - - def test_edge_case_no_decimal_pages(self): - """Edge Case: No decimal pages in exhibit (pure text)""" - text_dict = { - '5': 'Exhibit 5 header text', - '6': 'Page 6 text', - '7': 'Page 7 text' - } - exhibit_chunk_mapping = { - '5': '5', - '6': '5', - '7': '5' - } - - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '5.0': 'Exhibit 5 header text\nPage 6 text\nPage 7 text' - } - assert result == expected - - def test_example_from_docstring_1(self): - """Test first example from docstring""" - text_dict = { - '1.0': 'Page 1.0 text', - '2': 'Page 2 text', - '3.0': 'Page 3.0 text', - '3.1': 'Page 3.1 text', - '3.2': 'Page 3.2 text', - '4.0': 'Page 4.0 text' - } - exhibit_chunk_mapping = { - '1.0': '1.0', - '2': '2', - '3.0': '2', - '3.1': '2', - '3.2': '2', - '4.0': '2' - } - - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '1.0': 'Page 1.0 text', - '2.0': 'Page 2 text\nPage 3.0 text', - '2.1': 'Page 2 text\nPage 3.1 text', - '2.2': 'Page 2 text\nPage 3.2 text', - '2.3': 'Page 2 text\nPage 4.0 text' - } - assert result == expected - - def test_example_from_docstring_2(self): - """Test second example from docstring""" - text_dict = { - '1.0': 'Page 1.0 text', - '2': 'Page 2 text', - '3.0': 'Page 3.0 text', - '3.1': 'Page 3.1 text', - '3.2': 'Page 3.2 text', - '4': 'Page 4 text', - '5': 'Page 5 text' - } - exhibit_chunk_mapping = { - '1.0': '1.0', - '2': '2', - '3.0': '2', - '3.1': '2', - '3.2': '2', - '4': '2', - '5': '2' - } - - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - '1.0': 'Page 1.0 text', - '2.0': 'Page 2 text\nPage 3.0 text\nPage 4 text\nPage 5 text', - '2.1': 'Page 2 text\nPage 3.1 text\nPage 4 text\nPage 5 text', - '2.2': 'Page 2 text\nPage 3.2 text\nPage 4 text\nPage 5 text', - '2.3': 'Page 2 text\nPage 4 text\nPage 5 text' - } - assert result == expected - - def test_exhibit_with_decimal_pages(self): - """Test exhibit with decimal pages""" - text_dict = { - "23.0": "Exhibit page 23 header", - "24": "Page 24 text" - } - exhibit_chunk_mapping = { - "23.0": "23.0", - "24": "23.0" - } - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - "23.0.0": "Exhibit page 23 header\nPage 24 text", - "23.0.1": "Page 24 text" - } - assert result == expected - - def test_multiple_exhibits_with_tables(self): - """Test multiple different exhibits, each with their own base pages and tables""" - text_dict = { - '10': 'Exhibit A header', - '10.0': 'Exhibit A table 1', - '10.1': 'Exhibit A table 2', - '11': 'Exhibit A page 2', - - '20': 'Exhibit B header', - '21': 'Exhibit B page 2', - '21.0': 'Exhibit B table 1', - '22': 'Exhibit B page 3', - - '30': 'Exhibit C header', - '31': 'Exhibit C page 2' - } - exhibit_chunk_mapping = { - '10': 'A', - '10.0': 'A', - '10.1': 'A', - '11': 'A', - - '20': 'B', - '21': 'B', - '21.0': 'B', - '22': 'B', - - '30': 'C', - '31': 'C' - } - - result = get_exhibit_dict(text_dict, exhibit_chunk_mapping) - expected = { - # Exhibit A - 'A.0': 'Exhibit A header\nExhibit A table 1\nExhibit A page 2', - 'A.1': 'Exhibit A header\nExhibit A table 2\nExhibit A page 2', - 'A.2': 'Exhibit A header\nExhibit A page 2', - - # Exhibit B - 'B.0': 'Exhibit B header\nExhibit B page 2\nExhibit B table 1\nExhibit B page 3', - 'B.1': 'Exhibit B header\nExhibit B page 2\nExhibit B page 3', - - # Exhibit C (no decimal pages) - 'C.0': 'Exhibit C header\nExhibit C page 2' - } - assert result == expected \ No newline at end of file + # Verify prompt content and model + mock_invoke_claude.assert_any_call( + investment_prompts.EXHIBIT_HEADER("Second page content with exhibit header"[0:400]), + config.MODEL_ID_CLAUDE3_HAIKU, + filename, + max_tokens=150 + )