From 4478955e24eee197eef2fe4df117a796fda469ef Mon Sep 17 00:00:00 2001 From: FaizanM2000 Date: Tue, 9 Apr 2024 07:06:43 -0500 Subject: [PATCH 01/78] testing top down intial test complete --- .gitignore | 9 +- test.ipynb | 1481 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1489 insertions(+), 1 deletion(-) create mode 100644 test.ipynb diff --git a/.gitignore b/.gitignore index dd5df53..8a56daa 100644 --- a/.gitignore +++ b/.gitignore @@ -73,4 +73,11 @@ streamlit/results.csv streamlit/venv *.tfplan -*.pem \ No newline at end of file +*.pem +textfiles/ +texts/ +*.csv +*.json +myenv/ +*.env +*.txt diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 0000000..34d9ed0 --- /dev/null +++ b/test.ipynb @@ -0,0 +1,1481 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "# Load the CSV file into a DataFrame\n", + "df = pd.read_csv('class_B_data.csv')\n", + "\n", + "# Create a new DataFrame to store unique values\n", + "unique_values_df = pd.DataFrame()\n", + "\n", + "# Loop through each column in the original DataFrame and get unique values\n", + "for column in df.columns:\n", + " # Extract unique values for the column, sort them, and reset index for clean formatting\n", + " unique_values = pd.Series(df[column].unique()).sort_values().reset_index(drop=True)\n", + " unique_values_df[column] = unique_values\n", + "\n", + "# Now, unique_values_df contains all the unique values for each column\n", + "# You can print it to see its contents\n", + "print(unique_values_df)\n", + "\n", + "# If you want to save this DataFrame to a new CSV\n", + "unique_values_df.to_csv('unique_values_class_B_data.csv', index=False)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import boto3\n", + "import dotenv\n", + "\n", + "dotenv.load_dotenv()\n", + "\n", + "s3_client = boto3.client('s3',\n", + " region_name=\"us-east-2\",\n", + " aws_access_key_id=os.getenv('ACCESS_KEY'),\n", + " aws_secret_access_key=os.getenv('SECRET_KEY'),\n", + " aws_session_token=os.getenv('SESSION_TOKEN')\n", + " )\n", + "bucket = 'doczy-dev-infra-textract'\n", + "objects = s3_client.list_objects_v2(Bucket=bucket, Prefix=\"training-data/contract-text-file/\")\n", + "file_list = []\n", + "for obj in objects['Contents']:\n", + " if not obj['Key'].endswith('/'):\n", + " file_list.append(obj['Key'])\n", + "\n", + "contract_list = sorted(file_list)\n", + "\n", + "for contract in contract_list:\n", + " data = s3_client.get_object(Bucket=bucket, Key=contract)\n", + " contents = data['Body'].read()\n", + " context = contents.decode(\"utf-8\")\n", + " \n", + " # Modify the path where you want to save the file\n", + " # Split the path and the filename\n", + " path, filename = os.path.split(contract)\n", + " # Remove the file extension from the path\n", + " save_path = os.path.join(\"your_directory\", path) # Specify your directory here\n", + " # Ensure the directory exists\n", + " os.makedirs(save_path, exist_ok=True)\n", + " # Write the text to a new file for each contract\n", + " with open(os.path.join(save_path, filename[:-4] + '.txt'), 'w') as outfile:\n", + " outfile.write(context)\n", + "\n", + "print(contract_list)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#open text file from textfiles folder and read the content:\n", + "with open('textfiles/2001-11-01 California Emergency Physicians PSA_MU.txt', 'r') as infile:\n", + " context = infile.read()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "def METAL_LEVELS():\n", + " return \"\"\"Platinum\n", + "Gold\n", + "Silver\n", + "Bronze\n", + "\"\"\"\n", + "def LINE_OF_BUSINESS():\n", + "\n", + " return \"\"\"\n", + "COMMERCIAL \n", + "Commercial and Marketplace\n", + "Commercial/Group\n", + "LOB/Provider Type Specific\n", + "MARKETPLACE\n", + "MEDICAID\n", + "MEDICARE\n", + "Marketplace/Exchange\n", + "Medicaid-Medicare Plan\n", + "Medicaid/Medicare\n", + "Medicare Advantage\n", + "Medicare Supplemental\n", + "Self-Funded/ASO/TPA\n", + "Tailored Network\n", + "\"\"\"\n", + "\n", + "def PLANS():\n", + " return \"\"\"HMO\n", + "Leased PPO Benefits Program\n", + "Affinity\n", + "Beacon\n", + "Blue Network\n", + "\"\"\"\n", + "\n", + "def NETWORKS():\n", + " return \"\"\"HMO\n", + "POS\n", + "EPO\n", + "PPO\n", + "FFS\n", + "\"\"\"\n", + "\n", + "def SERVICE_AREAS():\n", + " return \"\"\"Urban/Rural\n", + "County\n", + "ZIP\n", + "Georgia\n", + "Indiana\n", + "Ohio\n", + "\"\"\"\n", + "\n", + "def TYPES_OF_PROVIDER():\n", + " return \"\"\"IP\n", + "Hospital\n", + "SNF\n", + "Home Health\n", + "Intermediate Care Facility\n", + "Outpatient\n", + "ASC\n", + "Primary Care\n", + "Specialist\n", + "Ancillary\n", + "ACS\n", + "AIR AMBULANCE\n", + "\"\"\"\n", + "\n", + "def PROVIDER_SPECIALTIES():\n", + " return \"\"\"Anesthesiology\n", + "Emergency Medicine\n", + "Family Medicine\n", + "General Practice\n", + "Pediatrics\n", + "Obstetrics & Gynecology\n", + "Orthopedic Surgery\n", + "Radiology\n", + "ASCs\n", + "Air Ambulance\n", + "All Physician and Mid-Level Healthcare Professionals\n", + "Behavioral Health\n", + "\"\"\"\n", + "\n", + "def PROGRAM_MEDICARE():\n", + " return \"\"\"SNP\n", + "C-SNP\n", + "D-SNP\n", + "I-SNP\n", + "LTSS\n", + "PACE\n", + "\"\"\"\n", + "\n", + "def PROGRAM_MEDICAID():\n", + " return \"\"\"ABD\n", + "CHIP\n", + "Foster\n", + "IDD\n", + "TANF\n", + "SSI\n", + "SSDI\n", + "\"\"\"\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def LOB_PROMPT_NEW(): \n", + " LOB = LINE_OF_BUSINESS()\n", + " metal_levels = METAL_LEVELS()\n", + " plans = PLANS()\n", + " networks = NETWORKS()\n", + " service_areas = SERVICE_AREAS()\n", + " \n", + " return f\"\"\"Extract ALL INSTANCES OF THE FOLLOWING INFORMATION FROM THE ABOVE TEXT. If there are multiple instances of this type of information, identify them all and return as seperate objects for each LOB. Extract information like the examples below. Note that the Line of Business (LOB), Plan, Network, and other attributes may vary based on the context provided:\n", + "\n", + "CONTRACT_LOB LOB_PRICING_TERMS_EFFECTIVE_DT Plan CONTRACT_NETWORK CONTRACT_SERVICE_AREA LOB_PRICING_TERMS_TERMINATION_DT \n", + "MARKETPLACE 1/02/2014 GOLD NARROW NETWORK PPO OHIO 1/02/2015 \n", + "\n", + "If the LOB is identified to be Marketplace, then follow the example below:\n", + "CONTRACT_LOB LOB_PRICING_TERMS_EFFECTIVE_DT Plan CONTRACT_NETWORK CONTRACT_SERVICE_AREA LOB_PRICING_TERMS_TERMINATION_DT CONTRACT_MARKETPLACE_METAL_LEVEL\n", + "MARKETPLACE 1/02/2014 GOLD NARROW NETWORK PPO OHIO 1/02/2015 Gold\n", + "\n", + "Choose from the following LOBs for reference. If not found in the list, do your best to match the closest: {LOB}\n", + "For Effective Date, What is start date of line of business specific compensation exhibit? If one is not explicitly stated, this is the same as the contract's effective date. This should be in a long date format and should be earlier than today. Only return the date in MM/DD/YYYY format, do not return any other content.\n", + "You must return the Plan for every entry. The Network, Program, and Service Area should be clearly identified based on the context provided.\n", + "Choose only from the following Line of Business: {LOB}. \n", + "Choose only from the following plans: {plans}.\n", + "Choose only from the following Networks: {networks}.\n", + "Choose only from the following Service Areas: {service_areas}.\n", + "For Date of Termination, identify the specific date upon which the compensation terminates, in the following format ONLY: MM/DD/YYYY.\n", + "Only identify the Metal Level if the LOB is Marketplace. Choose only from the following Metal Levels: {metal_levels}.\n", + "\n", + "Return only the extracted info as a JSON list based on the attributes of the LOB and its context.\"\"\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "##EXPLORED SPLITTING ON REGEX MATCHES ON ATTRIBUTES\n", + "\n", + "import regex as re\n", + "import chardet\n", + "\n", + "# Define the category functions\n", + "def METAL_LEVELS():\n", + " return \"\"\"\n", + " Platinum Gold Silver Bronze\n", + " \"\"\"\n", + "\n", + "def LINE_OF_BUSINESS():\n", + " return \"\"\"\n", + " COMMERCIAL Commercial and Marketplace Commercial/Group LOB/Provider Type Specific MARKETPLACE MEDICAID MEDICARE Marketplace/Exchange Medicaid-Medicare Plan Medicaid/Medicare Medicare Advantage Medicare Supplemental Self-Funded/ASO/TPA Tailored Network\n", + " \"\"\"\n", + "\n", + "\n", + "def NETWORKS():\n", + " return \"\"\"\n", + " HMO POS EPO PPO FFS\n", + " \"\"\"\n", + "\n", + "def SERVICE_AREAS():\n", + " return \"\"\"\n", + " Urban/Rural County ZIP Georgia Indiana Ohio\n", + " \"\"\"\n", + "\n", + "def TYPES_OF_PROVIDER():\n", + " return \"\"\"\n", + " IP Hospital SNF Home Health Intermediate Care Facility Outpatient ASC Primary Care Specialist Ancillary ACS AIR AMBULANCE\n", + " \"\"\"\n", + "\n", + "\n", + "def PROGRAM_MEDICARE():\n", + " return \"\"\"\n", + " SNP C-SNP D-SNP I-SNP LTSS PACE\n", + " \"\"\"\n", + "\n", + "def PROGRAM_MEDICAID():\n", + " return \"\"\"\n", + " ABD CHIP Foster IDD TANF SSI SSDI\n", + " \"\"\"\n", + "\n", + "def preprocess_text_file(file_path):\n", + " # Read the file\n", + " import chardet\n", + "\n", + " def detect_encoding(file_path):\n", + " with open(file_path, 'rb') as file:\n", + " raw_data = file.read(10000) # Read the first few bytes to guess the encoding\n", + " result = chardet.detect(raw_data)\n", + " return result['encoding']\n", + " encoding = detect_encoding(file_path)\n", + " print(encoding)\n", + "\n", + " with open(file_path, 'r', encoding='utf-8') as file:\n", + " text = file.read()\n", + " \n", + " # Split the text into pages\n", + " pages = re.split(r'Start of Page No\\. = \\d+', text)\n", + " \n", + " # Combine all categories into a single dictionary with regex patterns\n", + " categories = {\n", + " 'METAL_LEVELS': METAL_LEVELS(),\n", + " 'LINE_OF_BUSINESS': LINE_OF_BUSINESS(),\n", + " 'PLANS': PLANS(),\n", + " 'NETWORKS': NETWORKS(),\n", + " 'SERVICE_AREAS': SERVICE_AREAS(),\n", + " 'TYPES_OF_PROVIDER': TYPES_OF_PROVIDER(),\n", + "\n", + " }\n", + " \n", + " # Prepare regex patterns for each category\n", + " category_patterns = {key: re.compile('|'.join(value.split()), re.IGNORECASE) for key, value in categories.items()}\n", + " \n", + " matching_chunks = []\n", + "\n", + " for i, page in enumerate(pages):\n", + " matches = {category: bool(pattern.search(page)) for category, pattern in category_patterns.items()}\n", + " \n", + " # Count the number of categories with at least one match\n", + " match_count = sum(matches.values())\n", + " \n", + " # If 3 or more categories match, consider this page matched\n", + " if match_count >= 3:\n", + " matching_chunks.append((i, page))\n", + " \n", + " return matching_chunks\n", + "\n", + "directory_path = 'texts/training-data/contract-text-file/0. Professional Boilerplate/'\n", + "files = os.listdir(directory_path)\n", + "\n", + "for file_name in files:\n", + " # Construct the full file path\n", + " file_path = os.path.join(directory_path, file_name)\n", + " # Ensure we're working with a file\n", + " if os.path.isfile(file_path):\n", + " try:\n", + " # Process each file\n", + " matching_chunks = preprocess_text_file(file_path)\n", + " print(f\"File: {file_name}\")\n", + " print(matching_chunks)\n", + " print(f\"Number of matching chunks: {len(matching_chunks)}\\n\")\n", + " except Exception as e:\n", + " # Handle potential errors, e.g., file encoding issues, in a general manner\n", + " print(f\"Error processing file {file_name}: {e}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#prompt based search \n", + "#use prompt to pull all section titles, and surrouding texts and then pass to prompt based search to ask about reimbursement/fee schedules, or line of businesses. \n", + "\n", + "#summarize each page and then pass to prompt based search to ask about reimbursement/fee schedules, or line of businesses. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 50, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import dotenv\n", + "\n", + "# Load environment variables\n", + "dotenv.load_dotenv()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Hello! My name is Claude. I'm an AI assistant created by Anthropic.\n", + "I am an artificial intelligence created by a company called Anthropic. I'm an AI assistant, which means I'm designed to help humans with a variety of tasks like answering questions, providing analysis and information, and assisting with creative or problem-solving activities. My knowledge comes from machine learning on a large amount of text data, but I'm not a real person - I'm an AI system without a physical body. I'm here to help in whatever way I can, so please let me know if you have any questions or if there is anything I can assist you with.\n" + ] + } + ], + "source": [ + "import json\n", + "import boto3\n", + "import os\n", + "import dotenv\n", + "import anthropic\n", + "\n", + "# Load environment variables\n", + "dotenv.load_dotenv()\n", + "\n", + "# Constants for model IDs\n", + "MODEL_ID_CLAUDE3 = 'anthropic.claude-3-haiku-20240307-v1:0'\n", + "MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1'\n", + "\n", + "# Initialize Bedrock runtime client\n", + "bedrock_runtime = boto3.client(\n", + " service_name=\"bedrock-runtime\",\n", + " region_name=\"us-east-1\",\n", + " aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),\n", + " aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),\n", + " aws_session_token=os.getenv('AWS_SESSION_TOKEN')\n", + ")\n", + "\n", + "def invoke_claude_2(prompt, max_tokens):\n", + " body = json.dumps(\n", + " {\"prompt\": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, \n", + " \"max_tokens_to_sample\": 1024,\n", + " \"temperature\":0.0,\n", + " \"top_p\":1,\n", + " \"top_k\":250,\n", + " \"stop_sequences\":[anthropic.HUMAN_PROMPT]\n", + " }\n", + " )\n", + "\n", + " response = bedrock_runtime.invoke_model(\n", + " body=body, \n", + " modelId=MODEL_ID_CLAUDE2, \n", + " accept=\"application/json\", \n", + " contentType=\"application/json\"\n", + " )\n", + "\n", + " response_body = json.loads(response.get(\"body\").read())\n", + "\n", + " response_text = response_body['completion']\n", + " return response_text\n", + "\n", + "def invoke_claude_3(prompt, max_tokens = 0):\n", + " prompt = prompt\n", + " body = json.dumps({\n", + " \"anthropic_version\": \"bedrock-2023-05-31\",\n", + " \"max_tokens\": 1000,\n", + " \"messages\": [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\n", + " \"type\": \"text\",\n", + " \"text\":prompt\n", + " }\n", + " ]\n", + " }\n", + " ]\n", + " }\n", + " )\n", + "\n", + " response = bedrock_runtime.invoke_model(\n", + " body=body, \n", + " modelId=MODEL_ID_CLAUDE3, \n", + " accept=\"application/json\", \n", + " contentType=\"application/json\"\n", + " )\n", + "\n", + " response_body = json.loads(response.get(\"body\").read())\n", + "\n", + " return (response_body['content'][0]['text'])\n", + "\n", + "\n", + "# Example usage:\n", + "prompt = \"hello who are you\"\n", + "max_tokens = 1024\n", + "print(invoke_claude_2(prompt, max_tokens))\n", + "print(invoke_claude_3(prompt, max_tokens))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import regex as re\n", + "\n", + "def preprocess_text_file(file_path):\n", + " # Read the file\n", + " with open(file_path, 'r', encoding='utf-8') as file:\n", + " text = file.read()\n", + " \n", + " # Split the text into pages\n", + " pages = re.split(r'Start of Page No\\. = \\d+', text)\n", + " \n", + "\n", + " return pages" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File: 22-3146927_ICMProviderAgreement_273971_1 MU.txt\n", + "Page 1: This document appears to be a Participating Provider Agreement between WellCare New Jersey and healthcare providers for Medicare Advantage and Medicaid/FamilyCare programs. The key points are:\n", + "\n", + "1. The document outlines the contractual terms and conditions between WellCare and the participating providers, including provider-specific requirements, covered services, compensation, and regulatory attachments.\n", + "\n", + "2. The majority of the content consists of legal terminology and regulatory requirements, such as non-discrimination, fraud and abuse, and member financial protections. This indicates that the document contains important legal and compliance information rather than line of business details.\n", + "\n", + "3. There are some sections that provide information on specific services covered under the Medicaid/FamilyCare and Medicare Advantage programs, but these appear to be standard contractual terms rather than strategic business information.\n", + "\n", + "In summary, this document seems to be focused on the legal and regulatory framework governing the relationship between WellCare and its participating providers, rather than containing significant details about the line of business operations.\n", + "Page 2: This document appears to be a participating provider agreement between WellCare Health Plans of New Jersey, Inc. and Rutgers, The State University of New Jersey, on behalf of Robert Wood Johnson Medical School (RWJMS). The key points are:\n", + "\n", + "1. It is a contract between a health plan (WellCare) and a healthcare provider (Rutgers/RWJMS) to provide healthcare services to the health plan's enrollees.\n", + "\n", + "2. The agreement includes specific provisions related to the New Jersey Medicaid/FamilyCare managed care program, which take precedence over any inconsistent language in the rest of the agreement.\n", + "\n", + "3. The document contains extensive legal terminology and definitions, which suggests it is primarily focused on the contractual and regulatory requirements rather than the specifics of the line of business.\n", + "\n", + "Based on the information provided, this document appears to contain important legal and regulatory information related to the contractual relationship between the health plan and the healthcare provider, but does not seem to provide significant details about the line of business operations. It is mostly legal terminology and contractual provisions.\n", + "Page 3: This page appears to contain mostly legal terminology and definitions rather than information directly related to the line of business. The key points are:\n", + "\n", + "1. It defines terms used in the agreement, such as \"Affiliate\", \"Benefit Plan\", \"Clean Claim\", \"Credentialing Criteria\", \"Covered Services\", and \"Federal Health Care Program\".\n", + "2. The definitions provide the legal context and parameters for the agreement, but do not seem to contain any important information about the specific line of business.\n", + "3. The content is focused on establishing the contractual and regulatory framework for the relationship between the parties, rather than discussing the business operations or services.\n", + "4. Overall, this page seems to contain standard legal terminology and definitions typical of a contract or agreement, rather than substantive information about the line of business.\n", + "\n", + "In summary, this page appears to contain legal terminology and definitions, and does not appear to have important information directly related to the line of business.\n", + "Page 4: This page contains important legal terminology and definitions related to a healthcare contract. It does not appear to contain information directly related to a specific line of business. The key points are:\n", + "\n", + "1. Definitions of government authorities, ineligible persons, laws, medically necessary, members, member expenses, overpayments, and participating providers.\n", + "2. The definitions provide the legal context and parameters for the healthcare contract.\n", + "3. This information is typical legal terminology used in healthcare contracts and does not appear to have direct relevance to a specific line of business.\n", + "Page 5: This page appears to contain legal terminology and definitions related to a provider agreement rather than important information about a line of business. The key points are:\n", + "\n", + "1. It defines terms like \"Principal,\" \"Program,\" \"Program Attachment,\" \"Program Requirements,\" \"Provider,\" \"Provider Manual,\" \"State,\" and \"WellCare Companion Guide.\"\n", + "2. It states that providers may freely communicate with members about their treatment, and that Health Plan does not dictate or control clinical decisions.\n", + "3. It clarifies that this is a non-exclusive agreement and there is no guarantee that Health Plan will participate in any particular Program or that any particular Benefit Plan will remain in effect.\n", + "\n", + "Overall, this appears to be standard legal language for a provider agreement and does not contain important information specific to a line of business.\n", + "Page 6: This page appears to contain legal terminology and information related to provider responsibilities under a healthcare plan agreement. The key points are:\n", + "\n", + "1. The health plan reserves the right to create distinct provider networks and determine provider participation.\n", + "2. Providers must comply with requirements for disclosure of ownership, business transactions, and information for persons convicted of crimes against federal healthcare programs.\n", + "3. Providers must maintain and enforce policies and procedures that are consistent with the agreement, both for their employed providers and any subcontracted providers they use.\n", + "4. There are specific information and disclosure requirements for providers to provide to the health plan.\n", + "\n", + "This information seems to be primarily focused on legal and compliance requirements for providers under the healthcare plan agreement, rather than containing important information directly related to the line of business operations.\n", + "Page 7: This page appears to contain important legal and contractual information related to the line of business and operations of the provider. The key points are:\n", + "\n", + "1. It outlines the obligations of the contracted provider, including requirements for subcontracted providers and credentialing of all providers.\n", + "\n", + "2. It specifies the standards and requirements for providing covered services to members, including verifying eligibility and obtaining prior authorization.\n", + "\n", + "3. It discusses the need for providers to comply with laws, regulations, and accreditation standards.\n", + "\n", + "4. The content is primarily legal terminology and contractual language, rather than general business information. This page appears to be an excerpt from a larger agreement or contract between the provider and a health plan.\n", + "\n", + "In summary, this page contains important operational and compliance information for the provider's line of business, rather than just legal terminology.\n", + "Page 8: Based on the summary, this document appears to contain important information related to the line of business and contractual obligations between the health plan and providers. The key points are:\n", + "\n", + "1. The health plan may deny payment for covered services if the provider fails to meet the prior authorization requirements.\n", + "2. Providers are required to furnish complete information to other providers when making referrals.\n", + "3. Providers must inform members and obtain written agreement before providing non-covered services.\n", + "4. The health plan may enter into \"carve-out\" arrangements with third-party providers for certain covered services.\n", + "5. Providers must submit clean claims within a specified time limit and include NPI numbers and taxonomy codes, or the health plan may deny payment.\n", + "\n", + "This document contains important contractual terms and conditions regarding the provision of healthcare services, claims submission, and reimbursement, which are relevant to the line of business.\n", + "Page 9: This document appears to contain legal terminology and provisions related to the contractual relationship between a healthcare provider and a health plan. \n", + "\n", + "The key points are:\n", + "\n", + "1. It outlines requirements for electronic data interchange and claims submission in accordance with HIPAA and the health plan's companion guide.\n", + "\n", + "2. It addresses requirements for electronic funds transfer and remittance advice.\n", + "\n", + "3. It covers coordination of benefits and subrogation responsibilities of the provider.\n", + "\n", + "4. It includes provisions to protect members from being billed for amounts owed by the health plan.\n", + "\n", + "This document does not appear to contain information directly related to the provider's line of business or operations. It is focused on the contractual and regulatory obligations between the healthcare provider and the health plan.\n", + "Page 10: This document appears to be a legal agreement or contract, primarily focused on outlining the terms and requirements for healthcare providers participating in a health plan. It does not contain information directly related to a specific line of business, but rather covers standard contractual provisions such as:\n", + "\n", + "1. Member expenses and billing restrictions\n", + "2. Adherence to the health plan's provider manual and quality improvement programs\n", + "3. Cooperation with utilization management and grievance/appeal processes\n", + "\n", + "Overall, this document contains important legal and operational information for healthcare providers contracted with the health plan, but does not seem to provide any specific insights into a particular line of business. It is focused on the contractual relationship between the health plan and its provider network.\n", + "Page 11: The provided text appears to contain legal terminology and compliance requirements rather than specific information about the line of business. The key points are:\n", + "\n", + "1. Providers must comply with all applicable laws and program requirements, including cooperating with the Health Plan.\n", + "2. Providers must maintain member information and medical records in accordance with privacy laws like HIPAA.\n", + "3. Providers must comply with laws and regulations related to fraud, waste, and abuse prevention.\n", + "4. Providers must comply with the Health Plan's compliance program requirements, including reporting any suspected fraud, waste, or abuse.\n", + "5. Providers must acknowledge that claims and payments under this agreement may be from federal funds.\n", + "\n", + "This information seems to be focused on legal and regulatory compliance rather than details about the line of business. The content does not appear to have significant business-related information.\n", + "Page 12: This document appears to be a legal contract or agreement, and it contains important information related to the line of business. Here is a summary in 5 sentences:\n", + "\n", + "1. The document outlines provisions related to provider payments, including adjustments due to reductions in government funds.\n", + "2. It includes warranty and compliance requirements for the contracted provider, including being an \"Ineligible Person\" and cooperating with compliance audits. \n", + "3. The document discusses licensure and insurance requirements for the contracted provider.\n", + "4. It addresses the confidentiality of \"Proprietary Information\" shared by the health plan with the provider.\n", + "5. Overall, this document contains significant legal and regulatory terms that are important for the line of business, rather than just legal terminology.\n", + "Page 13: This document appears to contain primarily legal terminology and provisions related to a healthcare provider agreement. It does not seem to have significant information directly related to the line of business. The key points are:\n", + "\n", + "1. It covers confidentiality and disclosure requirements for proprietary information.\n", + "2. It outlines the provider's obligation to notify the health plan of certain events that could impact the provider's ability to comply with the agreement.\n", + "3. It describes the health plan's responsibilities, such as issuing member ID cards, claims processing, provider compensation, and medical record review.\n", + "4. It addresses the health plan's right to recover overpayments made to the provider.\n", + "\n", + "Overall, this document appears to be a standard healthcare provider contract and does not seem to contain any critical information about the line of business itself.\n", + "Page 14: This page appears to contain mostly legal terminology and provisions related to healthcare plan administration, rather than important information directly related to the line of business. The key points are:\n", + "\n", + "1. It outlines the process for addressing overpayments made by the health plan to the provider, including the provider's obligation to repay overpayments and the health plan's ability to offset overpayments against future payments.\n", + "\n", + "2. It discusses the health plan's ability to suspend payments to a provider during a government investigation of a credible allegation of fraud.\n", + "\n", + "3. It requires the provider to maintain detailed operational, financial, and administrative records related to the services provided to members, and grants the health plan access to audit those records.\n", + "\n", + "Overall, this appears to be standard contract language outlining the administrative and compliance requirements between the health plan and the provider, rather than information that is directly relevant to the line of business operations.\n", + "Page 15: This document appears to be a legal contract or agreement and contains mainly legal terminology. There is no information directly related to the line of business operations. The key points are:\n", + "\n", + "1. It outlines the requirements for contracted providers to maintain records and provide access/audit to the health plan.\n", + "2. It specifies the term and termination conditions of the agreement, including provisions for termination for convenience or for cause.\n", + "3. The requirements for records, access, and audit survive the expiration or termination of the agreement.\n", + "4. The agreement can be terminated by either party with 90 days' notice (or 180 days for hospital providers).\n", + "\n", + "Overall, this appears to be a standard legal contract governing the relationship between a health plan and its contracted providers, and does not contain any specific business-related information.\n", + "Page 16: This document appears to be a legal contract or agreement between a health plan and a provider. It contains information about the termination and dispute resolution process, as well as requirements for the provider to continue providing services to members during the transition period after termination.\n", + "\n", + "The information in this document is primarily legal terminology and does not seem to contain important information directly related to the line of business. It outlines the contractual obligations and rights of the parties involved, but does not provide any substantive information about the actual healthcare services or operations.\n", + "\n", + "In summary, this document is a standard legal contract and does not appear to have significant information related to the line of business.\n", + "Page 17: This page appears to contain legal terminology and provisions related to dispute resolution and arbitration, rather than important information about the line of business. The key points are:\n", + "\n", + "1. All claims and disputes between the Health Plan and a Provider related to the Agreement must be submitted to arbitration within one year, except for claims based on fraud.\n", + "\n", + "2. Before initiating arbitration, the parties must meet and confer in good faith to seek resolution of the dispute. \n", + "\n", + "3. If the dispute is not resolved within 30 days, either party may submit the dispute to binding arbitration in Newark, New Jersey, through the American Arbitration Association.\n", + "\n", + "4. The arbitration will be conducted by a single arbitrator or a panel of three arbitrators, depending on the amount in dispute.\n", + "\n", + "5. The Agreement is governed by the laws of the State of New Jersey, with federal law applying where New Jersey law is preempted.\n", + "\n", + "In summary, this page appears to contain standard legal provisions for dispute resolution and does not seem to have direct relevance to the line of business.\n", + "Page 18: This page contains legal terminology and information related to the jurisdiction, venue, and dispute resolution procedures for the agreement. It does not appear to contain any important information directly related to the line of business. The key points are:\n", + "\n", + "1. The agreement specifies that any legal proceedings related to the agreement will be handled in state or federal courts located in New Jersey.\n", + "2. The parties waive their right to a jury trial for any disputes.\n", + "3. The parties can seek injunctive or other equitable relief to enforce the agreement.\n", + "4. The relationship between the parties is that of independent contractors, not partners or joint ventures.\n", + "5. The agreement includes restrictions on offshore contracting and out-of-state payments related to certain government healthcare programs.\n", + "\n", + "This page appears to be focused on standard legal provisions governing the agreement, rather than operational or business-related details.\n", + "Page 19: This page contains important information related to the line of business, specifically the following:\n", + "\n", + "1. The agreement is subject to applicable laws, program requirements, and accreditation standards, which are deemed incorporated into the agreement.\n", + "2. The agreement can be amended by the health plan to comply with regulatory requirements, with 30 days' notice to the contracted provider.\n", + "3. Any assignment, delegation, or transfer of the agreement is subject to review and approval by the New Jersey Department of Banking and Insurance and the New Jersey Department of Human Services.\n", + "4. The contracted provider cannot assign, delegate, or transfer the agreement without the prior written consent of the health plan.\n", + "5. The parties cannot use each other's name, symbol, logo, or service mark without prior written approval, except for certain limited purposes.\n", + "\n", + "This page appears to contain legal terminology and contractual provisions that are important for the line of business, rather than just legal boilerplate.\n", + "Page 20: This document appears to contain mostly legal terminology and provisions, rather than information specific to a line of business. The key points are:\n", + "\n", + "1. It discusses payment terms and liability between the Health Plan and its Affiliates.\n", + "2. It includes a \"Force Majeure\" clause that excuses performance under the agreement in the event of certain uncontrollable circumstances.\n", + "3. It covers standard legal provisions such as severability, waiver, the entire agreement, and interpretation of the contract.\n", + "4. Overall, the document seems to be a general contract or agreement template, and does not contain any important information specific to a particular line of business.\n", + "Page 21: This document appears to contain primarily legal terminology and provisions rather than information directly related to the line of business. The key points are:\n", + "\n", + "1. It outlines the rights and remedies of the parties involved, including the cumulative nature of these rights.\n", + "2. It describes the execution of the agreement, including the use of counterparts and electronic signatures.\n", + "3. It includes warranties and representations made by the parties, such as their legal standing and authority to enter into the agreement.\n", + "4. It mentions that the health plan has not delegated any functions to the provider, unless there is a separate delegation schedule or addendum.\n", + "5. It lists the attachments that are incorporated into the agreement, including details on provider requirements, covered services, compensation, and various programs.\n", + "\n", + "Overall, this document seems to be focused on the legal structure and terms of the agreement rather than providing substantive information about the line of business.\n", + "Page 22: This page appears to be a signature page for an agreement between WellCare Health Plans of New Jersey, Inc. and Rutgers, The State University of New Jersey, by and on behalf of Robert Wood Johnson Medical School (RWJMS). It contains the signatures of the parties involved, their respective titles, and the effective date of the agreement. The information on this page is primarily legal terminology related to the execution and effective date of the agreement, and does not appear to contain important information directly related to the line of business. The content is limited to the formalities of signing the agreement.\n", + "Page 23: The provided document appears to be an attachment that outlines provider-specific requirements, covered services, and other relevant information. It is likely a part of a larger contract or agreement. The document seems to contain legal terminology and details specific to the provider and the services they offer, rather than high-level information about the line of business. Based on the information provided, this document seems to be focused on the contractual and operational aspects of the provider's relationship with the organization, rather than providing insights into the broader line of business.\n", + "Page 24: This document appears to contain important information related to the line of business, specifically regarding provider requirements and covered services. The key points are:\n", + "\n", + "1. It defines various terms, such as \"Assigned Member,\" \"Covering Provider,\" \"Nurse Practitioner,\" \"Physician,\" \"Primary Care Provider,\" \"Primary Care Services,\" \"Specialty Provider,\" and \"Specialty Services.\"\n", + "\n", + "2. It states that providers who offer primary care or specialty services have the sole responsibility to acquire and maintain hospital admission privileges, if necessary.\n", + "\n", + "3. It acknowledges that providers who offer primary care or specialty services have a responsibility to ensure 24-hour, 7-day-a-week coverage for their patients.\n", + "\n", + "This document does not appear to contain just legal terminology, but rather important operational and contractual requirements for providers in the line of business.\n", + "Page 25: This page appears to contain important information related to the line of business, specifically the requirements and responsibilities of contracted providers under the agreement. The key points are:\n", + "\n", + "1. Contracted providers must provide or arrange for the provision of all covered services within the scope of their licenses/certifications.\n", + "2. Primary care providers must coordinate the overall healthcare of assigned members, including providing primary care services and making appropriate referrals.\n", + "3. Specialty providers must provide consultation summaries, notify primary care providers of hospital admissions, and seek authorization for hospital admissions (except for emergency services).\n", + "4. Primary care providers must maintain member medical records as required by applicable laws.\n", + "5. Contracted providers must accept members as patients as long as they are accepting new patients, subject to provider-to-patient ratio requirements.\n", + "\n", + "This information appears to be directly relevant to the line of business and contractual obligations of the providers, rather than just general legal terminology.\n", + "Page 26: The provided information appears to be a section from a legal contract or agreement. It discusses a requirement for the provider to give the health plan 60 days' prior notice if the provider is not available to accept members as patients. This seems to be a standard contractual clause related to the provider's obligations and availability, rather than containing any important information about the line of business. The document appears to be focused on legal terminology and contractual provisions.\n", + "Page 27: This document is a Disclosure Statement for Ownership and Control Interest, Related Business Transactions, and Persons Convicted of a Crime, specifically for the New Jersey Medicaid/FamilyCare Program. It appears to be a legal and regulatory document that contains important information related to the line of business.\n", + "\n", + "The key points are:\n", + "\n", + "1. The contracted provider (Rutgers Health Group, Inc.) agrees to provide a complete and accurate disclosure form at the time of initial contracting, upon any changes, and upon request, in accordance with federal and state law.\n", + "\n", + "2. The document requires the disclosure of ownership and control interest, including the name, relationship, and percentage of ownership for the disclosing entity.\n", + "\n", + "3. The document also requires the disclosure of any subcontractors with whom the disclosing entity has a direct or indirect ownership interest of 5% or more.\n", + "\n", + "4. This appears to be a standard regulatory disclosure document required for participation in the New Jersey Medicaid/FamilyCare program, and does not contain general business information.\n", + "Page 28: This page contains legal terminology and requirements for disclosing ownership and control interests in a Medicaid provider, fiscal agent, or managed care entity. It does not appear to have any important information directly related to the line of business. The page outlines the information that needs to be provided, such as names, relationships, ownership percentages, addresses, and tax IDs for individuals and corporations with ownership or control interests. This information is likely required for regulatory or compliance purposes, rather than being directly relevant to the operations or strategy of the business.\n", + "Page 29: The provided text appears to be related to legal and regulatory requirements for disclosing information about managing employees, business transactions, and ownership of subcontractors. It does not seem to contain any information directly related to the line of business. The text outlines the requirements for the Disclosing Entity to provide various information, such as the names, addresses, relationships, dates of birth, and social security numbers of managing employees, as well as details about business transactions and subcontractor ownership. This information is likely required for regulatory and compliance purposes, but does not seem to have significant implications for the line of business operations.\n", + "Page 30: This page appears to contain information related to the line of business, specifically regarding disclosures of business transactions and persons with ownership or control interests who have been convicted of crimes. The key points are:\n", + "\n", + "1. Disclosure of any significant business transactions between the Disclosing Entity and its Wholly Owned Suppliers or Subcontractors during the past 5 years.\n", + "2. Information on types of transactions with \"parties in interest\" as defined in the Public Health Service Act and Social Security Act.\n", + "3. Disclosure of any persons with Ownership or Control Interest, or who are directors, officers, or managing employees of the Disclosing Entity, who have been convicted of a criminal offense related to their involvement in Medicare, Medicaid, CHIP, or Title XX services programs.\n", + "\n", + "This page appears to contain important information related to the line of business, as it covers various disclosures and transparency requirements.\n", + "Page 31: This document appears to be a Disclosure Statement that contains legal terminology and requirements. It does not seem to have any important information related to the line of business. The key points are:\n", + "\n", + "1. It warns against making false statements on the Disclosure Statement, which could result in prosecution or denial/termination of participation.\n", + "2. It requires the disclosure of the requested information, which if not fully and accurately disclosed, could also lead to denial or termination of participation.\n", + "3. The document is signed by Kathleen Bramwell, the Senior Vice Chancellor of Finance & Administration, on 3/2/2021.\n", + "4. The document is related to a contract (#166294) between New Jersey - Rutgers Health Group, dated 8/25/2020.\n", + "\n", + "Overall, this document appears to be a standard legal disclaimer and disclosure requirement, rather than containing any significant information about the line of business.\n", + "Page 32: This page provides definitions related to ownership and control interests in entities, including:\n", + "\n", + "1. Indirect Ownership Interest: An ownership interest in another entity that has an ownership interest in the relevant entity.\n", + "2. Managing Employee: An individual who exercises operational or managerial control over an institution.\n", + "3. Ownership Interest: The possession of equity in the capital, stock, or profits of an entity.\n", + "4. Ownership or Control Interest: Criteria for determining when a person or corporation has a 5% or more ownership or control interest in an entity.\n", + "5. Subcontractor: An individual, agency, or organization to which an entity has contracted or delegated some of its management functions or responsibilities.\n", + "6. Supplier: An individual, agency, or organization from which a provider purchases goods and services.\n", + "7. Wholly Owned Supplier: A supplier whose total ownership interest is held by a provider or by a person, persons, or other entity with an ownership or control interest in a provider.\n", + "\n", + "This information appears to be primarily legal terminology and definitions, and does not seem to contain important information directly related to the line of business.\n", + "Page 33: This page appears to be focused on providing information about contracted healthcare providers, including their names, addresses, contact details, professional licenses, identification numbers, and other relevant details. It does not seem to contain any information directly related to the line of business. Instead, it appears to be a list of the required information that healthcare providers need to provide as part of their contract or agreement. This information is likely important for regulatory and administrative purposes, but it does not directly impact the line of business operations. Overall, this page seems to be primarily focused on legal and compliance-related terminology and requirements, rather than business-specific information.\n", + "Page 34: This document appears to be an attachment to a larger contract, specifically \"PROGRAM ATTACHMENTS\" for a contract between New Jersey and Rutgers Health Group dated August 25, 2020. The document does not seem to contain any important information related to the line of business, but rather consists of legal terminology and attachments that are typically included in a contract. Based on the limited information provided, this appears to be a standard legal document with no significant business-related content.\n", + "Page 35: This document appears to be a regulatory attachment related to the New Jersey Medicaid/FamilyCare Program. It does not contain any information directly related to a specific line of business. The document outlines the following:\n", + "\n", + "1. This attachment is incorporated into the Participating Provider Agreement between the Health Plan and the Contracted Provider to govern the provision of covered services to Medicaid/FamilyCare Program members.\n", + "\n", + "2. It defines key terms such as \"Appeal\", \"Clean Claim\", \"Covered Services\", and references various New Jersey state agencies involved in the Medicaid/FamilyCare Program.\n", + "\n", + "3. The attachment is intended to supplement the main Participating Provider Agreement and take precedence over any inconsistent provisions in the agreement, specifically with respect to the Medicaid/FamilyCare Program.\n", + "\n", + "4. The document does not appear to contain any substantive information about a specific line of business, but rather legal and regulatory details regarding the Medicaid/FamilyCare Program in New Jersey.\n", + "\n", + "In summary, this document appears to be a regulatory attachment with legal terminology rather than information directly relevant to a line of business.\n", + "Page 36: This document appears to contain legal terminology and definitions related to a health insurance contract. It does not seem to have any direct information about a specific line of business. The key points are:\n", + "\n", + "1. It defines terms like \"Emergency Medical Condition\", \"Emergency Services\", \"Grievance\", \"Medicaid/FamilyCare Program\", \"Medical Necessity\", \"Member\", \"New Jersey Contract\", and \"Plan of Care\".\n", + "\n", + "2. These definitions are likely important for understanding the rights, responsibilities, and processes outlined in the health insurance contract.\n", + "\n", + "3. The document does not appear to contain any substantial information about the line of business or operations of the organization. It is focused on legal and regulatory terms.\n", + "\n", + "4. Overall, this seems to be a section of a larger contract or agreement, providing definitions of key terms rather than substantive information about the business.\n", + "\n", + "5. The content is heavily focused on legal and regulatory language, rather than details about the line of business.\n", + "Page 37: This page appears to contain legal terminology and information related to the submission, acknowledgement, and payment of claims under the agreement. It does not seem to have any important information directly related to the line of business. The key points are:\n", + "\n", + "1. The agreement follows the Health Information Electronic Data Interchange Act (HINT) requirements for claim submission and payment timelines.\n", + "2. Contracted providers must file claims within specific timeframes, and Health Plan has the right to deny claims filed outside those timeframes.\n", + "3. Contracted providers must hold members harmless for the cost of any covered services or supplies, and cannot balance bill members.\n", + "4. Health Plan's agreement with secondary contractors must include a provision requiring the contractors' network providers to hold members harmless.\n", + "\n", + "Overall, this page appears to contain standard legal language related to claims processing and member protections, rather than any specific information about the line of business.\n", + "Page 38: This document appears to be focused on the legal and regulatory aspects of the agreement between the health plan and the contracted provider. It outlines the requirements for termination of the agreement, including the notice period, the provider's right to a hearing, and the obligations of the provider to continue providing services to members for a certain period after termination. \n", + "\n", + "The information provided does not seem to contain important details about the line of business or operations of the parties involved. It is primarily focused on the legal and regulatory framework governing the contractual relationship between the health plan and the provider.\n", + "\n", + "In summary, this document appears to be mostly legal terminology and requirements, with no significant information related to the line of business of the parties involved.\n", + "Page 39: The summary of the given page is as follows:\n", + "\n", + "1. The page outlines the grievances and appeals procedures that must be followed, which comply with applicable laws as set forth in the Provider Manual.\n", + "2. It discusses the process for recoupment of overpayments and the resolution of payment disputes between the provider and the payor.\n", + "3. The page highlights the insurance coverage requirements for different types of healthcare providers, including minimum limits for professional malpractice and general liability insurance.\n", + "4. For healthcare facility providers, the page outlines specific procedures that must be followed regarding granting and attending privileges to physicians, as well as the admissions procedures for members.\n", + "\n", + "This page contains important information related to the legal and regulatory requirements for healthcare providers, but does not appear to have any specific details regarding the line of business.\n", + "Page 40: This document appears to contain important information related to the line of business, specifically the provider's agreement with the health plan. The key points are:\n", + "\n", + "1. The provider agrees to admit members to the hospital under certain conditions, and to notify the health plan when members present at the emergency room.\n", + "2. The document outlines billing and payment procedures, including timeframes for submitting clean claims and the health plan's obligation to pay providers.\n", + "3. The provider must conduct criminal background checks on employees and agents providing direct care to managed long-term services and supports (MLTSS) members.\n", + "4. The provider must comply with requirements for reporting critical incidents and notifying the health plan of substance abuse admissions.\n", + "5. The provider must comply with the health plan's training and education program.\n", + "\n", + "This appears to be a legal contract or agreement between the provider and the health plan, and contains important operational and compliance requirements for the provider.\n", + "Page 41: This page appears to be a contract between a healthcare provider and a health plan. It contains information about the provider's obligation to report any instances where they are unable to contact an MLTSS (Managed Long-Term Services and Supports) member, in accordance with the Provider Manual. This information is likely related to the line of business and the operational requirements of the provider's relationship with the health plan. The content does not seem to be primarily focused on legal terminology, but rather on the contractual obligations and procedures that the provider must follow.\n", + "Page 42: This page appears to contain important information related to the line of business, specifically the New Jersey Medicaid/FamilyCare Program. The key points are:\n", + "\n", + "1. The agreement/subcontract is subject to the terms and conditions of the contract between the Health Plan and the State of New Jersey.\n", + "2. The document outlines the \"Any Willing Provider\" and \"Any Willing Plan\" provisions for certain Medicaid providers in New Jersey, such as nursing facilities, assisted living providers, and long-term care pharmacies.\n", + "3. The \"Any Willing Provider\" status for these providers is in effect until the end of State Fiscal Year 2021 (June 30, 2021), after which the Health Plan can determine their network status based on utilization and access needs.\n", + "4. The \"Any Willing Plan\" status also expires on June 30, 2021.\n", + "\n", + "This document appears to contain important contractual and regulatory information related to the line of business, rather than just legal terminology.\n", + "Page 43: This document appears to contain important information related to the line of business, specifically regarding claims processing and payment for MLTSS (Managed Long-Term Services and Supports) members. The key points are:\n", + "\n", + "1. It outlines the timeframes for processing claims for various MLTSS services, such as assisted living providers, nursing facilities, and personal care services.\n", + "2. It mentions the Nursing Facility Quality Incentive Payment Program (NF QIPP), which replaces the previous Any Willing Qualified Provider (AWQP) initiative and focuses on quality outcomes for long-stay Medicaid residents in nursing facilities.\n", + "3. The document also emphasizes the need for the contracted provider/subcontractor to comply with federal and state laws and regulations, including the submission of claims within specific timeframes.\n", + "\n", + "Overall, this document appears to contain important operational and compliance information related to the line of business, rather than just legal terminology.\n", + "Page 44: This document appears to be related to a medical provider contract or subcontract. The key points are:\n", + "\n", + "1. The State reserves the right to review and approve or disapprove the agreement/subcontract and any amendments.\n", + "\n", + "2. The agreement/subcontract can only be amended unilaterally for statutory and regulatory changes, and upon mutual consent with State approval.\n", + "\n", + "3. The agreement/subcontract becomes effective when the health plan's agreement with the State takes effect.\n", + "\n", + "4. The health plan must notify the State at least 30 days prior to suspending, terminating, or the provider withdrawing from the network, and if the termination was \"for cause\" related to fraud, waste and abuse, the reasons must be provided.\n", + "\n", + "This appears to be focused on legal and regulatory requirements related to the provider contract, rather than specific details about the line of business. It does not seem to contain important information directly related to the line of business.\n", + "Page 45: This page appears to contain important information related to legal and contractual terms between a healthcare provider and a health plan. The key points are:\n", + "\n", + "1. The provider must participate in and cooperate with the health plan's quality and utilization review programs, as long as they are based on clinical evidence and do not restrict provider-patient communications.\n", + "\n", + "2. The health plan is restricted from terminating the agreement/subcontract if the provider expresses disagreement with the health plan's decisions or discusses treatment options with patients.\n", + "\n", + "3. The state may order termination of the agreement if the provider takes actions that threaten the health, safety, or welfare of members, or the fiscal integrity of the Medicaid program.\n", + "\n", + "This document seems to be focused on contractual and legal requirements, rather than specific line of business information.\n", + "Page 46: This page appears to contain legal terminology and requirements rather than important information related to the line of business. The key points are:\n", + "\n", + "1. It outlines the conditions under which the contracted provider/subcontractor's agreement may be terminated, such as suspension or revocation of certification, insolvency, bankruptcy, material breach of the agreement, or violations of state/federal law.\n", + "\n", + "2. It describes the contracted provider/subcontractor's obligations regarding non-discrimination, including that they must accept Medicaid/FamilyCare Program beneficiaries and comply with the Americans with Disabilities Act (ADA).\n", + "\n", + "3. The contracted provider/subcontractor is required to submit a written certification that they are in compliance with the ADA and will hold the state harmless from any liability related to non-compliance.\n", + "\n", + "Overall, this page seems to be focused on legal and compliance requirements rather than business-critical information.\n", + "Page 47: This document appears to contain legal terminology and information related to non-discrimination policies, rather than specifics about the line of business. The key points are:\n", + "\n", + "1. The contracted provider/subcontractor must abide by the provisions of the Rehabilitation Act of 1973 regarding access to programs and facilities by people with disabilities.\n", + "\n", + "2. The contracted provider/subcontractor shall not discriminate against eligible persons or members based on various factors like health status, need for services, or pre-existing conditions. \n", + "\n", + "3. The contracted provider/subcontractor must comply with various civil rights laws and regulations prohibiting discrimination on grounds of age, race, disability, sex, etc.\n", + "\n", + "4. The contracted provider/subcontractor agrees to forward any grievances alleging discrimination against members to the health plan for review and action.\n", + "\n", + "5. The contracted provider/subcontractor remains obligated to provide services for the duration of the period after the health plan's insolvency, and shall not bill, charge, or seek compensation from the members under any circumstances.\n", + "\n", + "Overall, this appears to be legal and contractual language about non-discrimination policies and obligations, rather than information directly related to the line of business.\n", + "Page 48: This page appears to contain legal terminology and provisions related to the contracted relationship between a health plan and a contracted provider/subcontractor. It does not seem to contain any important information directly related to the line of business. The key points are:\n", + "\n", + "1. The contracted provider/subcontractor agrees not to seek compensation or reimbursement from members for covered services, except as provided in the contract.\n", + "2. This provision survives the termination of the agreement and supersedes any contrary agreements between the provider/subcontractor and members.\n", + "3. The state, CMS, and other government entities have the right to inspect and audit records and facilities related to the contract.\n", + "4. There are specific types of records that can be inspected, including financial, medical, and administrative documents.\n", + "\n", + "Overall, this page appears to be focused on contractual and regulatory requirements rather than the line of business operations.\n", + "Page 49: This page appears to contain legal terminology and requirements related to record maintenance, retention, and provider/subcontractor documentation. The key points are:\n", + "\n", + "1. Contracted providers/subcontractors must maintain records fully disclosing the nature and extent of services provided to Medicaid recipients, in accordance with New Jersey regulations.\n", + "\n", + "2. Records must be retained for at least 10 years from the date of service or after the final payment, and must be accessible in New Jersey for review by state and federal agencies.\n", + "\n", + "3. Specific requirements are outlined for documentation of medical supplies, equipment, and DME, including the need for a dated prescription or Certificate of Medical Necessity.\n", + "\n", + "This information seems to be focused on legal and regulatory compliance requirements, rather than directly related to the line of business operations. The content appears to be important for ensuring proper documentation and record-keeping practices for Medicaid providers and contractors.\n", + "Page 50: The given text appears to be a legal document that outlines the requirements for medical equipment and laboratory test orders in New Jersey's Medicaid/NJ FamilyCare program. It does not contain any information specifically related to line of business. The key points are:\n", + "\n", + "1. Orders for medical equipment and supplies must include detailed information about the item, the patient's condition, and the prescriber's details.\n", + "2. Orders for laboratory tests must be patient-specific, contain the tests requested, and be available for review by Medicaid/NJ FamilyCare representatives.\n", + "3. Alternative forms of orders, such as electronic orders or orders based on the patient's medical record, are also accepted if they meet certain security and documentation requirements.\n", + "\n", + "Overall, this document appears to be focused on regulatory and legal requirements related to medical orders, rather than line of business information.\n", + "Page 51: The provided document appears to contain legal terminology and requirements related to laboratory orders and procedures. It outlines the specific information that must be included in laboratory orders, such as the name and address of the authorized person requesting the test, the patient's information, the specific tests to be performed, the source of the specimen, and other relevant details. The document also discusses standing orders and the requirements for them. Overall, this document seems to be focused on legal and regulatory requirements for laboratory orders and procedures, and does not appear to contain information directly related to a line of business.\n", + "Page 52: This page appears to contain important information related to legal and regulatory requirements for billing and record-keeping in the provision of medical services to Medicaid/NJ FamilyCare members. The key points are:\n", + "\n", + "1. Laboratories providing tests to Medicaid/NJ FamilyCare members must maintain detailed records of all orders, test results, and clinical and billing information, which must be available for review.\n", + "2. Medicaid/NJ FamilyCare has the right to inspect all records of in-state and out-of-state service and reference laboratories.\n", + "3. All laboratory test orders must be supported by documentation in the referring provider's medical records.\n", + "4. Psychologists providing services to Medicaid/NJ FamilyCare members must keep detailed individual records of the services provided, which must be made available to the Medicaid/NJ FamilyCare program upon request.\n", + "\n", + "This information appears to be directly relevant to the line of business and the regulatory requirements for providers participating in the Medicaid/NJ FamilyCare program.\n", + "Page 53: The given text appears to be legal terminology and guidelines related to mental health services provided by an independent clinic. It outlines the required components of a psychologist's treatment notes, such as the duration of service, modality used, progress notes, and other clinical details. It also specifies the requirements for an intake evaluation, including the assessment of the member's mental condition, the appropriateness of the treatment program, and the certification by the evaluation team. This information is related to the legal and regulatory aspects of providing mental health services, rather than specific business operations or line of business.\n", + "Page 54: This document appears to be primarily focused on legal terminology and requirements for mental health treatment plans and documentation, rather than containing important information directly related to a specific line of business. The key points are:\n", + "\n", + "1. A written, individualized Plan of Care must be developed for each patient receiving continued treatment, with specific details on treatment objectives, services, and schedules.\n", + "2. The mental health clinic must maintain detailed documentation to support each service provided, including the type of service, date, duration, provider signature, and any unusual occurrences or deviations from the treatment plan.\n", + "3. There may be instances where a temporary deviation from the written treatment plan occurs, which must be clearly explained in the documentation.\n", + "4. The document does not seem to contain information directly relevant to a specific line of business, but rather outlines general legal and regulatory requirements for mental health treatment plans and documentation.\n", + "Page 55: The provided text appears to contain detailed legal and regulatory requirements related to documentation and record-keeping for healthcare services provided to members or patients. The information seems focused on the specific requirements for clinical progress notes, treatment plans, and periodic reviews, as well as the documentation standards for Advanced Practice Nurse (APN) services. The content is primarily concerned with ensuring proper documentation and compliance with applicable rules and regulations, rather than providing information directly related to a particular line of business. Based on the level of detail and the nature of the content, this appears to be a legal or regulatory document, rather than a business-focused resource.\n", + "Page 56: This page appears to contain information related to documentation requirements for billing and reimbursement purposes, rather than direct information about a line of business. The key points are:\n", + "\n", + "1. It outlines the documentation required for an initial visit, including the chief complaint, history, physical exam, diagnosis, and plan of care.\n", + "2. It specifies the documentation required for routine office visits or follow-up care, including the patient's chief complaint, relevant history and physical findings, diagnostic tests, and diagnosis.\n", + "3. The documentation requirements differ slightly for visits conducted in an office/residential setting versus a hospital/nursing facility.\n", + "4. This seems to be primarily legal terminology related to billing and reimbursement procedures, rather than direct information about a specific line of business.\n", + "Page 57: This page appears to contain legal terminology and requirements related to the documentation of services provided by Advanced Practice Nurses (APNs) during inpatient stays and Early Periodic Screening, Diagnosis, and Treatment (EPSDT) examinations. The information covers the specific details that must be documented in the medical record to qualify for reimbursement, such as reviewing the patient's medical history, performing a physical examination, confirming or revising the diagnosis, and demonstrating the APN's personal involvement in the service rendered. This seems to be important information related to compliance and reimbursement requirements, but does not appear to contain significant information about a specific line of business.\n", + "Page 58: This page appears to contain legal terminology and requirements related to medical record-keeping practices for physicians. The information is not directly related to the line of business, but rather outlines the necessary documentation and record-keeping procedures that physicians must adhere to when providing services to patients. The summary includes details on the specific information that must be included in progress notes, the storage and retention of records, and the requirement to make records available to the New Jersey Medicaid/NJ FamilyCare program upon request. Overall, this page does not contain important information related to the line of business, but rather focuses on regulatory and compliance-related requirements for medical record-keeping.\n", + "Page 59: This page appears to contain detailed requirements for the minimum documentation that must be entered in a patient's medical record, depending on whether the patient is a new member or an established member. The information seems to be focused on legal and regulatory requirements rather than specifics about a line of business. It does not appear to contain important information directly related to a line of business, but rather outlines the necessary documentation standards that must be met. The content is mainly legal terminology and formatting.\n", + "Page 60: This document appears to be outlining the minimum documentation requirements for various medical services and settings, including:\n", + "\n", + "1. Laboratory, X-Ray, ECG, and other diagnostic tests with results\n", + "2. Home visits and house calls, including treatment plan status and recommendations\n", + "3. Hospital or nursing facility visits, including updates on symptoms, findings, procedures, and treatment changes\n", + "4. Hospital discharge medical summaries, including a summary of the patient's hospitalization and recommendations for further care\n", + "5. Mental health services, including written documentation to support the medical necessity of the services provided\n", + "\n", + "This information seems to be focused on the legal and regulatory requirements for medical record documentation, rather than directly related to a specific line of business. It does not appear to contain high-level strategic or operational information about the organization's business activities.\n", + "Page 61: This page appears to contain legal terminology and requirements related to healthcare and medical services documentation, rather than information directly related to a line of business. The key points are:\n", + "\n", + "1. Detailed documentation is required for each medical or remedial therapy, service, activity, or session, including the specific services rendered, date and time, duration, physician's signature, setting, and other relevant clinical information.\n", + "\n", + "2. Clinical progress, complications, and treatment that affect prognosis and progress must be documented in the member's medical record.\n", + "\n", + "3. For mental health services not included in the treatment regime, a detailed explanation must be submitted with the claim form.\n", + "\n", + "4. Pharmacies are required to maintain purchase records for prescription drugs and medical supplies for a minimum of 10 years.\n", + "\n", + "This information seems to be focused on legal and regulatory requirements for healthcare documentation and record-keeping, rather than directly related to a specific line of business.\n", + "Page 62: This page primarily contains legal terminology and requirements related to pharmacies, drug dispensing, and data reporting. The key points are:\n", + "\n", + "1. Pharmacies must maintain detailed records of drug purchases, including price, drug name, dosage, NDC, lot number, and quantity.\n", + "2. Pharmacies must obtain drugs from authorized, regulated sources and provide product tracing information to the state upon request.\n", + "3. Pharmacies must have the specific drug in stock before submitting a claim and use the correct NDC on the claim.\n", + "4. Pharmacies must keep records of compounded prescriptions, including all ingredients used.\n", + "5. Pharmacies must maintain detailed records of any inventory transfers between pharmacies and provide this information to the state.\n", + "\n", + "This information appears to be focused on compliance and regulatory requirements for pharmacies, rather than directly related to a specific line of business.\n", + "Page 63: This page appears to contain legal terminology and requirements rather than information directly related to the line of business. The key points are:\n", + "\n", + "1. The contracted provider/subcontractor must comply with the prohibition on the use of federal funds for lobbying.\n", + "2. The contracted provider/subcontractor must comply with financial disclosure requirements.\n", + "3. The contracted provider/subcontractor cannot impose cost-sharing charges on Medicaid or Medicaid/FamilyCare members.\n", + "4. The contracted provider/subcontractor agrees to indemnify and hold harmless the state, its officers, agents, employees, and members from various claims and liabilities.\n", + "\n", + "Overall, this page seems to outline legal obligations and requirements for the contracted provider/subcontractor rather than specific details about the line of business.\n", + "Page 64: This page appears to contain legal terminology and requirements related to confidentiality and indemnification, which are important for the contractual relationship between the parties involved. The key points are:\n", + "\n", + "1. The contracted provider/subcontractor must indemnify the state and its members from any injury, losses, or damages arising from the provider's negligence or violations of state/federal laws.\n", + "\n", + "2. The contracted provider/subcontractor must maintain the confidentiality of all information related to the members, the state, and the health plan in accordance with applicable laws and regulations.\n", + "\n", + "3. The contracted provider/subcontractor must protect the confidentiality of member-specific information and only use it for the purpose of the agreement.\n", + "\n", + "4. Employees of the contracted provider/subcontractor must be instructed to keep all confidential information related to the state, its members, and its operations.\n", + "\n", + "This information appears to be focused on legal requirements and compliance, rather than directly on the line of business operations. It sets forth important contractual obligations and restrictions for the contracted provider/subcontractor.\n", + "Page 65: This document appears to contain important information related to legal and compliance requirements for healthcare providers and subcontractors. The key points are:\n", + "\n", + "1. It outlines confidentiality and privacy requirements for handling member information, with provisions that survive contract termination.\n", + "2. It requires contracted providers/subcontractors to comply with federal and state laws related to data breach notification.\n", + "3. It mandates that all laboratory sites have the proper Clinical Laboratory Improvement Amendment (CLIA) certificates.\n", + "4. It obligates contracted providers/subcontractors to assist the health plan in identifying, investigating, and addressing fraud, waste, and abuse.\n", + "5. It allows the health plan to withhold payments or forward them to the state if the state has taken such actions against the contracted provider/subcontractor.\n", + "\n", + "This document appears to contain legal and compliance information rather than details about the line of business operations.\n", + "Page 66: This page appears to contain legal terminology and provisions related to the responsibilities of the Health Plan and the Medicaid Fraud Division (MFD) regarding audits, investigations, and recovery of funds from providers and members. It also includes provisions related to third-party liability and the circumstances under which a provider can bill the Health Plan first before coordinating with a liable third party.\n", + "\n", + "This information seems to be more focused on legal and operational requirements rather than directly related to the line of business. The content does not appear to contain any significant strategic or competitive insights.\n", + "Page 67: This page appears to contain important legal information related to third-party liability (TPL) and member protections against liability for payment. Specifically:\n", + "\n", + "1. It discusses the circumstances under which a contracted provider/subcontractor can bill the health plan without receiving a written denial from a third party.\n", + "2. It outlines the provider/subcontractor's obligations to notify the health plan about changes in a member's health insurance coverage, retained legal counsel, and the death of a member aged 55 or older.\n", + "3. It explains the general rule that providers cannot seek payment from the member, with some exceptions, such as when the service is not a covered service or is determined to be not medically necessary.\n", + "4. Overall, this page seems to contain important legal terminology and requirements related to the line of business, rather than just general legal terminology.\n", + "Page 68: This page appears to contain legal terminology and requirements related to healthcare provider agreements and Medicaid/Medicare reimbursement. It does not seem to have any important information directly related to the line of business. The key points are:\n", + "\n", + "1. Conditions under which a member can be billed directly by a provider, such as when the service is not covered or the member has failed to remit payment from a third-party insurer.\n", + "2. Circumstances where a member cannot be held responsible for the cost of care, such as emergency services or referrals to non-participating specialists.\n", + "3. A requirement that all services under the agreement be performed within the United States.\n", + "4. A prohibition on further delegation of any delegated activities.\n", + "\n", + "Overall, this page appears to be focused on legal and regulatory compliance rather than business operations.\n", + "Page 69: This document appears to be an attachment to a Medicare Advantage Program contract. The key points are:\n", + "\n", + "1. It outlines the network participation requirements for providers to provide covered services to members enrolled in Medicare Advantage (MA) plans and dual-eligible demonstration models.\n", + "\n", + "2. It defines various terms related to the MA program, such as \"CMS Contract,\" \"Deemed Eligible Member,\" \"Deeming Period,\" \"DSNP,\" \"Dual Eligible Member,\" \"Emergency Medical Condition,\" and \"Emergency Services.\"\n", + "\n", + "This document contains important information related to the line of business, specifically the Medicare Advantage program. It does not appear to be just legal terminology, but rather operational details for providers participating in the MA program.\n", + "Page 70: This document appears to contain legal terminology and details related to the Medicare Advantage (MA) program, rather than specific information about a line of business. The key points are:\n", + "\n", + "1. Definitions of terms related to the MA program, such as \"MA Benefit Plan\", \"Medically Necessary\", and \"Member\".\n", + "2. Listing of the laws, regulations, and program requirements that apply to the MA program, including the Social Security Act, 42 CFR Parts 422 and 423, and various CMS guidance and bulletins.\n", + "3. The information provided seems to be more focused on the legal and regulatory framework of the MA program, rather than details about a specific line of business.\n", + "\n", + "Based on the content, this document does not appear to contain important information related to a specific line of business, but rather focuses on the legal and regulatory aspects of the Medicare Advantage program.\n", + "Page 71: This page appears to contain mostly legal terminology and provisions related to the agreement between the parties involved. It does not seem to have any significant information directly related to the line of business. The key points are:\n", + "\n", + "1. The provisions in this attachment prevail over any inconsistent or contrary language in the main agreement, except where the agreement exceeds the minimum requirements of the attachment.\n", + "\n", + "2. The health plan is entitled to oversee the activities of the contracted provider and its subcontractors, and is accountable to CMS for such activities. \n", + "\n", + "3. The contracted provider and its providers must comply with all applicable Medicare laws, rules, regulations, and CMS instructions.\n", + "\n", + "4. The contracted provider and its providers must allow audits by CMS or its designees, maintain records for a minimum of 10 years, and comply with the health plan's policies and procedures.\n", + "\n", + "5. When assisting members with enrollment decisions, providers must remain neutral and provide an objective assessment of available options that are in the member's best interest.\n", + "\n", + "Overall, this page appears to contain standard legal and compliance-related provisions, rather than information directly pertaining to the line of business.\n", + "Page 72: This page appears to contain important information related to legal and compliance requirements for a contracted provider. The key points are:\n", + "\n", + "1. The contracted provider and its providers/subcontractors must maintain records related to the CMS contract for 10 years from the end of the contract period or completion of any audit, whichever is later.\n", + "\n", + "2. The health plan, DHHS, Comptroller General, or their designees have the right to audit, evaluate, collect, and inspect any records related to the CMS contract directly from the contracted provider and its subcontractors.\n", + "\n", + "3. The contracted provider and its subcontractors must make their premises, facilities, equipment, records, and other relevant information available for inspection, evaluation, and auditing by the health plan, DHHS, Comptroller General, or their designees.\n", + "\n", + "4. The contracted provider must ensure the privacy and accuracy of member records, including abiding by all federal and state laws regarding confidentiality and disclosure of medical and enrollment information.\n", + "\n", + "This page contains important legal and compliance information, but does not appear to have direct relevance to the line of business operations.\n", + "Page 73: This page appears to contain legal terminology and information related to compliance requirements for a healthcare provider contract. The key points are:\n", + "\n", + "1. Providers must ensure medical information is released only as per applicable laws and regulations.\n", + "2. Providers cannot require prior authorization for emergency services until the patient is stabilized.\n", + "3. Providers cannot hold members liable for fees that are the legal obligation of the health plan, including in cases of insolvency.\n", + "4. Providers must hold dual-eligible and deemed-eligible members harmless from being billed for Medicare Part A and B expenses.\n", + "\n", + "This information seems focused on legal and regulatory requirements for healthcare providers, and does not appear to contain important details about the line of business itself.\n", + "Page 74: This document outlines various contractual terms and obligations between a health plan and a contracted provider for Medicare Part A or B services. The key points are:\n", + "\n", + "1. The contracted provider must comply with the health plan's contractual obligations under CMS (Centers for Medicare & Medicaid Services) contracts.\n", + "\n", + "2. The health plan must promptly pay clean claims submitted by the contracted provider.\n", + "\n", + "3. The contracted provider must continue to provide covered services to members even after the agreement's expiration or termination.\n", + "\n", + "4. The contracted provider must ensure that service hours and availability do not discriminate against Medicare enrollees.\n", + "\n", + "This document mainly contains legal terminology and contractual requirements, and does not appear to have significant information directly related to the line of business. It outlines the obligations and expectations between the health plan and the contracted provider for Medicare-related services.\n", + "Page 75: This document appears to contain legal terminology and requirements related to the operation of a healthcare plan, rather than information directly relevant to a specific line of business. The key points are:\n", + "\n", + "1. The document outlines requirements for providers, contractors, and subcontractors to comply with Medicare program rules, including maintaining a compliance program and allowing for reporting of potential issues.\n", + "\n", + "2. It specifies that the healthcare plan retains the right to approve, suspend, or terminate any arrangements the provider makes with other providers, contractors, or subcontractors.\n", + "\n", + "3. It addresses requirements for delegating any services or activities to other entities, including the need for written agreements and ongoing monitoring by the healthcare plan.\n", + "\n", + "4. It includes a requirement for the healthcare plan to provide a level of payment to Federally Qualified Health Centers (FQHCs) that is not less than the amount paid to other similar providers.\n", + "\n", + "Overall, this document appears to be focused on regulatory and compliance requirements rather than specific business operations, and does not seem to contain important information directly related to a particular line of business.\n", + "Page 76: This page appears to be an attachment to a contract, specifically related to the compensation details. It does not contain any information directly related to the line of business or operations. The content is primarily legal terminology and contract language, detailing the compensation structure and any associated terms or conditions. There is no significant information here that would be considered important for understanding the core business activities or operations.\n", + "Page 77: The page outlines the compensation rates and payment terms for professional services provided to Medicaid/FamilyCare members in New Jersey under the agreement. The key points are:\n", + "\n", + "1. The compensation rates are based on the CMS Medicare physician fee schedule, with some exceptions for VFC vaccine administration and EPSDT screenings.\n", + "2. Health Plan is required to process and pay or deny clean claims within specific timeframes.\n", + "3. Health Plan will implement changes to the CMS Medicare fee schedules on a prospective basis.\n", + "4. Health Plan may implement successor codes for deleted or retired codes and determine rates for covered services not included in the CMS Medicare fee schedules.\n", + "\n", + "This appears to contain important information related to the line of business, as it outlines the specific compensation and payment terms for providing professional services to Medicaid/FamilyCare members in New Jersey.\n", + "Page 78: The provided text appears to be a section of a legal contract or agreement. It discusses the basis for determining the amount of compensation for a treating provider, which is based on the provider's licensure and the health plan's credentialing requirements for that discipline, rather than the provider's academic credentials. This information seems to be related to legal and contractual terms, rather than specific details about the line of business. The summary can be provided in 5 sentences as requested.\n", + "Page 79: This page contains information related to the Medicare Advantage Program Compensation for professional (fee-for-service) services. The key points are:\n", + "\n", + "1. The compensation rates in this attachment apply for Benefit Plans under CMS Contracts.\n", + "2. Compensation for professional Covered Services is based on the lesser of the Provider's usual and customary billed charges or a percentage of the CMS Medicare physician fee schedule.\n", + "3. The Health Plan will process and pay or deny Clean Claims within 45 days of receipt.\n", + "4. The Health Plan will implement changes to CMS's fee schedules on the later of certain specified dates.\n", + "5. The Health Plan may implement successor codes for deleted or retired codes, and will determine rates for Covered Services not included in the CMS Medicare fee schedule.\n", + "\n", + "This appears to be primarily legal terminology related to the compensation structure for providers under the Medicare Advantage Program, rather than containing important information about a specific line of business.\n", + "Page 80: The page provided contains information related to the compensation paid to providers based on their licensure and the credentialing requirements of the health plan, rather than their academic credentials. This information appears to be related to the line of business, as it provides details on the criteria used to determine the amount of compensation for healthcare providers. The page does not contain any significant legal terminology, and the information seems to be focused on the business aspects of provider compensation.\n", + "Page 81: This page appears to be an attachment titled \"List of Programs\" related to a contract between New Jersey and Rutgers Health Group. It lists various programs and their corresponding contract numbers, but does not seem to contain any information directly related to a specific line of business. The content appears to be more focused on legal terminology and contractual details rather than substantive business information.\n", + "\n", + "In summary, this page seems to provide a list of program details and contract information, but does not appear to have any important information related to a specific line of business. It appears to be primarily legal in nature.\n", + "Page 82: This document appears to be a confidentiality agreement between two parties. It outlines the obligations of both parties to maintain the confidentiality of information shared during the course of their business relationship. The agreement covers topics such as the definition of confidential information, the permitted use of such information, and the consequences of unauthorized disclosure. While this document contains legal terminology, it does not appear to have any specific information related to the line of business or operations of the parties involved. It is a standard confidentiality agreement that is commonly used in a variety of business contexts.\n", + "Processed 82 pages\n", + "\n" + ] + } + ], + "source": [ + "import os\n", + "import regex as re\n", + "\n", + "def run_prompt_preprocess(pages, max_tokens=512):\n", + "\n", + " summaries = {}\n", + " for i, page_text in enumerate(pages):\n", + " prompt = (f\"Summarize this page and indicate whether it has important information related \"\n", + " f\"to line of business or if it just legal terminology. No more than 5 sentences. \\n\\n{page_text}\")\n", + " \n", + " summary = invoke_claude_3(prompt, max_tokens)\n", + " \n", + " summaries[i+1] = summary # Page numbers are typically 1-indexed\n", + " return summaries\n", + "\n", + "\n", + "directory_path = 'tests'\n", + "files = os.listdir(directory_path)\n", + "\n", + "for file_name in files:\n", + " file_path = os.path.join(directory_path, file_name)\n", + " if os.path.isfile(file_path):\n", + " try:\n", + " pages = preprocess_text_file(file_path)\n", + " page_summaries = run_prompt_preprocess(pages)\n", + " print(f\"File: {file_name}\")\n", + " for page_number, summary in page_summaries.items():\n", + " print(f\"Page {page_number}: {summary}\")\n", + " print(f\"Processed {len(pages)} pages\\n\")\n", + " except Exception as e:\n", + " print(f\"Error processing file {file_name}: {e}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved page summaries to page_summaries.json\n" + ] + } + ], + "source": [ + "import json\n", + "\n", + "# Assuming page_summaries is your dictionary\n", + "# print(type(page_summaries))\n", + "\n", + "# Specify the file path where you want to save the JSON data\n", + "file_path = 'page_summaries.json'\n", + "\n", + "# Save the dictionary to a JSON file\n", + "with open(file_path, 'w', encoding='utf-8') as file:\n", + " json.dump(page_summaries, file, ensure_ascii=False, indent=4)\n", + "\n", + "print(f\"Saved page summaries to {file_path}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Page relevance to contract details:\n", + "Page 1: No.\n", + "Page 2: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract.\n", + "Page 3: No.\n", + "Page 4: No.\n", + "Page 5: No.\n", + "Page 6: No.\n", + "Page 7: Yes, this summary contains information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract.\n", + "Page 8: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on provider obligations and requirements related to prior authorization, referrals, non-covered services, carve-out arrangements, and claims submission, but does not mention the specific contract details requested.\n", + "Page 9: No.\n", + "Page 10: No.\n", + "Page 11: No.\n", + "Page 12: No.\n", + "Page 13: No.\n", + "Page 14: No.\n", + "Page 15: No.\n", + "Page 16: No.\n", + "Page 17: No.\n", + "Page 18: No.\n", + "Page 19: No, this summary does not contain information relevant to the specific contract details mentioned, such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary appears to focus on general contractual provisions and legal requirements, rather than the specific details of the contract itself.\n", + "Page 20: No.\n", + "Page 21: No.\n", + "Page 22: No.\n", + "Page 23: No.\n", + "Page 24: No.\n", + "Page 25: Yes.\n", + "Page 26: No.\n", + "Page 27: No.\n", + "Page 28: No.\n", + "Page 29: No.\n", + "Page 30: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on disclosures related to business transactions, ownership, and criminal convictions, but does not provide any details about the specific contract terms or coverage.\n", + "Page 31: No.\n", + "Page 32: No.\n", + "Page 33: No.\n", + "Page 34: No.\n", + "Page 35: No.\n", + "Page 36: No.\n", + "Page 37: No.\n", + "Page 38: No.\n", + "Page 39: No.\n", + "Page 40: No.\n", + "Page 41: No.\n", + "Page 42: Yes, the summary contains information relevant to contract details such as line of business (Medicaid/FamilyCare), effective date (until June 30, 2021), network for plan (Any Willing Provider and Any Willing Plan), and termination date for the contract (June 30, 2021).\n", + "Page 43: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on details related to claims processing and payment for MLTSS members, as well as quality initiatives and compliance requirements, but does not mention the specific contract details requested.\n", + "Page 44: No.\n", + "Page 45: No, the summary does not contain information relevant to specific contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on general contractual and legal requirements between a provider and a health plan, but does not provide the specific contract details mentioned in the question.\n", + "Page 46: No.\n", + "Page 47: No.\n", + "Page 48: No.\n", + "\n", + "The summary provided does not contain information relevant to the specific contract details like line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on contractual and regulatory requirements between the health plan and the contracted provider/subcontractor, but does not mention the key details about the contract itself.\n", + "Page 49: No.\n", + "Page 50: No.\n", + "Page 51: No.\n", + "Page 52: No, this summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on regulatory requirements for billing and record-keeping for providers participating in the Medicaid/NJ FamilyCare program, but does not provide any specific contract details.\n", + "Page 53: No.\n", + "Page 54: No.\n", + "Page 55: No.\n", + "Page 56: No.\n", + "Page 57: No.\n", + "Page 58: No.\n", + "Page 59: No.\n", + "Page 60: No.\n", + "Page 61: No.\n", + "Page 62: No.\n", + "Page 63: No.\n", + "Page 64: No.\n", + "Page 65: No.\n", + "Page 66: No.\n", + "Page 67: No.\n", + "Page 68: No.\n", + "Page 69: Yes.\n", + "Page 70: No.\n", + "Page 71: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on legal and compliance-related provisions of the agreement, but does not mention any specific contract details.\n", + "Page 72: No.\n", + "Page 73: No.\n", + "Page 74: No\n", + "Page 75: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary indicates that the document is focused on regulatory and compliance requirements rather than specific business operations and does not include details about a particular line of business.\n", + "Page 76: No.\n", + "Page 77: Yes, the summary contains information relevant to contract details such as line of business (Medicaid/FamilyCare), effective date (not explicitly stated but can be inferred), and service area (New Jersey). However, it does not explicitly mention the network for the plan or the termination date for the contract.\n", + "Page 78: No.\n", + "Page 79: No.\n", + "Page 80: No.\n", + "Page 81: No.\n", + "Page 82: No.\n" + ] + } + ], + "source": [ + "def identify_relevant_pages(pages_with_summaries, max_tokens=256):\n", + " \"\"\"\n", + " Identifies pages that contain relevant information related to contract details such as\n", + " CONTRACT_LOB, LOB_PRICING_TERMS_EFFECTIVE_DT, Plan, CONTRACT_NETWORK, \n", + " CONTRACT_SERVICE_AREA, LOB_PRICING_TERMS_TERMINATION_DT.\n", + "\n", + " Args:\n", + " - pages_with_summaries (dict): A dictionary where keys are page numbers and values are summaries.\n", + " - max_tokens (int): Maximum number of tokens for the model's response.\n", + "\n", + " Returns:\n", + " - dict: A dictionary where keys are page numbers and values are 'yes' or 'no', indicating the relevance.\n", + " \"\"\"\n", + " relevant_pages = {}\n", + " for page_number, summary in pages_with_summaries.items():\n", + " # Construct the prompt asking about the relevance of the page to the contract details\n", + " prompt = (f\"Does this summary contain information relevant to contract details such as \"\n", + " f\"line of business (Contract LOB), effective date, network for plan, service area, \"\n", + " f\"and termination date for the contract? Summary: \\\"{summary}\\\" Answer with 'yes' or 'no'.\")\n", + " \n", + " \n", + " response = invoke_claude_3(prompt, max_tokens)\n", + " \n", + " \n", + " relevant_pages[page_number] = response\n", + " \n", + " return relevant_pages\n", + "\n", + "\n", + "\n", + "import json\n", + "\n", + "# Specify the file path from which you want to load the JSON data\n", + "file_path = 'page_summaries.json'\n", + "\n", + "# Read the JSON file and convert it into a dictionary\n", + "with open(file_path, 'r', encoding='utf-8') as file:\n", + " page_summaries = json.load(file)\n", + " \n", + "\n", + "relevant_pages = identify_relevant_pages(pages_with_summaries=page_summaries, max_tokens=256)\n", + "print(\"Page relevance to contract details:\")\n", + "for page_number, is_relevant in relevant_pages.items():\n", + " print(f\"Page {page_number}: {is_relevant}\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saved page answers to relevant_pages.json\n" + ] + } + ], + "source": [ + "import json\n", + "\n", + "# Assuming page_summaries is your dictionary\n", + "# print(type(page_summaries))\n", + "\n", + "# Specify the file path where you want to save the JSON data\n", + "file_path = 'relevant_pages.json'\n", + "\n", + "# Save the dictionary to a JSON file\n", + "with open(file_path, 'w', encoding='utf-8') as file:\n", + " json.dump(relevant_pages, file, ensure_ascii=False, indent=4)\n", + "\n", + "print(f\"Saved page answers to {file_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Page 7: \n", + "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", + "information requested by Health Plan, or...\n", + "Page 25: \n", + "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", + "a week emergency and urgent care coverag...\n", + "Page 42: \n", + "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", + "ATTACHMENT B-2\n", + "NEW JERSEY MEDICAID/FAMIL...\n", + "Page 69: \n", + "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", + "ATTACHMENT B-3\n", + "MEDICARE ADVANTAGE PROGRA...\n", + "Page 77: \n", + "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", + "ATTACHMENT C-1\n", + "NEW JERSEY MEDICAID/FAMIL...\n" + ] + } + ], + "source": [ + "import json\n", + "\n", + "# Load the page answers from JSON file\n", + "page_answers_path = 'relevant_pages.json'\n", + "with open(page_answers_path, 'r', encoding='utf-8') as file:\n", + " page_answers = json.load(file)\n", + "\n", + "# Use regex to match \"yes\" responses, allowing for flexible matching\n", + "yes_regex = re.compile(r'\\byes\\b', re.IGNORECASE)\n", + "\n", + "# Filter pages with a \"Yes\" response using regex\n", + "yes_page_numbers = [int(page) for page, response in page_answers.items() if yes_regex.search(response.strip())]\n", + "\n", + "def get_yes_pages_text(file_path, yes_page_numbers):\n", + " # Get all pages\n", + " pages = preprocess_text_file(file_path)\n", + " \n", + " # Filter for pages with a \"Yes\" response\n", + " yes_pages_text = {page_number: pages[page_number-1] for page_number in yes_page_numbers if page_number <= len(pages)}\n", + " \n", + " return yes_pages_text\n", + "\n", + "# Example file path\n", + "file_path = 'tests/22-3146927_ICMProviderAgreement_273971_1 MU.txt'\n", + "\n", + "# Get text for pages with a \"Yes\" response\n", + "yes_pages_text = get_yes_pages_text(file_path, yes_page_numbers)\n", + "\n", + "for page_number, text in yes_pages_text.items():\n", + " print(f\"Page {page_number}: {text[:100]}...\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Page 7 response: Here are the extracted instances of information from the provided text:\n", + "\n", + "[\n", + " {\n", + " \"CONTRACT_LOB\": \"MEDICAID\",\n", + " \"LOB_PRICING_TERMS_EFFECTIVE_DT\": \"08/25/2020\",\n", + " \"Plan\": \"N/A\",\n", + " \"CONTRACT_NETWORK\": \"N/A\",\n", + " \"CONTRACT_SERVICE_AREA\": \"New Jersey\",\n", + " \"LOB_PRICING_TERMS_TERMINATION_DT\": \"N/A\",\n", + " \"CONTRACT_MARKETPLACE_METAL_LEVEL\": \"N/A\"\n", + " }\n", + "]\n", + "Page 25 response: There are no explicit references to the specific Line of Business, Plan, Network, Service Area, Effective Date, Termination Date, or Marketplace Metal Level in the provided text. The text appears to describe a general provider contract agreement, but does not contain the specific details requested in the instructions.\n", + "Page 42 response: Based on the provided text, there is no explicit information about a contract, line of business, pricing terms, plan, network, service area, or termination date. The text appears to be discussing the New Jersey Medicaid/FamilyCare Program and provisions related to \"MLTSS Any Willing Provider\" and \"Any Willing Plan\" status for certain provider types. There is no information that can be extracted in the format you requested.\n", + "Page 69 response: Based on the provided text, there is no explicit information about the Line of Business, Plan, Network, Service Area, Effective Date, or Termination Date. The text is about the Medicare Advantage program and contains definitions related to that program, such as \"CMS Contract\", \"Dual Eligible Member\", and \"Emergency Services\". There is no information about pricing terms or compensation that can be extracted in the requested format.\n", + "Page 77 response: Based on the provided text, the following information can be extracted:\n", + "\n", + "[\n", + " {\n", + " \"CONTRACT_LOB\": \"MEDICAID\",\n", + " \"LOB_PRICING_TERMS_EFFECTIVE_DT\": \"08/25/2020\",\n", + " \"Plan\": \"FFS\",\n", + " \"CONTRACT_NETWORK\": \"FFS\", \n", + " \"CONTRACT_SERVICE_AREA\": \"New Jersey\",\n", + " \"LOB_PRICING_TERMS_TERMINATION_DT\": null\n", + " }\n", + "]\n" + ] + } + ], + "source": [ + "def process_yes_pages_with_claude(yes_pages_text):\n", + " \"\"\"\n", + " Processes each page with a \"Yes\" response through invoke_claude_3, using prompts generated by LOB_PROMPT_NEW.\n", + "\n", + " Args:\n", + " - yes_pages_text (dict): Dictionary of page numbers and their corresponding texts that have a \"Yes\" response.\n", + "\n", + " Returns:\n", + " - dict: A dictionary where keys are page numbers and values are the responses from invoke_claude_3.\n", + " \"\"\"\n", + " claude_responses = {}\n", + "\n", + " for page_number, page_text in yes_pages_text.items():\n", + " # Generate the prompt for this page\n", + " prompt = page_text + \"\\n\\n\" + LOB_PROMPT_NEW() # Modify as needed\n", + " \n", + " # Invoke Claude 3 with the generated prompt\n", + " response = invoke_claude_3(prompt, max_tokens=2000) # Adjust max_tokens as necessary\n", + " \n", + " claude_responses[page_number] = response\n", + "\n", + " return claude_responses\n", + "\n", + "# Assume yes_pages_text is obtained as described in previous steps\n", + "yes_pages_text = get_yes_pages_text(file_path, yes_page_numbers) # This should be defined based on earlier steps\n", + "\n", + "# Process each \"Yes\" page through Claude 3\n", + "claude_responses = process_yes_pages_with_claude(yes_pages_text)\n", + "\n", + "for page_number, response in claude_responses.items():\n", + " print(f\"Page {page_number} response: {response}\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 2659c772b17455c2f064c83289d2e231de97d95c Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 18 Apr 2024 18:06:01 -0700 Subject: [PATCH 02/78] Bottom up prompts initial commit --- test.ipynb | 2156 ++++++++++++++++++++-------------------------------- 1 file changed, 823 insertions(+), 1333 deletions(-) diff --git a/test.ipynb b/test.ipynb index 34d9ed0..f15d5e2 100644 --- a/test.ipynb +++ b/test.ipynb @@ -2,413 +2,37 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "\n", - "# Load the CSV file into a DataFrame\n", - "df = pd.read_csv('class_B_data.csv')\n", - "\n", - "# Create a new DataFrame to store unique values\n", - "unique_values_df = pd.DataFrame()\n", - "\n", - "# Loop through each column in the original DataFrame and get unique values\n", - "for column in df.columns:\n", - " # Extract unique values for the column, sort them, and reset index for clean formatting\n", - " unique_values = pd.Series(df[column].unique()).sort_values().reset_index(drop=True)\n", - " unique_values_df[column] = unique_values\n", - "\n", - "# Now, unique_values_df contains all the unique values for each column\n", - "# You can print it to see its contents\n", - "print(unique_values_df)\n", - "\n", - "# If you want to save this DataFrame to a new CSV\n", - "unique_values_df.to_csv('unique_values_class_B_data.csv', index=False)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import boto3\n", - "import dotenv\n", - "\n", - "dotenv.load_dotenv()\n", - "\n", - "s3_client = boto3.client('s3',\n", - " region_name=\"us-east-2\",\n", - " aws_access_key_id=os.getenv('ACCESS_KEY'),\n", - " aws_secret_access_key=os.getenv('SECRET_KEY'),\n", - " aws_session_token=os.getenv('SESSION_TOKEN')\n", - " )\n", - "bucket = 'doczy-dev-infra-textract'\n", - "objects = s3_client.list_objects_v2(Bucket=bucket, Prefix=\"training-data/contract-text-file/\")\n", - "file_list = []\n", - "for obj in objects['Contents']:\n", - " if not obj['Key'].endswith('/'):\n", - " file_list.append(obj['Key'])\n", - "\n", - "contract_list = sorted(file_list)\n", - "\n", - "for contract in contract_list:\n", - " data = s3_client.get_object(Bucket=bucket, Key=contract)\n", - " contents = data['Body'].read()\n", - " context = contents.decode(\"utf-8\")\n", - " \n", - " # Modify the path where you want to save the file\n", - " # Split the path and the filename\n", - " path, filename = os.path.split(contract)\n", - " # Remove the file extension from the path\n", - " save_path = os.path.join(\"your_directory\", path) # Specify your directory here\n", - " # Ensure the directory exists\n", - " os.makedirs(save_path, exist_ok=True)\n", - " # Write the text to a new file for each contract\n", - " with open(os.path.join(save_path, filename[:-4] + '.txt'), 'w') as outfile:\n", - " outfile.write(context)\n", - "\n", - "print(contract_list)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#open text file from textfiles folder and read the content:\n", - "with open('textfiles/2001-11-01 California Emergency Physicians PSA_MU.txt', 'r') as infile:\n", - " context = infile.read()" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "def METAL_LEVELS():\n", - " return \"\"\"Platinum\n", - "Gold\n", - "Silver\n", - "Bronze\n", - "\"\"\"\n", - "def LINE_OF_BUSINESS():\n", - "\n", - " return \"\"\"\n", - "COMMERCIAL \n", - "Commercial and Marketplace\n", - "Commercial/Group\n", - "LOB/Provider Type Specific\n", - "MARKETPLACE\n", - "MEDICAID\n", - "MEDICARE\n", - "Marketplace/Exchange\n", - "Medicaid-Medicare Plan\n", - "Medicaid/Medicare\n", - "Medicare Advantage\n", - "Medicare Supplemental\n", - "Self-Funded/ASO/TPA\n", - "Tailored Network\n", - "\"\"\"\n", - "\n", - "def PLANS():\n", - " return \"\"\"HMO\n", - "Leased PPO Benefits Program\n", - "Affinity\n", - "Beacon\n", - "Blue Network\n", - "\"\"\"\n", - "\n", - "def NETWORKS():\n", - " return \"\"\"HMO\n", - "POS\n", - "EPO\n", - "PPO\n", - "FFS\n", - "\"\"\"\n", - "\n", - "def SERVICE_AREAS():\n", - " return \"\"\"Urban/Rural\n", - "County\n", - "ZIP\n", - "Georgia\n", - "Indiana\n", - "Ohio\n", - "\"\"\"\n", - "\n", - "def TYPES_OF_PROVIDER():\n", - " return \"\"\"IP\n", - "Hospital\n", - "SNF\n", - "Home Health\n", - "Intermediate Care Facility\n", - "Outpatient\n", - "ASC\n", - "Primary Care\n", - "Specialist\n", - "Ancillary\n", - "ACS\n", - "AIR AMBULANCE\n", - "\"\"\"\n", - "\n", - "def PROVIDER_SPECIALTIES():\n", - " return \"\"\"Anesthesiology\n", - "Emergency Medicine\n", - "Family Medicine\n", - "General Practice\n", - "Pediatrics\n", - "Obstetrics & Gynecology\n", - "Orthopedic Surgery\n", - "Radiology\n", - "ASCs\n", - "Air Ambulance\n", - "All Physician and Mid-Level Healthcare Professionals\n", - "Behavioral Health\n", - "\"\"\"\n", - "\n", - "def PROGRAM_MEDICARE():\n", - " return \"\"\"SNP\n", - "C-SNP\n", - "D-SNP\n", - "I-SNP\n", - "LTSS\n", - "PACE\n", - "\"\"\"\n", - "\n", - "def PROGRAM_MEDICAID():\n", - " return \"\"\"ABD\n", - "CHIP\n", - "Foster\n", - "IDD\n", - "TANF\n", - "SSI\n", - "SSDI\n", - "\"\"\"\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "def LOB_PROMPT_NEW(): \n", - " LOB = LINE_OF_BUSINESS()\n", - " metal_levels = METAL_LEVELS()\n", - " plans = PLANS()\n", - " networks = NETWORKS()\n", - " service_areas = SERVICE_AREAS()\n", - " \n", - " return f\"\"\"Extract ALL INSTANCES OF THE FOLLOWING INFORMATION FROM THE ABOVE TEXT. If there are multiple instances of this type of information, identify them all and return as seperate objects for each LOB. Extract information like the examples below. Note that the Line of Business (LOB), Plan, Network, and other attributes may vary based on the context provided:\n", - "\n", - "CONTRACT_LOB LOB_PRICING_TERMS_EFFECTIVE_DT Plan CONTRACT_NETWORK CONTRACT_SERVICE_AREA LOB_PRICING_TERMS_TERMINATION_DT \n", - "MARKETPLACE 1/02/2014 GOLD NARROW NETWORK PPO OHIO 1/02/2015 \n", - "\n", - "If the LOB is identified to be Marketplace, then follow the example below:\n", - "CONTRACT_LOB LOB_PRICING_TERMS_EFFECTIVE_DT Plan CONTRACT_NETWORK CONTRACT_SERVICE_AREA LOB_PRICING_TERMS_TERMINATION_DT CONTRACT_MARKETPLACE_METAL_LEVEL\n", - "MARKETPLACE 1/02/2014 GOLD NARROW NETWORK PPO OHIO 1/02/2015 Gold\n", - "\n", - "Choose from the following LOBs for reference. If not found in the list, do your best to match the closest: {LOB}\n", - "For Effective Date, What is start date of line of business specific compensation exhibit? If one is not explicitly stated, this is the same as the contract's effective date. This should be in a long date format and should be earlier than today. Only return the date in MM/DD/YYYY format, do not return any other content.\n", - "You must return the Plan for every entry. The Network, Program, and Service Area should be clearly identified based on the context provided.\n", - "Choose only from the following Line of Business: {LOB}. \n", - "Choose only from the following plans: {plans}.\n", - "Choose only from the following Networks: {networks}.\n", - "Choose only from the following Service Areas: {service_areas}.\n", - "For Date of Termination, identify the specific date upon which the compensation terminates, in the following format ONLY: MM/DD/YYYY.\n", - "Only identify the Metal Level if the LOB is Marketplace. Choose only from the following Metal Levels: {metal_levels}.\n", - "\n", - "Return only the extracted info as a JSON list based on the attributes of the LOB and its context.\"\"\"\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "##EXPLORED SPLITTING ON REGEX MATCHES ON ATTRIBUTES\n", - "\n", - "import regex as re\n", - "import chardet\n", - "\n", - "# Define the category functions\n", - "def METAL_LEVELS():\n", - " return \"\"\"\n", - " Platinum Gold Silver Bronze\n", - " \"\"\"\n", - "\n", - "def LINE_OF_BUSINESS():\n", - " return \"\"\"\n", - " COMMERCIAL Commercial and Marketplace Commercial/Group LOB/Provider Type Specific MARKETPLACE MEDICAID MEDICARE Marketplace/Exchange Medicaid-Medicare Plan Medicaid/Medicare Medicare Advantage Medicare Supplemental Self-Funded/ASO/TPA Tailored Network\n", - " \"\"\"\n", - "\n", - "\n", - "def NETWORKS():\n", - " return \"\"\"\n", - " HMO POS EPO PPO FFS\n", - " \"\"\"\n", - "\n", - "def SERVICE_AREAS():\n", - " return \"\"\"\n", - " Urban/Rural County ZIP Georgia Indiana Ohio\n", - " \"\"\"\n", - "\n", - "def TYPES_OF_PROVIDER():\n", - " return \"\"\"\n", - " IP Hospital SNF Home Health Intermediate Care Facility Outpatient ASC Primary Care Specialist Ancillary ACS AIR AMBULANCE\n", - " \"\"\"\n", - "\n", - "\n", - "def PROGRAM_MEDICARE():\n", - " return \"\"\"\n", - " SNP C-SNP D-SNP I-SNP LTSS PACE\n", - " \"\"\"\n", - "\n", - "def PROGRAM_MEDICAID():\n", - " return \"\"\"\n", - " ABD CHIP Foster IDD TANF SSI SSDI\n", - " \"\"\"\n", - "\n", - "def preprocess_text_file(file_path):\n", - " # Read the file\n", - " import chardet\n", - "\n", - " def detect_encoding(file_path):\n", - " with open(file_path, 'rb') as file:\n", - " raw_data = file.read(10000) # Read the first few bytes to guess the encoding\n", - " result = chardet.detect(raw_data)\n", - " return result['encoding']\n", - " encoding = detect_encoding(file_path)\n", - " print(encoding)\n", - "\n", - " with open(file_path, 'r', encoding='utf-8') as file:\n", - " text = file.read()\n", - " \n", - " # Split the text into pages\n", - " pages = re.split(r'Start of Page No\\. = \\d+', text)\n", - " \n", - " # Combine all categories into a single dictionary with regex patterns\n", - " categories = {\n", - " 'METAL_LEVELS': METAL_LEVELS(),\n", - " 'LINE_OF_BUSINESS': LINE_OF_BUSINESS(),\n", - " 'PLANS': PLANS(),\n", - " 'NETWORKS': NETWORKS(),\n", - " 'SERVICE_AREAS': SERVICE_AREAS(),\n", - " 'TYPES_OF_PROVIDER': TYPES_OF_PROVIDER(),\n", - "\n", - " }\n", - " \n", - " # Prepare regex patterns for each category\n", - " category_patterns = {key: re.compile('|'.join(value.split()), re.IGNORECASE) for key, value in categories.items()}\n", - " \n", - " matching_chunks = []\n", - "\n", - " for i, page in enumerate(pages):\n", - " matches = {category: bool(pattern.search(page)) for category, pattern in category_patterns.items()}\n", - " \n", - " # Count the number of categories with at least one match\n", - " match_count = sum(matches.values())\n", - " \n", - " # If 3 or more categories match, consider this page matched\n", - " if match_count >= 3:\n", - " matching_chunks.append((i, page))\n", - " \n", - " return matching_chunks\n", - "\n", - "directory_path = 'texts/training-data/contract-text-file/0. Professional Boilerplate/'\n", - "files = os.listdir(directory_path)\n", - "\n", - "for file_name in files:\n", - " # Construct the full file path\n", - " file_path = os.path.join(directory_path, file_name)\n", - " # Ensure we're working with a file\n", - " if os.path.isfile(file_path):\n", - " try:\n", - " # Process each file\n", - " matching_chunks = preprocess_text_file(file_path)\n", - " print(f\"File: {file_name}\")\n", - " print(matching_chunks)\n", - " print(f\"Number of matching chunks: {len(matching_chunks)}\\n\")\n", - " except Exception as e:\n", - " # Handle potential errors, e.g., file encoding issues, in a general manner\n", - " print(f\"Error processing file {file_name}: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#prompt based search \n", - "#use prompt to pull all section titles, and surrouding texts and then pass to prompt based search to ask about reimbursement/fee schedules, or line of businesses. \n", - "\n", - "#summarize each page and then pass to prompt based search to ask about reimbursement/fee schedules, or line of businesses. \n" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import dotenv\n", - "\n", - "# Load environment variables\n", - "dotenv.load_dotenv()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - " Hello! My name is Claude. I'm an AI assistant created by Anthropic.\n", - "I am an artificial intelligence created by a company called Anthropic. I'm an AI assistant, which means I'm designed to help humans with a variety of tasks like answering questions, providing analysis and information, and assisting with creative or problem-solving activities. My knowledge comes from machine learning on a large amount of text data, but I'm not a real person - I'm an AI system without a physical body. I'm here to help in whatever way I can, so please let me know if you have any questions or if there is anything I can assist you with.\n" - ] - } - ], - "source": [ + "# Imports\n", "import json\n", "import boto3\n", "import os\n", "import dotenv\n", "import anthropic\n", + "import re\n", + "import pandas as pd\n", + "\n", + "pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Globals\n", "\n", "# Load environment variables\n", "dotenv.load_dotenv()\n", "\n", "# Constants for model IDs\n", - "MODEL_ID_CLAUDE3 = 'anthropic.claude-3-haiku-20240307-v1:0'\n", + "#MODEL_ID_CLAUDE3 = 'anthropic.claude-3-haiku-20240307-v1:0'\n", + "MODEL_ID_CLAUDE3 = 'anthropic.claude-3-sonnet-20240229-v1:0'\n", "MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1'\n", "\n", "# Initialize Bedrock runtime client\n", @@ -418,12 +42,20 @@ " aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),\n", " aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),\n", " aws_session_token=os.getenv('AWS_SESSION_TOKEN')\n", - ")\n", - "\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Claude calls\n", "def invoke_claude_2(prompt, max_tokens):\n", " body = json.dumps(\n", " {\"prompt\": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, \n", - " \"max_tokens_to_sample\": 1024,\n", + " \"max_tokens_to_sample\": max_tokens,\n", " \"temperature\":0.0,\n", " \"top_p\":1,\n", " \"top_k\":250,\n", @@ -447,7 +79,8 @@ " prompt = prompt\n", " body = json.dumps({\n", " \"anthropic_version\": \"bedrock-2023-05-31\",\n", - " \"max_tokens\": 1000,\n", + " \"max_tokens\": max_tokens,\n", + " \"temperature\": 0.0,\n", " \"messages\": [\n", " {\n", " \"role\": \"user\",\n", @@ -471,678 +104,7 @@ "\n", " response_body = json.loads(response.get(\"body\").read())\n", "\n", - " return (response_body['content'][0]['text'])\n", - "\n", - "\n", - "# Example usage:\n", - "prompt = \"hello who are you\"\n", - "max_tokens = 1024\n", - "print(invoke_claude_2(prompt, max_tokens))\n", - "print(invoke_claude_3(prompt, max_tokens))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "import regex as re\n", - "\n", - "def preprocess_text_file(file_path):\n", - " # Read the file\n", - " with open(file_path, 'r', encoding='utf-8') as file:\n", - " text = file.read()\n", - " \n", - " # Split the text into pages\n", - " pages = re.split(r'Start of Page No\\. = \\d+', text)\n", - " \n", - "\n", - " return pages" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "File: 22-3146927_ICMProviderAgreement_273971_1 MU.txt\n", - "Page 1: This document appears to be a Participating Provider Agreement between WellCare New Jersey and healthcare providers for Medicare Advantage and Medicaid/FamilyCare programs. The key points are:\n", - "\n", - "1. The document outlines the contractual terms and conditions between WellCare and the participating providers, including provider-specific requirements, covered services, compensation, and regulatory attachments.\n", - "\n", - "2. The majority of the content consists of legal terminology and regulatory requirements, such as non-discrimination, fraud and abuse, and member financial protections. This indicates that the document contains important legal and compliance information rather than line of business details.\n", - "\n", - "3. There are some sections that provide information on specific services covered under the Medicaid/FamilyCare and Medicare Advantage programs, but these appear to be standard contractual terms rather than strategic business information.\n", - "\n", - "In summary, this document seems to be focused on the legal and regulatory framework governing the relationship between WellCare and its participating providers, rather than containing significant details about the line of business operations.\n", - "Page 2: This document appears to be a participating provider agreement between WellCare Health Plans of New Jersey, Inc. and Rutgers, The State University of New Jersey, on behalf of Robert Wood Johnson Medical School (RWJMS). The key points are:\n", - "\n", - "1. It is a contract between a health plan (WellCare) and a healthcare provider (Rutgers/RWJMS) to provide healthcare services to the health plan's enrollees.\n", - "\n", - "2. The agreement includes specific provisions related to the New Jersey Medicaid/FamilyCare managed care program, which take precedence over any inconsistent language in the rest of the agreement.\n", - "\n", - "3. The document contains extensive legal terminology and definitions, which suggests it is primarily focused on the contractual and regulatory requirements rather than the specifics of the line of business.\n", - "\n", - "Based on the information provided, this document appears to contain important legal and regulatory information related to the contractual relationship between the health plan and the healthcare provider, but does not seem to provide significant details about the line of business operations. It is mostly legal terminology and contractual provisions.\n", - "Page 3: This page appears to contain mostly legal terminology and definitions rather than information directly related to the line of business. The key points are:\n", - "\n", - "1. It defines terms used in the agreement, such as \"Affiliate\", \"Benefit Plan\", \"Clean Claim\", \"Credentialing Criteria\", \"Covered Services\", and \"Federal Health Care Program\".\n", - "2. The definitions provide the legal context and parameters for the agreement, but do not seem to contain any important information about the specific line of business.\n", - "3. The content is focused on establishing the contractual and regulatory framework for the relationship between the parties, rather than discussing the business operations or services.\n", - "4. Overall, this page seems to contain standard legal terminology and definitions typical of a contract or agreement, rather than substantive information about the line of business.\n", - "\n", - "In summary, this page appears to contain legal terminology and definitions, and does not appear to have important information directly related to the line of business.\n", - "Page 4: This page contains important legal terminology and definitions related to a healthcare contract. It does not appear to contain information directly related to a specific line of business. The key points are:\n", - "\n", - "1. Definitions of government authorities, ineligible persons, laws, medically necessary, members, member expenses, overpayments, and participating providers.\n", - "2. The definitions provide the legal context and parameters for the healthcare contract.\n", - "3. This information is typical legal terminology used in healthcare contracts and does not appear to have direct relevance to a specific line of business.\n", - "Page 5: This page appears to contain legal terminology and definitions related to a provider agreement rather than important information about a line of business. The key points are:\n", - "\n", - "1. It defines terms like \"Principal,\" \"Program,\" \"Program Attachment,\" \"Program Requirements,\" \"Provider,\" \"Provider Manual,\" \"State,\" and \"WellCare Companion Guide.\"\n", - "2. It states that providers may freely communicate with members about their treatment, and that Health Plan does not dictate or control clinical decisions.\n", - "3. It clarifies that this is a non-exclusive agreement and there is no guarantee that Health Plan will participate in any particular Program or that any particular Benefit Plan will remain in effect.\n", - "\n", - "Overall, this appears to be standard legal language for a provider agreement and does not contain important information specific to a line of business.\n", - "Page 6: This page appears to contain legal terminology and information related to provider responsibilities under a healthcare plan agreement. The key points are:\n", - "\n", - "1. The health plan reserves the right to create distinct provider networks and determine provider participation.\n", - "2. Providers must comply with requirements for disclosure of ownership, business transactions, and information for persons convicted of crimes against federal healthcare programs.\n", - "3. Providers must maintain and enforce policies and procedures that are consistent with the agreement, both for their employed providers and any subcontracted providers they use.\n", - "4. There are specific information and disclosure requirements for providers to provide to the health plan.\n", - "\n", - "This information seems to be primarily focused on legal and compliance requirements for providers under the healthcare plan agreement, rather than containing important information directly related to the line of business operations.\n", - "Page 7: This page appears to contain important legal and contractual information related to the line of business and operations of the provider. The key points are:\n", - "\n", - "1. It outlines the obligations of the contracted provider, including requirements for subcontracted providers and credentialing of all providers.\n", - "\n", - "2. It specifies the standards and requirements for providing covered services to members, including verifying eligibility and obtaining prior authorization.\n", - "\n", - "3. It discusses the need for providers to comply with laws, regulations, and accreditation standards.\n", - "\n", - "4. The content is primarily legal terminology and contractual language, rather than general business information. This page appears to be an excerpt from a larger agreement or contract between the provider and a health plan.\n", - "\n", - "In summary, this page contains important operational and compliance information for the provider's line of business, rather than just legal terminology.\n", - "Page 8: Based on the summary, this document appears to contain important information related to the line of business and contractual obligations between the health plan and providers. The key points are:\n", - "\n", - "1. The health plan may deny payment for covered services if the provider fails to meet the prior authorization requirements.\n", - "2. Providers are required to furnish complete information to other providers when making referrals.\n", - "3. Providers must inform members and obtain written agreement before providing non-covered services.\n", - "4. The health plan may enter into \"carve-out\" arrangements with third-party providers for certain covered services.\n", - "5. Providers must submit clean claims within a specified time limit and include NPI numbers and taxonomy codes, or the health plan may deny payment.\n", - "\n", - "This document contains important contractual terms and conditions regarding the provision of healthcare services, claims submission, and reimbursement, which are relevant to the line of business.\n", - "Page 9: This document appears to contain legal terminology and provisions related to the contractual relationship between a healthcare provider and a health plan. \n", - "\n", - "The key points are:\n", - "\n", - "1. It outlines requirements for electronic data interchange and claims submission in accordance with HIPAA and the health plan's companion guide.\n", - "\n", - "2. It addresses requirements for electronic funds transfer and remittance advice.\n", - "\n", - "3. It covers coordination of benefits and subrogation responsibilities of the provider.\n", - "\n", - "4. It includes provisions to protect members from being billed for amounts owed by the health plan.\n", - "\n", - "This document does not appear to contain information directly related to the provider's line of business or operations. It is focused on the contractual and regulatory obligations between the healthcare provider and the health plan.\n", - "Page 10: This document appears to be a legal agreement or contract, primarily focused on outlining the terms and requirements for healthcare providers participating in a health plan. It does not contain information directly related to a specific line of business, but rather covers standard contractual provisions such as:\n", - "\n", - "1. Member expenses and billing restrictions\n", - "2. Adherence to the health plan's provider manual and quality improvement programs\n", - "3. Cooperation with utilization management and grievance/appeal processes\n", - "\n", - "Overall, this document contains important legal and operational information for healthcare providers contracted with the health plan, but does not seem to provide any specific insights into a particular line of business. It is focused on the contractual relationship between the health plan and its provider network.\n", - "Page 11: The provided text appears to contain legal terminology and compliance requirements rather than specific information about the line of business. The key points are:\n", - "\n", - "1. Providers must comply with all applicable laws and program requirements, including cooperating with the Health Plan.\n", - "2. Providers must maintain member information and medical records in accordance with privacy laws like HIPAA.\n", - "3. Providers must comply with laws and regulations related to fraud, waste, and abuse prevention.\n", - "4. Providers must comply with the Health Plan's compliance program requirements, including reporting any suspected fraud, waste, or abuse.\n", - "5. Providers must acknowledge that claims and payments under this agreement may be from federal funds.\n", - "\n", - "This information seems to be focused on legal and regulatory compliance rather than details about the line of business. The content does not appear to have significant business-related information.\n", - "Page 12: This document appears to be a legal contract or agreement, and it contains important information related to the line of business. Here is a summary in 5 sentences:\n", - "\n", - "1. The document outlines provisions related to provider payments, including adjustments due to reductions in government funds.\n", - "2. It includes warranty and compliance requirements for the contracted provider, including being an \"Ineligible Person\" and cooperating with compliance audits. \n", - "3. The document discusses licensure and insurance requirements for the contracted provider.\n", - "4. It addresses the confidentiality of \"Proprietary Information\" shared by the health plan with the provider.\n", - "5. Overall, this document contains significant legal and regulatory terms that are important for the line of business, rather than just legal terminology.\n", - "Page 13: This document appears to contain primarily legal terminology and provisions related to a healthcare provider agreement. It does not seem to have significant information directly related to the line of business. The key points are:\n", - "\n", - "1. It covers confidentiality and disclosure requirements for proprietary information.\n", - "2. It outlines the provider's obligation to notify the health plan of certain events that could impact the provider's ability to comply with the agreement.\n", - "3. It describes the health plan's responsibilities, such as issuing member ID cards, claims processing, provider compensation, and medical record review.\n", - "4. It addresses the health plan's right to recover overpayments made to the provider.\n", - "\n", - "Overall, this document appears to be a standard healthcare provider contract and does not seem to contain any critical information about the line of business itself.\n", - "Page 14: This page appears to contain mostly legal terminology and provisions related to healthcare plan administration, rather than important information directly related to the line of business. The key points are:\n", - "\n", - "1. It outlines the process for addressing overpayments made by the health plan to the provider, including the provider's obligation to repay overpayments and the health plan's ability to offset overpayments against future payments.\n", - "\n", - "2. It discusses the health plan's ability to suspend payments to a provider during a government investigation of a credible allegation of fraud.\n", - "\n", - "3. It requires the provider to maintain detailed operational, financial, and administrative records related to the services provided to members, and grants the health plan access to audit those records.\n", - "\n", - "Overall, this appears to be standard contract language outlining the administrative and compliance requirements between the health plan and the provider, rather than information that is directly relevant to the line of business operations.\n", - "Page 15: This document appears to be a legal contract or agreement and contains mainly legal terminology. There is no information directly related to the line of business operations. The key points are:\n", - "\n", - "1. It outlines the requirements for contracted providers to maintain records and provide access/audit to the health plan.\n", - "2. It specifies the term and termination conditions of the agreement, including provisions for termination for convenience or for cause.\n", - "3. The requirements for records, access, and audit survive the expiration or termination of the agreement.\n", - "4. The agreement can be terminated by either party with 90 days' notice (or 180 days for hospital providers).\n", - "\n", - "Overall, this appears to be a standard legal contract governing the relationship between a health plan and its contracted providers, and does not contain any specific business-related information.\n", - "Page 16: This document appears to be a legal contract or agreement between a health plan and a provider. It contains information about the termination and dispute resolution process, as well as requirements for the provider to continue providing services to members during the transition period after termination.\n", - "\n", - "The information in this document is primarily legal terminology and does not seem to contain important information directly related to the line of business. It outlines the contractual obligations and rights of the parties involved, but does not provide any substantive information about the actual healthcare services or operations.\n", - "\n", - "In summary, this document is a standard legal contract and does not appear to have significant information related to the line of business.\n", - "Page 17: This page appears to contain legal terminology and provisions related to dispute resolution and arbitration, rather than important information about the line of business. The key points are:\n", - "\n", - "1. All claims and disputes between the Health Plan and a Provider related to the Agreement must be submitted to arbitration within one year, except for claims based on fraud.\n", - "\n", - "2. Before initiating arbitration, the parties must meet and confer in good faith to seek resolution of the dispute. \n", - "\n", - "3. If the dispute is not resolved within 30 days, either party may submit the dispute to binding arbitration in Newark, New Jersey, through the American Arbitration Association.\n", - "\n", - "4. The arbitration will be conducted by a single arbitrator or a panel of three arbitrators, depending on the amount in dispute.\n", - "\n", - "5. The Agreement is governed by the laws of the State of New Jersey, with federal law applying where New Jersey law is preempted.\n", - "\n", - "In summary, this page appears to contain standard legal provisions for dispute resolution and does not seem to have direct relevance to the line of business.\n", - "Page 18: This page contains legal terminology and information related to the jurisdiction, venue, and dispute resolution procedures for the agreement. It does not appear to contain any important information directly related to the line of business. The key points are:\n", - "\n", - "1. The agreement specifies that any legal proceedings related to the agreement will be handled in state or federal courts located in New Jersey.\n", - "2. The parties waive their right to a jury trial for any disputes.\n", - "3. The parties can seek injunctive or other equitable relief to enforce the agreement.\n", - "4. The relationship between the parties is that of independent contractors, not partners or joint ventures.\n", - "5. The agreement includes restrictions on offshore contracting and out-of-state payments related to certain government healthcare programs.\n", - "\n", - "This page appears to be focused on standard legal provisions governing the agreement, rather than operational or business-related details.\n", - "Page 19: This page contains important information related to the line of business, specifically the following:\n", - "\n", - "1. The agreement is subject to applicable laws, program requirements, and accreditation standards, which are deemed incorporated into the agreement.\n", - "2. The agreement can be amended by the health plan to comply with regulatory requirements, with 30 days' notice to the contracted provider.\n", - "3. Any assignment, delegation, or transfer of the agreement is subject to review and approval by the New Jersey Department of Banking and Insurance and the New Jersey Department of Human Services.\n", - "4. The contracted provider cannot assign, delegate, or transfer the agreement without the prior written consent of the health plan.\n", - "5. The parties cannot use each other's name, symbol, logo, or service mark without prior written approval, except for certain limited purposes.\n", - "\n", - "This page appears to contain legal terminology and contractual provisions that are important for the line of business, rather than just legal boilerplate.\n", - "Page 20: This document appears to contain mostly legal terminology and provisions, rather than information specific to a line of business. The key points are:\n", - "\n", - "1. It discusses payment terms and liability between the Health Plan and its Affiliates.\n", - "2. It includes a \"Force Majeure\" clause that excuses performance under the agreement in the event of certain uncontrollable circumstances.\n", - "3. It covers standard legal provisions such as severability, waiver, the entire agreement, and interpretation of the contract.\n", - "4. Overall, the document seems to be a general contract or agreement template, and does not contain any important information specific to a particular line of business.\n", - "Page 21: This document appears to contain primarily legal terminology and provisions rather than information directly related to the line of business. The key points are:\n", - "\n", - "1. It outlines the rights and remedies of the parties involved, including the cumulative nature of these rights.\n", - "2. It describes the execution of the agreement, including the use of counterparts and electronic signatures.\n", - "3. It includes warranties and representations made by the parties, such as their legal standing and authority to enter into the agreement.\n", - "4. It mentions that the health plan has not delegated any functions to the provider, unless there is a separate delegation schedule or addendum.\n", - "5. It lists the attachments that are incorporated into the agreement, including details on provider requirements, covered services, compensation, and various programs.\n", - "\n", - "Overall, this document seems to be focused on the legal structure and terms of the agreement rather than providing substantive information about the line of business.\n", - "Page 22: This page appears to be a signature page for an agreement between WellCare Health Plans of New Jersey, Inc. and Rutgers, The State University of New Jersey, by and on behalf of Robert Wood Johnson Medical School (RWJMS). It contains the signatures of the parties involved, their respective titles, and the effective date of the agreement. The information on this page is primarily legal terminology related to the execution and effective date of the agreement, and does not appear to contain important information directly related to the line of business. The content is limited to the formalities of signing the agreement.\n", - "Page 23: The provided document appears to be an attachment that outlines provider-specific requirements, covered services, and other relevant information. It is likely a part of a larger contract or agreement. The document seems to contain legal terminology and details specific to the provider and the services they offer, rather than high-level information about the line of business. Based on the information provided, this document seems to be focused on the contractual and operational aspects of the provider's relationship with the organization, rather than providing insights into the broader line of business.\n", - "Page 24: This document appears to contain important information related to the line of business, specifically regarding provider requirements and covered services. The key points are:\n", - "\n", - "1. It defines various terms, such as \"Assigned Member,\" \"Covering Provider,\" \"Nurse Practitioner,\" \"Physician,\" \"Primary Care Provider,\" \"Primary Care Services,\" \"Specialty Provider,\" and \"Specialty Services.\"\n", - "\n", - "2. It states that providers who offer primary care or specialty services have the sole responsibility to acquire and maintain hospital admission privileges, if necessary.\n", - "\n", - "3. It acknowledges that providers who offer primary care or specialty services have a responsibility to ensure 24-hour, 7-day-a-week coverage for their patients.\n", - "\n", - "This document does not appear to contain just legal terminology, but rather important operational and contractual requirements for providers in the line of business.\n", - "Page 25: This page appears to contain important information related to the line of business, specifically the requirements and responsibilities of contracted providers under the agreement. The key points are:\n", - "\n", - "1. Contracted providers must provide or arrange for the provision of all covered services within the scope of their licenses/certifications.\n", - "2. Primary care providers must coordinate the overall healthcare of assigned members, including providing primary care services and making appropriate referrals.\n", - "3. Specialty providers must provide consultation summaries, notify primary care providers of hospital admissions, and seek authorization for hospital admissions (except for emergency services).\n", - "4. Primary care providers must maintain member medical records as required by applicable laws.\n", - "5. Contracted providers must accept members as patients as long as they are accepting new patients, subject to provider-to-patient ratio requirements.\n", - "\n", - "This information appears to be directly relevant to the line of business and contractual obligations of the providers, rather than just general legal terminology.\n", - "Page 26: The provided information appears to be a section from a legal contract or agreement. It discusses a requirement for the provider to give the health plan 60 days' prior notice if the provider is not available to accept members as patients. This seems to be a standard contractual clause related to the provider's obligations and availability, rather than containing any important information about the line of business. The document appears to be focused on legal terminology and contractual provisions.\n", - "Page 27: This document is a Disclosure Statement for Ownership and Control Interest, Related Business Transactions, and Persons Convicted of a Crime, specifically for the New Jersey Medicaid/FamilyCare Program. It appears to be a legal and regulatory document that contains important information related to the line of business.\n", - "\n", - "The key points are:\n", - "\n", - "1. The contracted provider (Rutgers Health Group, Inc.) agrees to provide a complete and accurate disclosure form at the time of initial contracting, upon any changes, and upon request, in accordance with federal and state law.\n", - "\n", - "2. The document requires the disclosure of ownership and control interest, including the name, relationship, and percentage of ownership for the disclosing entity.\n", - "\n", - "3. The document also requires the disclosure of any subcontractors with whom the disclosing entity has a direct or indirect ownership interest of 5% or more.\n", - "\n", - "4. This appears to be a standard regulatory disclosure document required for participation in the New Jersey Medicaid/FamilyCare program, and does not contain general business information.\n", - "Page 28: This page contains legal terminology and requirements for disclosing ownership and control interests in a Medicaid provider, fiscal agent, or managed care entity. It does not appear to have any important information directly related to the line of business. The page outlines the information that needs to be provided, such as names, relationships, ownership percentages, addresses, and tax IDs for individuals and corporations with ownership or control interests. This information is likely required for regulatory or compliance purposes, rather than being directly relevant to the operations or strategy of the business.\n", - "Page 29: The provided text appears to be related to legal and regulatory requirements for disclosing information about managing employees, business transactions, and ownership of subcontractors. It does not seem to contain any information directly related to the line of business. The text outlines the requirements for the Disclosing Entity to provide various information, such as the names, addresses, relationships, dates of birth, and social security numbers of managing employees, as well as details about business transactions and subcontractor ownership. This information is likely required for regulatory and compliance purposes, but does not seem to have significant implications for the line of business operations.\n", - "Page 30: This page appears to contain information related to the line of business, specifically regarding disclosures of business transactions and persons with ownership or control interests who have been convicted of crimes. The key points are:\n", - "\n", - "1. Disclosure of any significant business transactions between the Disclosing Entity and its Wholly Owned Suppliers or Subcontractors during the past 5 years.\n", - "2. Information on types of transactions with \"parties in interest\" as defined in the Public Health Service Act and Social Security Act.\n", - "3. Disclosure of any persons with Ownership or Control Interest, or who are directors, officers, or managing employees of the Disclosing Entity, who have been convicted of a criminal offense related to their involvement in Medicare, Medicaid, CHIP, or Title XX services programs.\n", - "\n", - "This page appears to contain important information related to the line of business, as it covers various disclosures and transparency requirements.\n", - "Page 31: This document appears to be a Disclosure Statement that contains legal terminology and requirements. It does not seem to have any important information related to the line of business. The key points are:\n", - "\n", - "1. It warns against making false statements on the Disclosure Statement, which could result in prosecution or denial/termination of participation.\n", - "2. It requires the disclosure of the requested information, which if not fully and accurately disclosed, could also lead to denial or termination of participation.\n", - "3. The document is signed by Kathleen Bramwell, the Senior Vice Chancellor of Finance & Administration, on 3/2/2021.\n", - "4. The document is related to a contract (#166294) between New Jersey - Rutgers Health Group, dated 8/25/2020.\n", - "\n", - "Overall, this document appears to be a standard legal disclaimer and disclosure requirement, rather than containing any significant information about the line of business.\n", - "Page 32: This page provides definitions related to ownership and control interests in entities, including:\n", - "\n", - "1. Indirect Ownership Interest: An ownership interest in another entity that has an ownership interest in the relevant entity.\n", - "2. Managing Employee: An individual who exercises operational or managerial control over an institution.\n", - "3. Ownership Interest: The possession of equity in the capital, stock, or profits of an entity.\n", - "4. Ownership or Control Interest: Criteria for determining when a person or corporation has a 5% or more ownership or control interest in an entity.\n", - "5. Subcontractor: An individual, agency, or organization to which an entity has contracted or delegated some of its management functions or responsibilities.\n", - "6. Supplier: An individual, agency, or organization from which a provider purchases goods and services.\n", - "7. Wholly Owned Supplier: A supplier whose total ownership interest is held by a provider or by a person, persons, or other entity with an ownership or control interest in a provider.\n", - "\n", - "This information appears to be primarily legal terminology and definitions, and does not seem to contain important information directly related to the line of business.\n", - "Page 33: This page appears to be focused on providing information about contracted healthcare providers, including their names, addresses, contact details, professional licenses, identification numbers, and other relevant details. It does not seem to contain any information directly related to the line of business. Instead, it appears to be a list of the required information that healthcare providers need to provide as part of their contract or agreement. This information is likely important for regulatory and administrative purposes, but it does not directly impact the line of business operations. Overall, this page seems to be primarily focused on legal and compliance-related terminology and requirements, rather than business-specific information.\n", - "Page 34: This document appears to be an attachment to a larger contract, specifically \"PROGRAM ATTACHMENTS\" for a contract between New Jersey and Rutgers Health Group dated August 25, 2020. The document does not seem to contain any important information related to the line of business, but rather consists of legal terminology and attachments that are typically included in a contract. Based on the limited information provided, this appears to be a standard legal document with no significant business-related content.\n", - "Page 35: This document appears to be a regulatory attachment related to the New Jersey Medicaid/FamilyCare Program. It does not contain any information directly related to a specific line of business. The document outlines the following:\n", - "\n", - "1. This attachment is incorporated into the Participating Provider Agreement between the Health Plan and the Contracted Provider to govern the provision of covered services to Medicaid/FamilyCare Program members.\n", - "\n", - "2. It defines key terms such as \"Appeal\", \"Clean Claim\", \"Covered Services\", and references various New Jersey state agencies involved in the Medicaid/FamilyCare Program.\n", - "\n", - "3. The attachment is intended to supplement the main Participating Provider Agreement and take precedence over any inconsistent provisions in the agreement, specifically with respect to the Medicaid/FamilyCare Program.\n", - "\n", - "4. The document does not appear to contain any substantive information about a specific line of business, but rather legal and regulatory details regarding the Medicaid/FamilyCare Program in New Jersey.\n", - "\n", - "In summary, this document appears to be a regulatory attachment with legal terminology rather than information directly relevant to a line of business.\n", - "Page 36: This document appears to contain legal terminology and definitions related to a health insurance contract. It does not seem to have any direct information about a specific line of business. The key points are:\n", - "\n", - "1. It defines terms like \"Emergency Medical Condition\", \"Emergency Services\", \"Grievance\", \"Medicaid/FamilyCare Program\", \"Medical Necessity\", \"Member\", \"New Jersey Contract\", and \"Plan of Care\".\n", - "\n", - "2. These definitions are likely important for understanding the rights, responsibilities, and processes outlined in the health insurance contract.\n", - "\n", - "3. The document does not appear to contain any substantial information about the line of business or operations of the organization. It is focused on legal and regulatory terms.\n", - "\n", - "4. Overall, this seems to be a section of a larger contract or agreement, providing definitions of key terms rather than substantive information about the business.\n", - "\n", - "5. The content is heavily focused on legal and regulatory language, rather than details about the line of business.\n", - "Page 37: This page appears to contain legal terminology and information related to the submission, acknowledgement, and payment of claims under the agreement. It does not seem to have any important information directly related to the line of business. The key points are:\n", - "\n", - "1. The agreement follows the Health Information Electronic Data Interchange Act (HINT) requirements for claim submission and payment timelines.\n", - "2. Contracted providers must file claims within specific timeframes, and Health Plan has the right to deny claims filed outside those timeframes.\n", - "3. Contracted providers must hold members harmless for the cost of any covered services or supplies, and cannot balance bill members.\n", - "4. Health Plan's agreement with secondary contractors must include a provision requiring the contractors' network providers to hold members harmless.\n", - "\n", - "Overall, this page appears to contain standard legal language related to claims processing and member protections, rather than any specific information about the line of business.\n", - "Page 38: This document appears to be focused on the legal and regulatory aspects of the agreement between the health plan and the contracted provider. It outlines the requirements for termination of the agreement, including the notice period, the provider's right to a hearing, and the obligations of the provider to continue providing services to members for a certain period after termination. \n", - "\n", - "The information provided does not seem to contain important details about the line of business or operations of the parties involved. It is primarily focused on the legal and regulatory framework governing the contractual relationship between the health plan and the provider.\n", - "\n", - "In summary, this document appears to be mostly legal terminology and requirements, with no significant information related to the line of business of the parties involved.\n", - "Page 39: The summary of the given page is as follows:\n", - "\n", - "1. The page outlines the grievances and appeals procedures that must be followed, which comply with applicable laws as set forth in the Provider Manual.\n", - "2. It discusses the process for recoupment of overpayments and the resolution of payment disputes between the provider and the payor.\n", - "3. The page highlights the insurance coverage requirements for different types of healthcare providers, including minimum limits for professional malpractice and general liability insurance.\n", - "4. For healthcare facility providers, the page outlines specific procedures that must be followed regarding granting and attending privileges to physicians, as well as the admissions procedures for members.\n", - "\n", - "This page contains important information related to the legal and regulatory requirements for healthcare providers, but does not appear to have any specific details regarding the line of business.\n", - "Page 40: This document appears to contain important information related to the line of business, specifically the provider's agreement with the health plan. The key points are:\n", - "\n", - "1. The provider agrees to admit members to the hospital under certain conditions, and to notify the health plan when members present at the emergency room.\n", - "2. The document outlines billing and payment procedures, including timeframes for submitting clean claims and the health plan's obligation to pay providers.\n", - "3. The provider must conduct criminal background checks on employees and agents providing direct care to managed long-term services and supports (MLTSS) members.\n", - "4. The provider must comply with requirements for reporting critical incidents and notifying the health plan of substance abuse admissions.\n", - "5. The provider must comply with the health plan's training and education program.\n", - "\n", - "This appears to be a legal contract or agreement between the provider and the health plan, and contains important operational and compliance requirements for the provider.\n", - "Page 41: This page appears to be a contract between a healthcare provider and a health plan. It contains information about the provider's obligation to report any instances where they are unable to contact an MLTSS (Managed Long-Term Services and Supports) member, in accordance with the Provider Manual. This information is likely related to the line of business and the operational requirements of the provider's relationship with the health plan. The content does not seem to be primarily focused on legal terminology, but rather on the contractual obligations and procedures that the provider must follow.\n", - "Page 42: This page appears to contain important information related to the line of business, specifically the New Jersey Medicaid/FamilyCare Program. The key points are:\n", - "\n", - "1. The agreement/subcontract is subject to the terms and conditions of the contract between the Health Plan and the State of New Jersey.\n", - "2. The document outlines the \"Any Willing Provider\" and \"Any Willing Plan\" provisions for certain Medicaid providers in New Jersey, such as nursing facilities, assisted living providers, and long-term care pharmacies.\n", - "3. The \"Any Willing Provider\" status for these providers is in effect until the end of State Fiscal Year 2021 (June 30, 2021), after which the Health Plan can determine their network status based on utilization and access needs.\n", - "4. The \"Any Willing Plan\" status also expires on June 30, 2021.\n", - "\n", - "This document appears to contain important contractual and regulatory information related to the line of business, rather than just legal terminology.\n", - "Page 43: This document appears to contain important information related to the line of business, specifically regarding claims processing and payment for MLTSS (Managed Long-Term Services and Supports) members. The key points are:\n", - "\n", - "1. It outlines the timeframes for processing claims for various MLTSS services, such as assisted living providers, nursing facilities, and personal care services.\n", - "2. It mentions the Nursing Facility Quality Incentive Payment Program (NF QIPP), which replaces the previous Any Willing Qualified Provider (AWQP) initiative and focuses on quality outcomes for long-stay Medicaid residents in nursing facilities.\n", - "3. The document also emphasizes the need for the contracted provider/subcontractor to comply with federal and state laws and regulations, including the submission of claims within specific timeframes.\n", - "\n", - "Overall, this document appears to contain important operational and compliance information related to the line of business, rather than just legal terminology.\n", - "Page 44: This document appears to be related to a medical provider contract or subcontract. The key points are:\n", - "\n", - "1. The State reserves the right to review and approve or disapprove the agreement/subcontract and any amendments.\n", - "\n", - "2. The agreement/subcontract can only be amended unilaterally for statutory and regulatory changes, and upon mutual consent with State approval.\n", - "\n", - "3. The agreement/subcontract becomes effective when the health plan's agreement with the State takes effect.\n", - "\n", - "4. The health plan must notify the State at least 30 days prior to suspending, terminating, or the provider withdrawing from the network, and if the termination was \"for cause\" related to fraud, waste and abuse, the reasons must be provided.\n", - "\n", - "This appears to be focused on legal and regulatory requirements related to the provider contract, rather than specific details about the line of business. It does not seem to contain important information directly related to the line of business.\n", - "Page 45: This page appears to contain important information related to legal and contractual terms between a healthcare provider and a health plan. The key points are:\n", - "\n", - "1. The provider must participate in and cooperate with the health plan's quality and utilization review programs, as long as they are based on clinical evidence and do not restrict provider-patient communications.\n", - "\n", - "2. The health plan is restricted from terminating the agreement/subcontract if the provider expresses disagreement with the health plan's decisions or discusses treatment options with patients.\n", - "\n", - "3. The state may order termination of the agreement if the provider takes actions that threaten the health, safety, or welfare of members, or the fiscal integrity of the Medicaid program.\n", - "\n", - "This document seems to be focused on contractual and legal requirements, rather than specific line of business information.\n", - "Page 46: This page appears to contain legal terminology and requirements rather than important information related to the line of business. The key points are:\n", - "\n", - "1. It outlines the conditions under which the contracted provider/subcontractor's agreement may be terminated, such as suspension or revocation of certification, insolvency, bankruptcy, material breach of the agreement, or violations of state/federal law.\n", - "\n", - "2. It describes the contracted provider/subcontractor's obligations regarding non-discrimination, including that they must accept Medicaid/FamilyCare Program beneficiaries and comply with the Americans with Disabilities Act (ADA).\n", - "\n", - "3. The contracted provider/subcontractor is required to submit a written certification that they are in compliance with the ADA and will hold the state harmless from any liability related to non-compliance.\n", - "\n", - "Overall, this page seems to be focused on legal and compliance requirements rather than business-critical information.\n", - "Page 47: This document appears to contain legal terminology and information related to non-discrimination policies, rather than specifics about the line of business. The key points are:\n", - "\n", - "1. The contracted provider/subcontractor must abide by the provisions of the Rehabilitation Act of 1973 regarding access to programs and facilities by people with disabilities.\n", - "\n", - "2. The contracted provider/subcontractor shall not discriminate against eligible persons or members based on various factors like health status, need for services, or pre-existing conditions. \n", - "\n", - "3. The contracted provider/subcontractor must comply with various civil rights laws and regulations prohibiting discrimination on grounds of age, race, disability, sex, etc.\n", - "\n", - "4. The contracted provider/subcontractor agrees to forward any grievances alleging discrimination against members to the health plan for review and action.\n", - "\n", - "5. The contracted provider/subcontractor remains obligated to provide services for the duration of the period after the health plan's insolvency, and shall not bill, charge, or seek compensation from the members under any circumstances.\n", - "\n", - "Overall, this appears to be legal and contractual language about non-discrimination policies and obligations, rather than information directly related to the line of business.\n", - "Page 48: This page appears to contain legal terminology and provisions related to the contracted relationship between a health plan and a contracted provider/subcontractor. It does not seem to contain any important information directly related to the line of business. The key points are:\n", - "\n", - "1. The contracted provider/subcontractor agrees not to seek compensation or reimbursement from members for covered services, except as provided in the contract.\n", - "2. This provision survives the termination of the agreement and supersedes any contrary agreements between the provider/subcontractor and members.\n", - "3. The state, CMS, and other government entities have the right to inspect and audit records and facilities related to the contract.\n", - "4. There are specific types of records that can be inspected, including financial, medical, and administrative documents.\n", - "\n", - "Overall, this page appears to be focused on contractual and regulatory requirements rather than the line of business operations.\n", - "Page 49: This page appears to contain legal terminology and requirements related to record maintenance, retention, and provider/subcontractor documentation. The key points are:\n", - "\n", - "1. Contracted providers/subcontractors must maintain records fully disclosing the nature and extent of services provided to Medicaid recipients, in accordance with New Jersey regulations.\n", - "\n", - "2. Records must be retained for at least 10 years from the date of service or after the final payment, and must be accessible in New Jersey for review by state and federal agencies.\n", - "\n", - "3. Specific requirements are outlined for documentation of medical supplies, equipment, and DME, including the need for a dated prescription or Certificate of Medical Necessity.\n", - "\n", - "This information seems to be focused on legal and regulatory compliance requirements, rather than directly related to the line of business operations. The content appears to be important for ensuring proper documentation and record-keeping practices for Medicaid providers and contractors.\n", - "Page 50: The given text appears to be a legal document that outlines the requirements for medical equipment and laboratory test orders in New Jersey's Medicaid/NJ FamilyCare program. It does not contain any information specifically related to line of business. The key points are:\n", - "\n", - "1. Orders for medical equipment and supplies must include detailed information about the item, the patient's condition, and the prescriber's details.\n", - "2. Orders for laboratory tests must be patient-specific, contain the tests requested, and be available for review by Medicaid/NJ FamilyCare representatives.\n", - "3. Alternative forms of orders, such as electronic orders or orders based on the patient's medical record, are also accepted if they meet certain security and documentation requirements.\n", - "\n", - "Overall, this document appears to be focused on regulatory and legal requirements related to medical orders, rather than line of business information.\n", - "Page 51: The provided document appears to contain legal terminology and requirements related to laboratory orders and procedures. It outlines the specific information that must be included in laboratory orders, such as the name and address of the authorized person requesting the test, the patient's information, the specific tests to be performed, the source of the specimen, and other relevant details. The document also discusses standing orders and the requirements for them. Overall, this document seems to be focused on legal and regulatory requirements for laboratory orders and procedures, and does not appear to contain information directly related to a line of business.\n", - "Page 52: This page appears to contain important information related to legal and regulatory requirements for billing and record-keeping in the provision of medical services to Medicaid/NJ FamilyCare members. The key points are:\n", - "\n", - "1. Laboratories providing tests to Medicaid/NJ FamilyCare members must maintain detailed records of all orders, test results, and clinical and billing information, which must be available for review.\n", - "2. Medicaid/NJ FamilyCare has the right to inspect all records of in-state and out-of-state service and reference laboratories.\n", - "3. All laboratory test orders must be supported by documentation in the referring provider's medical records.\n", - "4. Psychologists providing services to Medicaid/NJ FamilyCare members must keep detailed individual records of the services provided, which must be made available to the Medicaid/NJ FamilyCare program upon request.\n", - "\n", - "This information appears to be directly relevant to the line of business and the regulatory requirements for providers participating in the Medicaid/NJ FamilyCare program.\n", - "Page 53: The given text appears to be legal terminology and guidelines related to mental health services provided by an independent clinic. It outlines the required components of a psychologist's treatment notes, such as the duration of service, modality used, progress notes, and other clinical details. It also specifies the requirements for an intake evaluation, including the assessment of the member's mental condition, the appropriateness of the treatment program, and the certification by the evaluation team. This information is related to the legal and regulatory aspects of providing mental health services, rather than specific business operations or line of business.\n", - "Page 54: This document appears to be primarily focused on legal terminology and requirements for mental health treatment plans and documentation, rather than containing important information directly related to a specific line of business. The key points are:\n", - "\n", - "1. A written, individualized Plan of Care must be developed for each patient receiving continued treatment, with specific details on treatment objectives, services, and schedules.\n", - "2. The mental health clinic must maintain detailed documentation to support each service provided, including the type of service, date, duration, provider signature, and any unusual occurrences or deviations from the treatment plan.\n", - "3. There may be instances where a temporary deviation from the written treatment plan occurs, which must be clearly explained in the documentation.\n", - "4. The document does not seem to contain information directly relevant to a specific line of business, but rather outlines general legal and regulatory requirements for mental health treatment plans and documentation.\n", - "Page 55: The provided text appears to contain detailed legal and regulatory requirements related to documentation and record-keeping for healthcare services provided to members or patients. The information seems focused on the specific requirements for clinical progress notes, treatment plans, and periodic reviews, as well as the documentation standards for Advanced Practice Nurse (APN) services. The content is primarily concerned with ensuring proper documentation and compliance with applicable rules and regulations, rather than providing information directly related to a particular line of business. Based on the level of detail and the nature of the content, this appears to be a legal or regulatory document, rather than a business-focused resource.\n", - "Page 56: This page appears to contain information related to documentation requirements for billing and reimbursement purposes, rather than direct information about a line of business. The key points are:\n", - "\n", - "1. It outlines the documentation required for an initial visit, including the chief complaint, history, physical exam, diagnosis, and plan of care.\n", - "2. It specifies the documentation required for routine office visits or follow-up care, including the patient's chief complaint, relevant history and physical findings, diagnostic tests, and diagnosis.\n", - "3. The documentation requirements differ slightly for visits conducted in an office/residential setting versus a hospital/nursing facility.\n", - "4. This seems to be primarily legal terminology related to billing and reimbursement procedures, rather than direct information about a specific line of business.\n", - "Page 57: This page appears to contain legal terminology and requirements related to the documentation of services provided by Advanced Practice Nurses (APNs) during inpatient stays and Early Periodic Screening, Diagnosis, and Treatment (EPSDT) examinations. The information covers the specific details that must be documented in the medical record to qualify for reimbursement, such as reviewing the patient's medical history, performing a physical examination, confirming or revising the diagnosis, and demonstrating the APN's personal involvement in the service rendered. This seems to be important information related to compliance and reimbursement requirements, but does not appear to contain significant information about a specific line of business.\n", - "Page 58: This page appears to contain legal terminology and requirements related to medical record-keeping practices for physicians. The information is not directly related to the line of business, but rather outlines the necessary documentation and record-keeping procedures that physicians must adhere to when providing services to patients. The summary includes details on the specific information that must be included in progress notes, the storage and retention of records, and the requirement to make records available to the New Jersey Medicaid/NJ FamilyCare program upon request. Overall, this page does not contain important information related to the line of business, but rather focuses on regulatory and compliance-related requirements for medical record-keeping.\n", - "Page 59: This page appears to contain detailed requirements for the minimum documentation that must be entered in a patient's medical record, depending on whether the patient is a new member or an established member. The information seems to be focused on legal and regulatory requirements rather than specifics about a line of business. It does not appear to contain important information directly related to a line of business, but rather outlines the necessary documentation standards that must be met. The content is mainly legal terminology and formatting.\n", - "Page 60: This document appears to be outlining the minimum documentation requirements for various medical services and settings, including:\n", - "\n", - "1. Laboratory, X-Ray, ECG, and other diagnostic tests with results\n", - "2. Home visits and house calls, including treatment plan status and recommendations\n", - "3. Hospital or nursing facility visits, including updates on symptoms, findings, procedures, and treatment changes\n", - "4. Hospital discharge medical summaries, including a summary of the patient's hospitalization and recommendations for further care\n", - "5. Mental health services, including written documentation to support the medical necessity of the services provided\n", - "\n", - "This information seems to be focused on the legal and regulatory requirements for medical record documentation, rather than directly related to a specific line of business. It does not appear to contain high-level strategic or operational information about the organization's business activities.\n", - "Page 61: This page appears to contain legal terminology and requirements related to healthcare and medical services documentation, rather than information directly related to a line of business. The key points are:\n", - "\n", - "1. Detailed documentation is required for each medical or remedial therapy, service, activity, or session, including the specific services rendered, date and time, duration, physician's signature, setting, and other relevant clinical information.\n", - "\n", - "2. Clinical progress, complications, and treatment that affect prognosis and progress must be documented in the member's medical record.\n", - "\n", - "3. For mental health services not included in the treatment regime, a detailed explanation must be submitted with the claim form.\n", - "\n", - "4. Pharmacies are required to maintain purchase records for prescription drugs and medical supplies for a minimum of 10 years.\n", - "\n", - "This information seems to be focused on legal and regulatory requirements for healthcare documentation and record-keeping, rather than directly related to a specific line of business.\n", - "Page 62: This page primarily contains legal terminology and requirements related to pharmacies, drug dispensing, and data reporting. The key points are:\n", - "\n", - "1. Pharmacies must maintain detailed records of drug purchases, including price, drug name, dosage, NDC, lot number, and quantity.\n", - "2. Pharmacies must obtain drugs from authorized, regulated sources and provide product tracing information to the state upon request.\n", - "3. Pharmacies must have the specific drug in stock before submitting a claim and use the correct NDC on the claim.\n", - "4. Pharmacies must keep records of compounded prescriptions, including all ingredients used.\n", - "5. Pharmacies must maintain detailed records of any inventory transfers between pharmacies and provide this information to the state.\n", - "\n", - "This information appears to be focused on compliance and regulatory requirements for pharmacies, rather than directly related to a specific line of business.\n", - "Page 63: This page appears to contain legal terminology and requirements rather than information directly related to the line of business. The key points are:\n", - "\n", - "1. The contracted provider/subcontractor must comply with the prohibition on the use of federal funds for lobbying.\n", - "2. The contracted provider/subcontractor must comply with financial disclosure requirements.\n", - "3. The contracted provider/subcontractor cannot impose cost-sharing charges on Medicaid or Medicaid/FamilyCare members.\n", - "4. The contracted provider/subcontractor agrees to indemnify and hold harmless the state, its officers, agents, employees, and members from various claims and liabilities.\n", - "\n", - "Overall, this page seems to outline legal obligations and requirements for the contracted provider/subcontractor rather than specific details about the line of business.\n", - "Page 64: This page appears to contain legal terminology and requirements related to confidentiality and indemnification, which are important for the contractual relationship between the parties involved. The key points are:\n", - "\n", - "1. The contracted provider/subcontractor must indemnify the state and its members from any injury, losses, or damages arising from the provider's negligence or violations of state/federal laws.\n", - "\n", - "2. The contracted provider/subcontractor must maintain the confidentiality of all information related to the members, the state, and the health plan in accordance with applicable laws and regulations.\n", - "\n", - "3. The contracted provider/subcontractor must protect the confidentiality of member-specific information and only use it for the purpose of the agreement.\n", - "\n", - "4. Employees of the contracted provider/subcontractor must be instructed to keep all confidential information related to the state, its members, and its operations.\n", - "\n", - "This information appears to be focused on legal requirements and compliance, rather than directly on the line of business operations. It sets forth important contractual obligations and restrictions for the contracted provider/subcontractor.\n", - "Page 65: This document appears to contain important information related to legal and compliance requirements for healthcare providers and subcontractors. The key points are:\n", - "\n", - "1. It outlines confidentiality and privacy requirements for handling member information, with provisions that survive contract termination.\n", - "2. It requires contracted providers/subcontractors to comply with federal and state laws related to data breach notification.\n", - "3. It mandates that all laboratory sites have the proper Clinical Laboratory Improvement Amendment (CLIA) certificates.\n", - "4. It obligates contracted providers/subcontractors to assist the health plan in identifying, investigating, and addressing fraud, waste, and abuse.\n", - "5. It allows the health plan to withhold payments or forward them to the state if the state has taken such actions against the contracted provider/subcontractor.\n", - "\n", - "This document appears to contain legal and compliance information rather than details about the line of business operations.\n", - "Page 66: This page appears to contain legal terminology and provisions related to the responsibilities of the Health Plan and the Medicaid Fraud Division (MFD) regarding audits, investigations, and recovery of funds from providers and members. It also includes provisions related to third-party liability and the circumstances under which a provider can bill the Health Plan first before coordinating with a liable third party.\n", - "\n", - "This information seems to be more focused on legal and operational requirements rather than directly related to the line of business. The content does not appear to contain any significant strategic or competitive insights.\n", - "Page 67: This page appears to contain important legal information related to third-party liability (TPL) and member protections against liability for payment. Specifically:\n", - "\n", - "1. It discusses the circumstances under which a contracted provider/subcontractor can bill the health plan without receiving a written denial from a third party.\n", - "2. It outlines the provider/subcontractor's obligations to notify the health plan about changes in a member's health insurance coverage, retained legal counsel, and the death of a member aged 55 or older.\n", - "3. It explains the general rule that providers cannot seek payment from the member, with some exceptions, such as when the service is not a covered service or is determined to be not medically necessary.\n", - "4. Overall, this page seems to contain important legal terminology and requirements related to the line of business, rather than just general legal terminology.\n", - "Page 68: This page appears to contain legal terminology and requirements related to healthcare provider agreements and Medicaid/Medicare reimbursement. It does not seem to have any important information directly related to the line of business. The key points are:\n", - "\n", - "1. Conditions under which a member can be billed directly by a provider, such as when the service is not covered or the member has failed to remit payment from a third-party insurer.\n", - "2. Circumstances where a member cannot be held responsible for the cost of care, such as emergency services or referrals to non-participating specialists.\n", - "3. A requirement that all services under the agreement be performed within the United States.\n", - "4. A prohibition on further delegation of any delegated activities.\n", - "\n", - "Overall, this page appears to be focused on legal and regulatory compliance rather than business operations.\n", - "Page 69: This document appears to be an attachment to a Medicare Advantage Program contract. The key points are:\n", - "\n", - "1. It outlines the network participation requirements for providers to provide covered services to members enrolled in Medicare Advantage (MA) plans and dual-eligible demonstration models.\n", - "\n", - "2. It defines various terms related to the MA program, such as \"CMS Contract,\" \"Deemed Eligible Member,\" \"Deeming Period,\" \"DSNP,\" \"Dual Eligible Member,\" \"Emergency Medical Condition,\" and \"Emergency Services.\"\n", - "\n", - "This document contains important information related to the line of business, specifically the Medicare Advantage program. It does not appear to be just legal terminology, but rather operational details for providers participating in the MA program.\n", - "Page 70: This document appears to contain legal terminology and details related to the Medicare Advantage (MA) program, rather than specific information about a line of business. The key points are:\n", - "\n", - "1. Definitions of terms related to the MA program, such as \"MA Benefit Plan\", \"Medically Necessary\", and \"Member\".\n", - "2. Listing of the laws, regulations, and program requirements that apply to the MA program, including the Social Security Act, 42 CFR Parts 422 and 423, and various CMS guidance and bulletins.\n", - "3. The information provided seems to be more focused on the legal and regulatory framework of the MA program, rather than details about a specific line of business.\n", - "\n", - "Based on the content, this document does not appear to contain important information related to a specific line of business, but rather focuses on the legal and regulatory aspects of the Medicare Advantage program.\n", - "Page 71: This page appears to contain mostly legal terminology and provisions related to the agreement between the parties involved. It does not seem to have any significant information directly related to the line of business. The key points are:\n", - "\n", - "1. The provisions in this attachment prevail over any inconsistent or contrary language in the main agreement, except where the agreement exceeds the minimum requirements of the attachment.\n", - "\n", - "2. The health plan is entitled to oversee the activities of the contracted provider and its subcontractors, and is accountable to CMS for such activities. \n", - "\n", - "3. The contracted provider and its providers must comply with all applicable Medicare laws, rules, regulations, and CMS instructions.\n", - "\n", - "4. The contracted provider and its providers must allow audits by CMS or its designees, maintain records for a minimum of 10 years, and comply with the health plan's policies and procedures.\n", - "\n", - "5. When assisting members with enrollment decisions, providers must remain neutral and provide an objective assessment of available options that are in the member's best interest.\n", - "\n", - "Overall, this page appears to contain standard legal and compliance-related provisions, rather than information directly pertaining to the line of business.\n", - "Page 72: This page appears to contain important information related to legal and compliance requirements for a contracted provider. The key points are:\n", - "\n", - "1. The contracted provider and its providers/subcontractors must maintain records related to the CMS contract for 10 years from the end of the contract period or completion of any audit, whichever is later.\n", - "\n", - "2. The health plan, DHHS, Comptroller General, or their designees have the right to audit, evaluate, collect, and inspect any records related to the CMS contract directly from the contracted provider and its subcontractors.\n", - "\n", - "3. The contracted provider and its subcontractors must make their premises, facilities, equipment, records, and other relevant information available for inspection, evaluation, and auditing by the health plan, DHHS, Comptroller General, or their designees.\n", - "\n", - "4. The contracted provider must ensure the privacy and accuracy of member records, including abiding by all federal and state laws regarding confidentiality and disclosure of medical and enrollment information.\n", - "\n", - "This page contains important legal and compliance information, but does not appear to have direct relevance to the line of business operations.\n", - "Page 73: This page appears to contain legal terminology and information related to compliance requirements for a healthcare provider contract. The key points are:\n", - "\n", - "1. Providers must ensure medical information is released only as per applicable laws and regulations.\n", - "2. Providers cannot require prior authorization for emergency services until the patient is stabilized.\n", - "3. Providers cannot hold members liable for fees that are the legal obligation of the health plan, including in cases of insolvency.\n", - "4. Providers must hold dual-eligible and deemed-eligible members harmless from being billed for Medicare Part A and B expenses.\n", - "\n", - "This information seems focused on legal and regulatory requirements for healthcare providers, and does not appear to contain important details about the line of business itself.\n", - "Page 74: This document outlines various contractual terms and obligations between a health plan and a contracted provider for Medicare Part A or B services. The key points are:\n", - "\n", - "1. The contracted provider must comply with the health plan's contractual obligations under CMS (Centers for Medicare & Medicaid Services) contracts.\n", - "\n", - "2. The health plan must promptly pay clean claims submitted by the contracted provider.\n", - "\n", - "3. The contracted provider must continue to provide covered services to members even after the agreement's expiration or termination.\n", - "\n", - "4. The contracted provider must ensure that service hours and availability do not discriminate against Medicare enrollees.\n", - "\n", - "This document mainly contains legal terminology and contractual requirements, and does not appear to have significant information directly related to the line of business. It outlines the obligations and expectations between the health plan and the contracted provider for Medicare-related services.\n", - "Page 75: This document appears to contain legal terminology and requirements related to the operation of a healthcare plan, rather than information directly relevant to a specific line of business. The key points are:\n", - "\n", - "1. The document outlines requirements for providers, contractors, and subcontractors to comply with Medicare program rules, including maintaining a compliance program and allowing for reporting of potential issues.\n", - "\n", - "2. It specifies that the healthcare plan retains the right to approve, suspend, or terminate any arrangements the provider makes with other providers, contractors, or subcontractors.\n", - "\n", - "3. It addresses requirements for delegating any services or activities to other entities, including the need for written agreements and ongoing monitoring by the healthcare plan.\n", - "\n", - "4. It includes a requirement for the healthcare plan to provide a level of payment to Federally Qualified Health Centers (FQHCs) that is not less than the amount paid to other similar providers.\n", - "\n", - "Overall, this document appears to be focused on regulatory and compliance requirements rather than specific business operations, and does not seem to contain important information directly related to a particular line of business.\n", - "Page 76: This page appears to be an attachment to a contract, specifically related to the compensation details. It does not contain any information directly related to the line of business or operations. The content is primarily legal terminology and contract language, detailing the compensation structure and any associated terms or conditions. There is no significant information here that would be considered important for understanding the core business activities or operations.\n", - "Page 77: The page outlines the compensation rates and payment terms for professional services provided to Medicaid/FamilyCare members in New Jersey under the agreement. The key points are:\n", - "\n", - "1. The compensation rates are based on the CMS Medicare physician fee schedule, with some exceptions for VFC vaccine administration and EPSDT screenings.\n", - "2. Health Plan is required to process and pay or deny clean claims within specific timeframes.\n", - "3. Health Plan will implement changes to the CMS Medicare fee schedules on a prospective basis.\n", - "4. Health Plan may implement successor codes for deleted or retired codes and determine rates for covered services not included in the CMS Medicare fee schedules.\n", - "\n", - "This appears to contain important information related to the line of business, as it outlines the specific compensation and payment terms for providing professional services to Medicaid/FamilyCare members in New Jersey.\n", - "Page 78: The provided text appears to be a section of a legal contract or agreement. It discusses the basis for determining the amount of compensation for a treating provider, which is based on the provider's licensure and the health plan's credentialing requirements for that discipline, rather than the provider's academic credentials. This information seems to be related to legal and contractual terms, rather than specific details about the line of business. The summary can be provided in 5 sentences as requested.\n", - "Page 79: This page contains information related to the Medicare Advantage Program Compensation for professional (fee-for-service) services. The key points are:\n", - "\n", - "1. The compensation rates in this attachment apply for Benefit Plans under CMS Contracts.\n", - "2. Compensation for professional Covered Services is based on the lesser of the Provider's usual and customary billed charges or a percentage of the CMS Medicare physician fee schedule.\n", - "3. The Health Plan will process and pay or deny Clean Claims within 45 days of receipt.\n", - "4. The Health Plan will implement changes to CMS's fee schedules on the later of certain specified dates.\n", - "5. The Health Plan may implement successor codes for deleted or retired codes, and will determine rates for Covered Services not included in the CMS Medicare fee schedule.\n", - "\n", - "This appears to be primarily legal terminology related to the compensation structure for providers under the Medicare Advantage Program, rather than containing important information about a specific line of business.\n", - "Page 80: The page provided contains information related to the compensation paid to providers based on their licensure and the credentialing requirements of the health plan, rather than their academic credentials. This information appears to be related to the line of business, as it provides details on the criteria used to determine the amount of compensation for healthcare providers. The page does not contain any significant legal terminology, and the information seems to be focused on the business aspects of provider compensation.\n", - "Page 81: This page appears to be an attachment titled \"List of Programs\" related to a contract between New Jersey and Rutgers Health Group. It lists various programs and their corresponding contract numbers, but does not seem to contain any information directly related to a specific line of business. The content appears to be more focused on legal terminology and contractual details rather than substantive business information.\n", - "\n", - "In summary, this page seems to provide a list of program details and contract information, but does not appear to have any important information related to a specific line of business. It appears to be primarily legal in nature.\n", - "Page 82: This document appears to be a confidentiality agreement between two parties. It outlines the obligations of both parties to maintain the confidentiality of information shared during the course of their business relationship. The agreement covers topics such as the definition of confidential information, the permitted use of such information, and the consequences of unauthorized disclosure. While this document contains legal terminology, it does not appear to have any specific information related to the line of business or operations of the parties involved. It is a standard confidentiality agreement that is commonly used in a variety of business contexts.\n", - "Processed 82 pages\n", - "\n" - ] - } - ], - "source": [ - "import os\n", - "import regex as re\n", - "\n", - "def run_prompt_preprocess(pages, max_tokens=512):\n", - "\n", - " summaries = {}\n", - " for i, page_text in enumerate(pages):\n", - " prompt = (f\"Summarize this page and indicate whether it has important information related \"\n", - " f\"to line of business or if it just legal terminology. No more than 5 sentences. \\n\\n{page_text}\")\n", - " \n", - " summary = invoke_claude_3(prompt, max_tokens)\n", - " \n", - " summaries[i+1] = summary # Page numbers are typically 1-indexed\n", - " return summaries\n", - "\n", - "\n", - "directory_path = 'tests'\n", - "files = os.listdir(directory_path)\n", - "\n", - "for file_name in files:\n", - " file_path = os.path.join(directory_path, file_name)\n", - " if os.path.isfile(file_path):\n", - " try:\n", - " pages = preprocess_text_file(file_path)\n", - " page_summaries = run_prompt_preprocess(pages)\n", - " print(f\"File: {file_name}\")\n", - " for page_number, summary in page_summaries.items():\n", - " print(f\"Page {page_number}: {summary}\")\n", - " print(f\"Processed {len(pages)} pages\\n\")\n", - " except Exception as e:\n", - " print(f\"Error processing file {file_name}: {e}\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Saved page summaries to page_summaries.json\n" - ] - } - ], - "source": [ - "import json\n", - "\n", - "# Assuming page_summaries is your dictionary\n", - "# print(type(page_summaries))\n", - "\n", - "# Specify the file path where you want to save the JSON data\n", - "file_path = 'page_summaries.json'\n", - "\n", - "# Save the dictionary to a JSON file\n", - "with open(file_path, 'w', encoding='utf-8') as file:\n", - " json.dump(page_summaries, file, ensure_ascii=False, indent=4)\n", - "\n", - "print(f\"Saved page summaries to {file_path}\")\n" + " return (response_body['content'][0]['text'])" ] }, { @@ -1151,309 +113,837 @@ "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Page relevance to contract details:\n", - "Page 1: No.\n", - "Page 2: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract.\n", - "Page 3: No.\n", - "Page 4: No.\n", - "Page 5: No.\n", - "Page 6: No.\n", - "Page 7: Yes, this summary contains information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract.\n", - "Page 8: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on provider obligations and requirements related to prior authorization, referrals, non-covered services, carve-out arrangements, and claims submission, but does not mention the specific contract details requested.\n", - "Page 9: No.\n", - "Page 10: No.\n", - "Page 11: No.\n", - "Page 12: No.\n", - "Page 13: No.\n", - "Page 14: No.\n", - "Page 15: No.\n", - "Page 16: No.\n", - "Page 17: No.\n", - "Page 18: No.\n", - "Page 19: No, this summary does not contain information relevant to the specific contract details mentioned, such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary appears to focus on general contractual provisions and legal requirements, rather than the specific details of the contract itself.\n", - "Page 20: No.\n", - "Page 21: No.\n", - "Page 22: No.\n", - "Page 23: No.\n", - "Page 24: No.\n", - "Page 25: Yes.\n", - "Page 26: No.\n", - "Page 27: No.\n", - "Page 28: No.\n", - "Page 29: No.\n", - "Page 30: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on disclosures related to business transactions, ownership, and criminal convictions, but does not provide any details about the specific contract terms or coverage.\n", - "Page 31: No.\n", - "Page 32: No.\n", - "Page 33: No.\n", - "Page 34: No.\n", - "Page 35: No.\n", - "Page 36: No.\n", - "Page 37: No.\n", - "Page 38: No.\n", - "Page 39: No.\n", - "Page 40: No.\n", - "Page 41: No.\n", - "Page 42: Yes, the summary contains information relevant to contract details such as line of business (Medicaid/FamilyCare), effective date (until June 30, 2021), network for plan (Any Willing Provider and Any Willing Plan), and termination date for the contract (June 30, 2021).\n", - "Page 43: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on details related to claims processing and payment for MLTSS members, as well as quality initiatives and compliance requirements, but does not mention the specific contract details requested.\n", - "Page 44: No.\n", - "Page 45: No, the summary does not contain information relevant to specific contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on general contractual and legal requirements between a provider and a health plan, but does not provide the specific contract details mentioned in the question.\n", - "Page 46: No.\n", - "Page 47: No.\n", - "Page 48: No.\n", - "\n", - "The summary provided does not contain information relevant to the specific contract details like line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on contractual and regulatory requirements between the health plan and the contracted provider/subcontractor, but does not mention the key details about the contract itself.\n", - "Page 49: No.\n", - "Page 50: No.\n", - "Page 51: No.\n", - "Page 52: No, this summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on regulatory requirements for billing and record-keeping for providers participating in the Medicaid/NJ FamilyCare program, but does not provide any specific contract details.\n", - "Page 53: No.\n", - "Page 54: No.\n", - "Page 55: No.\n", - "Page 56: No.\n", - "Page 57: No.\n", - "Page 58: No.\n", - "Page 59: No.\n", - "Page 60: No.\n", - "Page 61: No.\n", - "Page 62: No.\n", - "Page 63: No.\n", - "Page 64: No.\n", - "Page 65: No.\n", - "Page 66: No.\n", - "Page 67: No.\n", - "Page 68: No.\n", - "Page 69: Yes.\n", - "Page 70: No.\n", - "Page 71: No, the summary provided does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary focuses on legal and compliance-related provisions of the agreement, but does not mention any specific contract details.\n", - "Page 72: No.\n", - "Page 73: No.\n", - "Page 74: No\n", - "Page 75: No, the summary does not contain information relevant to contract details such as line of business (Contract LOB), effective date, network for plan, service area, and termination date for the contract. The summary indicates that the document is focused on regulatory and compliance requirements rather than specific business operations and does not include details about a particular line of business.\n", - "Page 76: No.\n", - "Page 77: Yes, the summary contains information relevant to contract details such as line of business (Medicaid/FamilyCare), effective date (not explicitly stated but can be inferred), and service area (New Jersey). However, it does not explicitly mention the network for the plan or the termination date for the contract.\n", - "Page 78: No.\n", - "Page 79: No.\n", - "Page 80: No.\n", - "Page 81: No.\n", - "Page 82: No.\n" - ] + "data": { + "text/plain": [ + "'test'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "def identify_relevant_pages(pages_with_summaries, max_tokens=256):\n", - " \"\"\"\n", - " Identifies pages that contain relevant information related to contract details such as\n", - " CONTRACT_LOB, LOB_PRICING_TERMS_EFFECTIVE_DT, Plan, CONTRACT_NETWORK, \n", - " CONTRACT_SERVICE_AREA, LOB_PRICING_TERMS_TERMINATION_DT.\n", - "\n", - " Args:\n", - " - pages_with_summaries (dict): A dictionary where keys are page numbers and values are summaries.\n", - " - max_tokens (int): Maximum number of tokens for the model's response.\n", - "\n", - " Returns:\n", - " - dict: A dictionary where keys are page numbers and values are 'yes' or 'no', indicating the relevance.\n", - " \"\"\"\n", - " relevant_pages = {}\n", - " for page_number, summary in pages_with_summaries.items():\n", - " # Construct the prompt asking about the relevance of the page to the contract details\n", - " prompt = (f\"Does this summary contain information relevant to contract details such as \"\n", - " f\"line of business (Contract LOB), effective date, network for plan, service area, \"\n", - " f\"and termination date for the contract? Summary: \\\"{summary}\\\" Answer with 'yes' or 'no'.\")\n", - " \n", - " \n", - " response = invoke_claude_3(prompt, max_tokens)\n", - " \n", - " \n", - " relevant_pages[page_number] = response\n", - " \n", - " return relevant_pages\n", - "\n", - "\n", - "\n", - "import json\n", - "\n", - "# Specify the file path from which you want to load the JSON data\n", - "file_path = 'page_summaries.json'\n", - "\n", - "# Read the JSON file and convert it into a dictionary\n", - "with open(file_path, 'r', encoding='utf-8') as file:\n", - " page_summaries = json.load(file)\n", - " \n", - "\n", - "relevant_pages = identify_relevant_pages(pages_with_summaries=page_summaries, max_tokens=256)\n", - "print(\"Page relevance to contract details:\")\n", - "for page_number, is_relevant in relevant_pages.items():\n", - " print(f\"Page {page_number}: {is_relevant}\")\n" + "# Test\n", + "invoke_claude_3(\"Write 'test', nothing more.\", max_tokens=100)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# read texts\n", + "# Define the path to the file\n", + "folder_path = r'C:\\\\Users\\\\kminhas\\\\Documents\\\\Doczy\\\\doczy.ai\\\\texts\\\\training-data\\\\contract-text-file\\\\test1\\\\'\n", + "\n", + "file_contents = {}\n", + "\n", + "for filename in os.listdir(folder_path):\n", + " # Create full file path\n", + " file_path = os.path.join(folder_path, filename)\n", + " # Check if the file is a text file\n", + " if os.path.isfile(file_path) and file_path.endswith('.txt'):\n", + " try:\n", + " # First attempt to open the file with UTF-8 encoding\n", + " with open(file_path, 'r', encoding='utf-8') as file:\n", + " file_contents[filename] = file.read()\n", + " except UnicodeDecodeError:\n", + " # If UTF-8 fails, try reading the file with ANSI encoding\n", + " try:\n", + " with open(file_path, 'r', encoding='cp1252') as file:\n", + " file_contents[filename] = file.read()\n", + " except UnicodeDecodeError:\n", + " # If ANSI also fails, log an error message or handle it accordingly\n", + " print(f\"Failed to decode {file_path} with UTF-8 and cp1252 encodings.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "def BOTTOM_UP_MAIN(page, payer):\n", + " return f\"\"\"\n", + "### PAGE START ### {page} ### PAGE END\n", + "\n", + "The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. \n", + "\n", + "Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more.\n", + "\n", + "If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.\n", + "\n", + "Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. \n", + "It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list.\n", + "\n", + "Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:\n", + "\n", + "'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional.\n", + "'PLAN' : What Plan is the reimbursement schedule for? This the brand name of the health plan.\n", + "'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar.\n", + "'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement.\n", + "'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A.\n", + "'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program.\n", + "'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'.\n", + "'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'.\n", + "'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. \n", + "'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based.\n", + "'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable.\n", + "'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'.\n", + "'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met.\n", + "'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar.\n", + "'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.\n", + "'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.\n", + "\n", + "When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of.\n", + "As an example, if you see that a reimbursement says something like \"specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule\", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step.\n", + "\n", + "Only return the list, with no other commentary or explanation.\n", + "\"\"\"\n", + "\n", + "test_prompt = \"\"\"\n", + "Definitions:\n", + "Primary Reimbursement - The generalized rate for which a specialty or service is reimbursed.\n", + "Carveout Reimbursement - A specific rate that is different or specific than the primary service\n", + "\n", + "Text: \n", + "Services:\n", + "Professional Services\n", + "Physician agrees to participate in the Benefit Plan/Program described in this Exhibit and authorizes, through its\n", + "signature below, the transfer of all payment/reimbursement terms and obligations under the Agreement to Payors\n", + "as set forth in this Agreement.\n", + "Subject to allowed reductions that Community may apply as a result of Physician's non-compliance with any\n", + "applicable credentialing, billing, Prior Authorization, Referral, or Utilization Management guidelines, or other\n", + "Community Protocols referenced in this Agreement, Community shall compensate Physician for Covered Services\n", + "rendered by Physician or Healthcare Provider according to the lessor of Contracted Physician's Billed Charges or\n", + "the following schedule, less any Member Expense:\n", + "One hundred and five percent (105%) of the then current Texas Medicaid Fee Schedule unless otherwise\n", + "stated herein, in accordance with the Texas Medicaid reimbursement methodology.\n", + "Clinical Laboratory services (Current Procedural Coding (CPT) codes 80000 through 87999) shall be\n", + "reimbursed at one hundred percent (100%) of the then current Texas Medicaid Clinical Laboratory Fee\n", + "Schedule, in accordance with Texas Medicaid reimbursement methodology. Physician agrees to only bill\n", + "for and Community shall be obligated to only pay for clinical laboratory services for which Physician holds\n", + "CLIA waiver.\n", + "Radiology services (CPT codes 70000 through 79999) shall be reimbursed at one hundred percent (100%)\n", + "of the Texas Medicaid Fee Schedule, in accordance with Texas Medicaid reimbursement methodology.\n", + "Drugs dispensed and administered by Physician that are Covered Services under this Agreement shall be\n", + "reimbursed at one hundred percent (100%) of the Texas Medicaid reimbursement, in accordance with Texas\n", + "Medicaid reimbursement methodology.\n", + "Compensation Notes:\n", + "If Physieian bills for a Covered Service for which no reimbursement is defined by any of the methods listed above,\n", + "Community shall compensate Physician according to thirty percent (30%) of Physician's Billed Charges.\n", + "\n", + "\n", + "Based on the definitions provided, in the text above, which are Primary Reimbursements and which are Carveout Reimbursements?\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Saved page answers to relevant_pages.json\n" + "Based on the definitions and the text provided, here is how I would categorize the reimbursements mentioned:\n", + "\n", + "Primary Reimbursement:\n", + "- 105% of the then current Texas Medicaid Fee Schedule (for most professional services)\n", + "\n", + "Carveout Reimbursements (specific rates different than the primary):\n", + "- 100% of the then current Texas Medicaid Clinical Laboratory Fee Schedule (for clinical laboratory services CPT 80000-87999)\n", + "- 100% of the Texas Medicaid Fee Schedule (for radiology services CPT 70000-79999) \n", + "- 100% of the Texas Medicaid reimbursement (for drugs dispensed and administered by the physician)\n", + "\n", + "The text states that most professional services will be reimbursed at 105% of the Texas Medicaid Fee Schedule, which appears to be the generalized or primary reimbursement rate. However, it then carves out specific reimbursement rates for clinical laboratory services, radiology services, and drugs that are different from the primary 105% rate.\n" ] } ], "source": [ - "import json\n", - "\n", - "# Assuming page_summaries is your dictionary\n", - "# print(type(page_summaries))\n", - "\n", - "# Specify the file path where you want to save the JSON data\n", - "file_path = 'relevant_pages.json'\n", - "\n", - "# Save the dictionary to a JSON file\n", - "with open(file_path, 'w', encoding='utf-8') as file:\n", - " json.dump(relevant_pages, file, ensure_ascii=False, indent=4)\n", - "\n", - "print(f\"Saved page answers to {file_path}\")" + "print(invoke_claude_3(test_prompt, max_tokens=1000))" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 40, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Page 7: \n", - "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", - "information requested by Health Plan, or...\n", - "Page 25: \n", - "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", - "a week emergency and urgent care coverag...\n", - "Page 42: \n", - "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", - "ATTACHMENT B-2\n", - "NEW JERSEY MEDICAID/FAMIL...\n", - "Page 69: \n", - "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", - "ATTACHMENT B-3\n", - "MEDICARE ADVANTAGE PROGRA...\n", - "Page 77: \n", - "DocuSign Envelope ID: 86A03D83-00DE-4D11-8084-A43F77EB059B\n", - "ATTACHMENT C-1\n", - "NEW JERSEY MEDICAID/FAMIL...\n" - ] - } - ], + "outputs": [], "source": [ - "import json\n", + "# Add 100% to billed charges when there isn't already a percentage\n", + "def clean_billed_charges(text):\n", + " # This function finds all occurrences of 'billed charges' and checks the preceding 15 characters for '%'\n", + " def replace(match):\n", + " start_pos = match.start() # Get the start position of the match\n", + " pre_text = text[max(0, start_pos-15):start_pos] # Extract up to 15 characters before the match\n", + " if '%' in pre_text:\n", + " return match.group() # If '%' is found, return the original 'billed charges'\n", + " else:\n", + " return '100% of billed charges' # If '%' is not found, prepend '100% of'\n", "\n", - "# Load the page answers from JSON file\n", - "page_answers_path = 'relevant_pages.json'\n", - "with open(page_answers_path, 'r', encoding='utf-8') as file:\n", - " page_answers = json.load(file)\n", + " # Use regular expression to replace 'billed charges' with the modified version if needed, using re.IGNORECASE for case insensitivity\n", + " return re.sub(r'\\bbilled charges\\b', replace, text, flags=re.IGNORECASE)\n", "\n", - "# Use regex to match \"yes\" responses, allowing for flexible matching\n", - "yes_regex = re.compile(r'\\byes\\b', re.IGNORECASE)\n", + "def split_text(text):\n", + " text_list = text.split('Start of Page No. = ')\n", + " text_dict = {page.split()[0] : page for page in text_list}\n", + " return text_dict\n", "\n", - "# Filter pages with a \"Yes\" response using regex\n", - "yes_page_numbers = [int(page) for page, response in page_answers.items() if yes_regex.search(response.strip())]\n", - "\n", - "def get_yes_pages_text(file_path, yes_page_numbers):\n", - " # Get all pages\n", - " pages = preprocess_text_file(file_path)\n", + "def string_to_dict(string_dict):\n", + " # Regular expression to capture key-value pairs\n", + " regex_pattern = r'\\{\\s*([^}]*)\\s*\\}'\n", + " key_value_pattern = r'\"([^\"]+)\":\\s*\"([^\"]+)\"'\n", + " data = []\n", + " for page in string_dict.keys():\n", + " dict_string = string_dict[page]\n", + " groups = re.findall(regex_pattern, dict_string)\n", + " for group in groups:\n", + " key_values = re.findall(key_value_pattern, group)\n", + " key_value_dict = {key: value for key, value in key_values}\n", + " key_value_dict['page_num'] = page\n", + " data.append(key_value_dict)\n", + " return data\n", " \n", - " # Filter for pages with a \"Yes\" response\n", - " yes_pages_text = {page_number: pages[page_number-1] for page_number in yes_page_numbers if page_number <= len(pages)}\n", + "def run_bottom_up_prompt(text_dict, client_name, tokens):\n", + " answer_dict = {}\n", + " for page_number in text_dict.keys():\n", + " if page_number.isdigit():\n", + " if int(page_number) in (22, 23):\n", + " prompt = BOTTOM_UP_MAIN(text_dict[page_number], client_name)\n", + " #print(len(prompt))\n", + " answer = invoke_claude_3(prompt, max_tokens=tokens)\n", + " answer_dict[page_number] = answer\n", + " return answer_dict\n", + "\n", + "def run_lesser_of_prompt(text_dict, result_dicts):\n", + " valid_dicts = [dict for dict in result_dicts if dict['LESSER_OF_LANGUAGE_IND'] == 'Y']\n", + " pages_to_check = list(set([dict['page_num'] for dict in valid_dicts]))\n", + " for page_num in pages_to_check:\n", + " page_text = text_dict[page_num]\n", + " dicts_on_page = [dict for dict in valid_dicts if dict['page_num'] == page_num]\n", + " print(dicts_on_page)\n", + "\n", + "\n", + "def append_secondary(result_dicts):\n", + " from collections import defaultdict\n", + "\n", + " # Helper function to create a key from dict excluding certain keys\n", + " def create_key(d):\n", + " return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT'))\n", + "\n", + " # Group dictionaries by non-REIMBURSEMENT fields\n", + " grouped = defaultdict(list)\n", + " for d in result_dicts:\n", + " grouped[create_key(d)].append(d)\n", + "\n", + " # Process groups to create combined entries\n", + " combined_results = []\n", + " for key, group in grouped.items():\n", + " # Create a new dictionary based on non-REIMBURSEMENT fields\n", + " new_dict = {k: v for k, v in key}\n", + " # Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary\n", + " for i, entry in enumerate(group[:3]): # Process only the first three entries\n", + " for k, v in entry.items():\n", + " if k.startswith('REIMBURSEMENT'):\n", + " new_key = f'{[\"PRIMARY\", \"SECONDARY\", \"TERTIARY\"][i]}_{k}'\n", + " new_dict[new_key] = v\n", + " combined_results.append(new_dict)\n", + "\n", + " return combined_results\n", " \n", - " return yes_pages_text\n", "\n", - "# Example file path\n", - "file_path = 'tests/22-3146927_ICMProviderAgreement_273971_1 MU.txt'\n", + "def bottom_up_extract(filename, client_name, contract_text):\n", + " # add '100% of ' to billed charges\n", + " contract_text = clean_billed_charges(contract_text)\n", "\n", - "# Get text for pages with a \"Yes\" response\n", - "yes_pages_text = get_yes_pages_text(file_path, yes_page_numbers)\n", + " # Split text into pages\n", + " text_dict = split_text(contract_text)\n", "\n", - "for page_number, text in yes_pages_text.items():\n", - " print(f\"Page {page_number}: {text[:100]}...\")\n", - "\n" + " # Extract for each page\n", + " answer_dicts = run_bottom_up_prompt(text_dict, client_name, 8000)\n", + "\n", + " # Convert results to list of dictionary\n", + " answer_dicts = string_to_dict(answer_dicts)\n", + "\n", + " # combine dicts to primary, secondary, tertiary fields\n", + " answer_dicts = append_secondary(answer_dicts)\n", + "\n", + " # Add filename\n", + " for dict in answer_dicts:\n", + " dict['Filename'] = filename\n", + " \n", + " return answer_dicts\n" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_dicts = []\n", + "for filename, file_text in file_contents.items():\n", + " file_results_list = bottom_up_extract(filename, 'CareSource', file_text)\n", + " all_dicts += file_results_list" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "all_results_df = pd.DataFrame(all_dicts)" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [], + "source": [ + "all_results_df.to_csv('test_output_20240417.csv')" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [], + "source": [ + "filename = '2017 01 01 Raul A. Rivera & Associates, P.A. (Master)PHY Executed MU.txt'\n", + "result = bottom_up_extract(filename, 'Community Health Choice', file_contents[filename])" + ] + }, + { + "cell_type": "code", + "execution_count": 43, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Page 7 response: Here are the extracted instances of information from the provided text:\n", - "\n", - "[\n", - " {\n", - " \"CONTRACT_LOB\": \"MEDICAID\",\n", - " \"LOB_PRICING_TERMS_EFFECTIVE_DT\": \"08/25/2020\",\n", - " \"Plan\": \"N/A\",\n", - " \"CONTRACT_NETWORK\": \"N/A\",\n", - " \"CONTRACT_SERVICE_AREA\": \"New Jersey\",\n", - " \"LOB_PRICING_TERMS_TERMINATION_DT\": \"N/A\",\n", - " \"CONTRACT_MARKETPLACE_METAL_LEVEL\": \"N/A\"\n", - " }\n", - "]\n", - "Page 25 response: There are no explicit references to the specific Line of Business, Plan, Network, Service Area, Effective Date, Termination Date, or Marketplace Metal Level in the provided text. The text appears to describe a general provider contract agreement, but does not contain the specific details requested in the instructions.\n", - "Page 42 response: Based on the provided text, there is no explicit information about a contract, line of business, pricing terms, plan, network, service area, or termination date. The text appears to be discussing the New Jersey Medicaid/FamilyCare Program and provisions related to \"MLTSS Any Willing Provider\" and \"Any Willing Plan\" status for certain provider types. There is no information that can be extracted in the format you requested.\n", - "Page 69 response: Based on the provided text, there is no explicit information about the Line of Business, Plan, Network, Service Area, Effective Date, or Termination Date. The text is about the Medicare Advantage program and contains definitions related to that program, such as \"CMS Contract\", \"Dual Eligible Member\", and \"Emergency Services\". There is no information about pricing terms or compensation that can be extracted in the requested format.\n", - "Page 77 response: Based on the provided text, the following information can be extracted:\n", - "\n", - "[\n", - " {\n", - " \"CONTRACT_LOB\": \"MEDICAID\",\n", - " \"LOB_PRICING_TERMS_EFFECTIVE_DT\": \"08/25/2020\",\n", - " \"Plan\": \"FFS\",\n", - " \"CONTRACT_NETWORK\": \"FFS\", \n", - " \"CONTRACT_SERVICE_AREA\": \"New Jersey\",\n", - " \"LOB_PRICING_TERMS_TERMINATION_DT\": null\n", - " }\n", - "]\n" - ] + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
CONTRACT_LOBPLANPROGRAMPROV_TYPEPROV_SPECIALTYAPPLICABLE_TOGREATER_OF_LANGUAGE_INDLESSER_OF_LANGUAGE_INDpage_numPRIMARY_REIMBURSEMENT_METHODOLOGYPRIMARY_REIMBURSEMENT_FEE_SCHEDULEPRIMARY_REIMBURSEMENT_CODESPRIMARY_REIMBURSEMENT_RATE_ESCALATOR_INDPRIMARY_REIMBURSEMENT_EXCEPTION_INDPRIMARY_REIMBURSEMENT_GROUPERPRIMARY_REIMBURSEMENT_FLAT_FEEPRIMARY_REIMBURSEMENT_RATESECONDARY_REIMBURSEMENT_METHODOLOGYSECONDARY_REIMBURSEMENT_FEE_SCHEDULESECONDARY_REIMBURSEMENT_CODESSECONDARY_REIMBURSEMENT_RATE_ESCALATOR_INDSECONDARY_REIMBURSEMENT_EXCEPTION_INDSECONDARY_REIMBURSEMENT_GROUPERSECONDARY_REIMBURSEMENT_FLAT_FEESECONDARY_REIMBURSEMENT_RATEFilename
0MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Family Practice, Internal Medicine, General Pr...N/ANY22Billed ChargesN/AN/ANNN/AN/A100%Texas Medicaid Fee ScheduleTexas Medicaid Fee ScheduleN/ANNN/AN/A105%2017 01 01 Raul A. Rivera & Associates, P.A. (...
1MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Clinical Laboratory servicesN/ANN22Texas Medicaid Clinical Laboratory Fee ScheduleTexas Medicaid Clinical Laboratory Fee Schedule80000 through 87999NYN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
2MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Radiology servicesN/ANN22Texas Medicaid Fee ScheduleTexas Medicaid Fee Schedule70000 through 79999NNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
3MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Drugs dispensed and administeredN/ANN22Texas Medicaid reimbursementN/AN/ANNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
4MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Professional ServicesN/ANN22Billed ChargesN/AN/ANYN/AN/A30%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
5MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Professional ServicesN/ANY23Medicare Fee ScheduleMedicare Physician Fee ScheduleN/ANNN/AN/A100%Billed ChargesN/AN/ANNN/AN/A100%2017 01 01 Raul A. Rivera & Associates, P.A. (...
6MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Clinical Laboratory servicesN/ANN23Medicare Fee ScheduleMedicare Clinical Laboratory Fee Schedule80000 through 87999NYN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
7MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Radiology servicesN/ANN23Medicare Fee ScheduleMedicare Professional Fee Schedule70000 through 79999NNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
8MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Drugs dispensed and administeredN/ANN23APS reimbursementN/AN/ANNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
9MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...N/AN/ANN23Billed ChargesN/AN/ANNN/AN/A30%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
\n", + "
" + ], + "text/plain": [ + " CONTRACT_LOB PLAN PROGRAM \\\n", + "0 Medicaid N/A STAR \n", + "1 Medicaid N/A STAR \n", + "2 Medicaid N/A STAR \n", + "3 Medicaid N/A STAR \n", + "4 Medicaid N/A STAR \n", + "5 Marketplace N/A N/A \n", + "6 Marketplace N/A N/A \n", + "7 Marketplace N/A N/A \n", + "8 Marketplace N/A N/A \n", + "9 Marketplace N/A N/A \n", + "\n", + " PROV_TYPE \\\n", + "0 Physician and/or Mid-Level Practitioners - Pri... \n", + "1 Physician and/or Mid-Level Practitioners - Pri... \n", + "2 Physician and/or Mid-Level Practitioners - Pri... \n", + "3 Physician and/or Mid-Level Practitioners - Pri... \n", + "4 Physician and/or Mid-Level Practitioners - Pri... \n", + "5 All Physician and Mid-level Healthcare Profess... \n", + "6 All Physician and Mid-level Healthcare Profess... \n", + "7 All Physician and Mid-level Healthcare Profess... \n", + "8 All Physician and Mid-level Healthcare Profess... \n", + "9 All Physician and Mid-level Healthcare Profess... \n", + "\n", + " PROV_SPECIALTY APPLICABLE_TO \\\n", + "0 Family Practice, Internal Medicine, General Pr... N/A \n", + "1 Clinical Laboratory services N/A \n", + "2 Radiology services N/A \n", + "3 Drugs dispensed and administered N/A \n", + "4 Professional Services N/A \n", + "5 Professional Services N/A \n", + "6 Clinical Laboratory services N/A \n", + "7 Radiology services N/A \n", + "8 Drugs dispensed and administered N/A \n", + "9 N/A N/A \n", + "\n", + " GREATER_OF_LANGUAGE_IND LESSER_OF_LANGUAGE_IND page_num \\\n", + "0 N Y 22 \n", + "1 N N 22 \n", + "2 N N 22 \n", + "3 N N 22 \n", + "4 N N 22 \n", + "5 N Y 23 \n", + "6 N N 23 \n", + "7 N N 23 \n", + "8 N N 23 \n", + "9 N N 23 \n", + "\n", + " PRIMARY_REIMBURSEMENT_METHODOLOGY \\\n", + "0 Billed Charges \n", + "1 Texas Medicaid Clinical Laboratory Fee Schedule \n", + "2 Texas Medicaid Fee Schedule \n", + "3 Texas Medicaid reimbursement \n", + "4 Billed Charges \n", + "5 Medicare Fee Schedule \n", + "6 Medicare Fee Schedule \n", + "7 Medicare Fee Schedule \n", + "8 APS reimbursement \n", + "9 Billed Charges \n", + "\n", + " PRIMARY_REIMBURSEMENT_FEE_SCHEDULE \\\n", + "0 N/A \n", + "1 Texas Medicaid Clinical Laboratory Fee Schedule \n", + "2 Texas Medicaid Fee Schedule \n", + "3 N/A \n", + "4 N/A \n", + "5 Medicare Physician Fee Schedule \n", + "6 Medicare Clinical Laboratory Fee Schedule \n", + "7 Medicare Professional Fee Schedule \n", + "8 N/A \n", + "9 N/A \n", + "\n", + " PRIMARY_REIMBURSEMENT_CODES PRIMARY_REIMBURSEMENT_RATE_ESCALATOR_IND \\\n", + "0 N/A N \n", + "1 80000 through 87999 N \n", + "2 70000 through 79999 N \n", + "3 N/A N \n", + "4 N/A N \n", + "5 N/A N \n", + "6 80000 through 87999 N \n", + "7 70000 through 79999 N \n", + "8 N/A N \n", + "9 N/A N \n", + "\n", + " PRIMARY_REIMBURSEMENT_EXCEPTION_IND PRIMARY_REIMBURSEMENT_GROUPER \\\n", + "0 N N/A \n", + "1 Y N/A \n", + "2 N N/A \n", + "3 N N/A \n", + "4 Y N/A \n", + "5 N N/A \n", + "6 Y N/A \n", + "7 N N/A \n", + "8 N N/A \n", + "9 N N/A \n", + "\n", + " PRIMARY_REIMBURSEMENT_FLAT_FEE PRIMARY_REIMBURSEMENT_RATE \\\n", + "0 N/A 100% \n", + "1 N/A 100% \n", + "2 N/A 100% \n", + "3 N/A 100% \n", + "4 N/A 30% \n", + "5 N/A 100% \n", + "6 N/A 100% \n", + "7 N/A 100% \n", + "8 N/A 100% \n", + "9 N/A 30% \n", + "\n", + " SECONDARY_REIMBURSEMENT_METHODOLOGY SECONDARY_REIMBURSEMENT_FEE_SCHEDULE \\\n", + "0 Texas Medicaid Fee Schedule Texas Medicaid Fee Schedule \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "5 Billed Charges N/A \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 NaN NaN \n", + "\n", + " SECONDARY_REIMBURSEMENT_CODES SECONDARY_REIMBURSEMENT_RATE_ESCALATOR_IND \\\n", + "0 N/A N \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "5 N/A N \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 NaN NaN \n", + "\n", + " SECONDARY_REIMBURSEMENT_EXCEPTION_IND SECONDARY_REIMBURSEMENT_GROUPER \\\n", + "0 N N/A \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "5 N N/A \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 NaN NaN \n", + "\n", + " SECONDARY_REIMBURSEMENT_FLAT_FEE SECONDARY_REIMBURSEMENT_RATE \\\n", + "0 N/A 105% \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "5 N/A 100% \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", + "9 NaN NaN \n", + "\n", + " Filename \n", + "0 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "1 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "2 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "3 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "4 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "5 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "6 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "7 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "8 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", + "9 2017 01 01 Raul A. Rivera & Associates, P.A. (... " + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "def process_yes_pages_with_claude(yes_pages_text):\n", - " \"\"\"\n", - " Processes each page with a \"Yes\" response through invoke_claude_3, using prompts generated by LOB_PROMPT_NEW.\n", - "\n", - " Args:\n", - " - yes_pages_text (dict): Dictionary of page numbers and their corresponding texts that have a \"Yes\" response.\n", - "\n", - " Returns:\n", - " - dict: A dictionary where keys are page numbers and values are the responses from invoke_claude_3.\n", - " \"\"\"\n", - " claude_responses = {}\n", - "\n", - " for page_number, page_text in yes_pages_text.items():\n", - " # Generate the prompt for this page\n", - " prompt = page_text + \"\\n\\n\" + LOB_PROMPT_NEW() # Modify as needed\n", - " \n", - " # Invoke Claude 3 with the generated prompt\n", - " response = invoke_claude_3(prompt, max_tokens=2000) # Adjust max_tokens as necessary\n", - " \n", - " claude_responses[page_number] = response\n", - "\n", - " return claude_responses\n", - "\n", - "# Assume yes_pages_text is obtained as described in previous steps\n", - "yes_pages_text = get_yes_pages_text(file_path, yes_page_numbers) # This should be defined based on earlier steps\n", - "\n", - "# Process each \"Yes\" page through Claude 3\n", - "claude_responses = process_yes_pages_with_claude(yes_pages_text)\n", - "\n", - "for page_number, response in claude_responses.items():\n", - " print(f\"Page {page_number} response: {response}\")\n" + "pd.DataFrame(result)" ] } ], @@ -1473,7 +963,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.11" + "version": "3.11.4" } }, "nbformat": 4, From e118f88254ce7ebaa6c6c3d5351d961b78409b40 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 23 Apr 2024 14:31:24 -0700 Subject: [PATCH 03/78] modularization --- src/bottom_up_funcs.py | 133 +++++++++++++++++++++++++++++++++++++++++ src/bottom_up_main.py | 18 ++++++ src/claude_funcs.py | 62 +++++++++++++++++++ src/config.py | 45 ++++++++++++++ src/prompts.py | 40 +++++++++++++ src/utils.py | 84 ++++++++++++++++++++++++++ 6 files changed, 382 insertions(+) create mode 100644 src/bottom_up_funcs.py create mode 100644 src/bottom_up_main.py create mode 100644 src/claude_funcs.py create mode 100644 src/config.py create mode 100644 src/prompts.py create mode 100644 src/utils.py diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py new file mode 100644 index 0000000..4017734 --- /dev/null +++ b/src/bottom_up_funcs.py @@ -0,0 +1,133 @@ +import re +import os +import shutil +import pandas as pd +from collections import defaultdict +import concurrent.futures + +import config +import utils +import prompts +import claude_funcs + +def clean_billed_charges(text): + def replace(match): + start_pos = match.start() + pre_text = text[max(0, start_pos-15):start_pos] + if '%' in pre_text: + return match.group() + else: + return '100% of billed charges' + return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) + +def split_text(text): + text_list = text.split('Start of Page No. = ') + text_dict = {page.split()[0] : page for page in text_list} + return text_dict + +def string_to_dict(string_dict): + # Regular expression to capture key-value pairs + regex_pattern = r'\{\s*([^}]*)\s*\}' + key_value_pattern = r'"([^"]+)":\s*"([^"]+)"' + data = [] + for page in string_dict.keys(): + dict_string = string_dict[page] + groups = re.findall(regex_pattern, dict_string) + for group in groups: + key_values = re.findall(key_value_pattern, group) + key_value_dict = {key: value for key, value in key_values} + key_value_dict['page_num'] = page + data.append(key_value_dict) + return data + +def run_bottom_up_primary(text_dict, tokens): + answer_dict = {} + for page_number in text_dict.keys(): + if page_number.isdigit(): + if True: #int(page_number) in (22, 23): + prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) + #print(len(prompt)) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + answer_dict[page_number] = answer + return answer_dict + +def run_lesser_of_prompt(text_dict, result_dicts): + + valid_dicts = [dict for dict in result_dicts if dict['LESSER_OF_LANGUAGE_IND'] == 'Y'] + pages_to_check = list(set([dict['page_num'] for dict in valid_dicts])) + for page_num in pages_to_check: + page_text = text_dict[page_num] + dicts_on_page = [dict for dict in valid_dicts if dict['page_num'] == page_num] + print(dicts_on_page) + +def append_secondary(result_dicts): + + # Helper function to create a key from dict excluding certain keys + def create_key(d): + return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT')) + + # Group dictionaries by non-REIMBURSEMENT fields + grouped = defaultdict(list) + for d in result_dicts: + grouped[create_key(d)].append(d) + + # Process groups to create combined entries + combined_results = [] + for key, group in grouped.items(): + # Create a new dictionary based on non-REIMBURSEMENT fields + new_dict = {k: v for k, v in key} + # Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary + for i, entry in enumerate(group[:3]): # Process only the first three entries + for k, v in entry.items(): + if k.startswith('REIMBURSEMENT'): + new_key = f'{["PRIMARY", "SECONDARY", "TERTIARY"][i]}_{k}' + new_dict[new_key] = v + combined_results.append(new_dict) + return combined_results + + +def bottom_up_primary(file_object): + + filename, contract_text = file_object + + contract_text = clean_billed_charges(contract_text) + + text_dict = split_text(contract_text) + + answer_dicts = run_bottom_up_primary(text_dict, 8000) + + answer_dicts = string_to_dict(answer_dicts) + + answer_dicts = append_secondary(answer_dicts) + + for dict in answer_dicts: + dict['Filename'] = filename + + answer_df = pd.DataFrame(answer_dicts) + + # Write output + if config.OUTPUT_MODE == '_INDIVIDUAL_': + answer_df.to_csv(f"results/{filename}_results.csv") + elif config.OUTPUT_MODE == '_CONSOLIDATED_': + if not os.path.exists('temp'): + os.makedirs('temp') + answer_df.to_csv(f"temp/{filename}_results.csv") + + +def run_bottom_up(input_dict): + + if not os.path.exists('results'): + os.makedirs('results') + + # Run prompts with multithreading + with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + executor.map(bottom_up_primary, input_dict.items()) + + # Output + if config.OUTPUT_MODE == '_CONSOLIDATED_': + consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file + utils.write_consolidated(consolidated_df) + consolidated_df.to_csv(os.path.join('results', f'consolidated_results_{config.TODAY}.csv')) # Write output + + + diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py new file mode 100644 index 0000000..00fa6de --- /dev/null +++ b/src/bottom_up_main.py @@ -0,0 +1,18 @@ +# Imports + +import utils +import config +import bottom_up_funcs + + +def main(): + input_dict = utils.read_input() + + bottom_up_funcs.run_bottom_up(input_dict) + + +if __name__ == "__main__": + main() + + + diff --git a/src/claude_funcs.py b/src/claude_funcs.py new file mode 100644 index 0000000..5dd2cde --- /dev/null +++ b/src/claude_funcs.py @@ -0,0 +1,62 @@ +import json +import anthropic + +import config + + + + +# Claude calls +def invoke_claude_2(prompt, max_tokens): + body = json.dumps( + {"prompt": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, + "max_tokens_to_sample": max_tokens, + "temperature":0.0, + "top_p":1, + "top_k":250, + "stop_sequences":[anthropic.HUMAN_PROMPT] + } + ) + + response = config.BEDROCK_RUNTIME.invoke_model( + body=body, + modelId=config.MODEL_ID_CLAUDE2, + accept="application/json", + contentType="application/json" + ) + + response_body = json.loads(response.get("body").read()) + + response_text = response_body['completion'] + return response_text + +def invoke_claude_3(prompt, max_tokens = 0): + prompt = prompt + body = json.dumps({ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": max_tokens, + "temperature": 0.0, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text":prompt + } + ] + } + ] + } + ) + + response = config.BEDROCK_RUNTIME.invoke_model( + body=body, + modelId=config.MODEL_ID_CLAUDE3, + accept="application/json", + contentType="application/json" + ) + + response_body = json.loads(response.get("body").read()) + + return (response_body['content'][0]['text']) \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 0000000..8b14dd1 --- /dev/null +++ b/src/config.py @@ -0,0 +1,45 @@ +import os +from datetime import datetime +import boto3 + +# General Settings +WRITE_OUTPUT = True +VERBOSE = True +CLIENT_NAME = 'CareSource' +TODAY = datetime.now().strftime("%Y%m%d") + +# Multithread Settings +MAX_WORKERS = 20 + +# AWS Keys +AWS_ACCESS_KEY_ID="replace daily" +AWS_SECRET_ACCESS_KEY="replace daily" +AWS_SESSION_TOKEN="replace daily" + +# I/O Options +READ_MODE = '_LOCAL_' # OR '_S3_' +OUTPUT_MODE = '_CONSOLIDATED_' # or '_INDIVIDUAL_' + +# File Paths +LOCAL_PATH = 'docs/test/' # Replace with local + +# S3 Settings +S3_CLIENT = boto3.client('s3', + region_name="us-east-2", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN + ) +BUCKET = "doczy-dev-infra-textract" +PREFIX = "batches/batch_1/contract-text-file/" # replace with s3 path + +# Bedrock Settings +BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime", + region_name="us-east-1", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN +) +MODEL_ID_CLAUDE3 = 'anthropic.claude-3-sonnet-20240229-v1:0' +MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1' + diff --git a/src/prompts.py b/src/prompts.py new file mode 100644 index 0000000..b02a25b --- /dev/null +++ b/src/prompts.py @@ -0,0 +1,40 @@ + +def BOTTOM_UP_PRIMARY(page, payer): + return f""" +### PAGE START ### {page} ### PAGE END + +The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. + +Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more. + +If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. + +Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. +It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list. + +Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: + +'IS_CARVEOUT' : A Primary Reimbursement is a generalized rate for which a specialty or service is reimbursed. A Carveout reimbursement is a specific rate that is different or more detailed than the primary service. If the rate is a carveout, write 'Y'. If not, write, 'N'. +'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional. +'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. +'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. +'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement. +'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. +'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program. +'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'. +'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'. +'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. +'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based. +'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable. +'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. +'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met. +'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar. +'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. +'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. + +When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of. +As an example, if you see that a reimbursement says something like "specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step. + +Only return the list, with no other commentary or explanation. +""" + diff --git a/src/utils.py b/src/utils.py new file mode 100644 index 0000000..a8ff78b --- /dev/null +++ b/src/utils.py @@ -0,0 +1,84 @@ +import os +import pandas as pd +import shutil +import boto3 + +import config + +def read_local(file_path): + # Check if the file is a text file + if os.path.isfile(file_path) and file_path.endswith('.txt'): + try: + # First attempt to open the file with UTF-8 encoding + with open(file_path, 'r', encoding='utf-8') as file: + file_contents = file.read() + return file_contents + except UnicodeDecodeError: + # If UTF-8 fails, try reading the file with ANSI encoding + try: + with open(file_path, 'r', encoding='cp1252') as file: + file_contents = file.read() + return file_contents + except UnicodeDecodeError: + # If ANSI also fails, log an error message or handle it accordingly + print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.") + +def read_s3(): + + s3_client = boto3.client('s3', + region_name="us-east-2", + aws_access_key_id=config.AWS_ACCESS_KEY_ID, + aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY, + aws_session_token=config.AWS_SESSION_TOKEN + ) + objects = s3_client.list_objects_v2(Bucket=config.BUCKET, Prefix=config.PREFIX) + file_list = [] + for obj in objects['Contents']: + if not obj['Key'].endswith('/'): + file_list.append(obj['Key']) + contract_list = sorted(file_list) + files = {} + for contract in contract_list: + data = s3_client.get_object(Bucket=config.BUCKET, Key=contract) + contents = data['Body'].read() + context = contents.decode('utf-8') + path, filename = os.path.split(contract) + files[filename] = context + return files + +def read_input(mode=config.READ_MODE): + if mode == '_LOCAL_': + files = {} + for file in os.listdir(config.LOCAL_PATH): + full_path = os.path.join(config.LOCAL_PATH, file) + file_text = read_local(full_path) + files[file] = file_text + return files + elif mode == '_S3_': + return read_s3() + +def consolidate_individual(): + dfs = [] + for filename in os.listdir('temp'): + if filename.endswith('.csv'): + filepath = os.path.join('temp', filename) + dfs.append(pd.read_csv(filepath)) + + # Remove temp folder + if os.path.exists('temp'): + shutil.rmtree('temp') + + consolidated_df = pd.concat(dfs, ignore_index=True) + + # Write output + version = 1 + existing_files = [filename for filename in os.listdir('results') if filename.startswith(f'consolidated_results_{config.TODAY}')] + if existing_files: + versions = [int(file.split('_v')[1].split('.')[0]) for file in existing_files if '_v' in file] + if versions: + version = max(versions) + 1 + filename = f'consolidated_results_{config.TODAY}_v{version}.csv' + consolidated_df.to_csv(os.path.join('results', filename)) + + + From a5d38ef0094f65a8b40b1fd2fca72713ddbcfa6c Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 23 Apr 2024 15:02:16 -0700 Subject: [PATCH 04/78] consolidation updates --- src/bottom_up_funcs.py | 31 +------------------------------ src/bottom_up_main.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 31 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 4017734..681f8a1 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -3,7 +3,7 @@ import os import shutil import pandas as pd from collections import defaultdict -import concurrent.futures + import config import utils @@ -51,17 +51,7 @@ def run_bottom_up_primary(text_dict, tokens): answer_dict[page_number] = answer return answer_dict -def run_lesser_of_prompt(text_dict, result_dicts): - - valid_dicts = [dict for dict in result_dicts if dict['LESSER_OF_LANGUAGE_IND'] == 'Y'] - pages_to_check = list(set([dict['page_num'] for dict in valid_dicts])) - for page_num in pages_to_check: - page_text = text_dict[page_num] - dicts_on_page = [dict for dict in valid_dicts if dict['page_num'] == page_num] - print(dicts_on_page) - def append_secondary(result_dicts): - # Helper function to create a key from dict excluding certain keys def create_key(d): return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT')) @@ -112,22 +102,3 @@ def bottom_up_primary(file_object): if not os.path.exists('temp'): os.makedirs('temp') answer_df.to_csv(f"temp/{filename}_results.csv") - - -def run_bottom_up(input_dict): - - if not os.path.exists('results'): - os.makedirs('results') - - # Run prompts with multithreading - with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: - executor.map(bottom_up_primary, input_dict.items()) - - # Output - if config.OUTPUT_MODE == '_CONSOLIDATED_': - consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file - utils.write_consolidated(consolidated_df) - consolidated_df.to_csv(os.path.join('results', f'consolidated_results_{config.TODAY}.csv')) # Write output - - - diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py index 00fa6de..de05c38 100644 --- a/src/bottom_up_main.py +++ b/src/bottom_up_main.py @@ -1,4 +1,6 @@ # Imports +import os +import concurrent.futures import utils import config @@ -6,9 +8,20 @@ import bottom_up_funcs def main(): + # Read contract txt input_dict = utils.read_input() + + # Set up results folder + if not os.path.exists('results'): + os.makedirs('results') + + # Run bottom up primary with multithreading + with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + executor.map(bottom_up_funcs.bottom_up_primary, input_dict.items()) - bottom_up_funcs.run_bottom_up(input_dict) + # Write Output + if config.OUTPUT_MODE == '_CONSOLIDATED_': + consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file, then write output if __name__ == "__main__": From 1e63399648753ee64225ea6f010b1e9c1c99fa34 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 29 Apr 2024 22:47:01 -0700 Subject: [PATCH 05/78] New prompt structure --- src/bottom_up_funcs.py | 167 ++++++++++++++++++++--------------------- src/bottom_up_main.py | 34 +++++---- src/claude_funcs.py | 7 +- src/config.py | 14 ++-- src/dict_operations.py | 47 ++++++++++++ src/preprocess.py | 46 ++++++++++++ src/prompts.py | 149 ++++++++++++++++++++++++++++++------ src/table_funcs.py | 23 ++++++ src/utils.py | 8 +- 9 files changed, 356 insertions(+), 139 deletions(-) create mode 100644 src/dict_operations.py create mode 100644 src/preprocess.py create mode 100644 src/table_funcs.py diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 681f8a1..66caadd 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -1,104 +1,99 @@ -import re -import os -import shutil -import pandas as pd -from collections import defaultdict +import os +import pandas as pd import config -import utils import prompts import claude_funcs +import preprocess +import dict_operations -def clean_billed_charges(text): - def replace(match): - start_pos = match.start() - pre_text = text[max(0, start_pos-15):start_pos] - if '%' in pre_text: - return match.group() - else: - return '100% of billed charges' - return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) - -def split_text(text): - text_list = text.split('Start of Page No. = ') - text_dict = {page.split()[0] : page for page in text_list} - return text_dict - -def string_to_dict(string_dict): - # Regular expression to capture key-value pairs - regex_pattern = r'\{\s*([^}]*)\s*\}' - key_value_pattern = r'"([^"]+)":\s*"([^"]+)"' - data = [] - for page in string_dict.keys(): - dict_string = string_dict[page] - groups = re.findall(regex_pattern, dict_string) - for group in groups: - key_values = re.findall(key_value_pattern, group) - key_value_dict = {key: value for key, value in key_values} - key_value_dict['page_num'] = page - data.append(key_value_dict) - return data def run_bottom_up_primary(text_dict, tokens): + #chunk_dict = preprocess.chunk_text(text_dict) + answer_dict = {} + + #for page_number in chunk_dict.keys(): + #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: + #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): - if page_number.isdigit(): - if True: #int(page_number) in (22, 23): - prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) - #print(len(prompt)) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - answer_dict[page_number] = answer + if page_number == '24':#'%' in text_dict[page_number] or '$' in text_dict[page_number]: + prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) + + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + answer_dict[page_number] = answer return answer_dict -def append_secondary(result_dicts): - # Helper function to create a key from dict excluding certain keys - def create_key(d): - return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT')) - - # Group dictionaries by non-REIMBURSEMENT fields - grouped = defaultdict(list) - for d in result_dicts: - grouped[create_key(d)].append(d) - - # Process groups to create combined entries - combined_results = [] - for key, group in grouped.items(): - # Create a new dictionary based on non-REIMBURSEMENT fields - new_dict = {k: v for k, v in key} - # Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary - for i, entry in enumerate(group[:3]): # Process only the first three entries - for k, v in entry.items(): - if k.startswith('REIMBURSEMENT'): - new_key = f'{["PRIMARY", "SECONDARY", "TERTIARY"][i]}_{k}' - new_dict[new_key] = v - combined_results.append(new_dict) - return combined_results - - -def bottom_up_primary(file_object): - - filename, contract_text = file_object - - contract_text = clean_billed_charges(contract_text) - - text_dict = split_text(contract_text) - - answer_dicts = run_bottom_up_primary(text_dict, 8000) - - answer_dicts = string_to_dict(answer_dicts) - - answer_dicts = append_secondary(answer_dicts) +def run_bottom_up_secondary(answer_dicts, text_dict, tokens): + temp_dicts = [] + + # New rows for lesser of for dict in answer_dicts: + temp_dicts.append(dict) + + # Bottom Up Lesser + prompt = prompts.BOTTOM_UP_LESSER(dict, text_dict[dict['page_num']]) + lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + lesser_of_dict = dict_operations.dict_string_to_dict(lesser_of_answer) + lesser_of_dict['SERVICE'] = dict['SERVICE'] + lesser_of_dict['Filename'] = dict['Filename'] + temp_dicts.append(lesser_of_dict) + + # Add additional fields to each row + final_dicts = [] + for dict in temp_dicts: + # Bottom Up LOB + prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[dict['page_num']]) + lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + lob_dict = dict_operations.dict_string_to_dict(lob_answer) + dict = dict.update(lob_dict) + + # Bottom Up Escalator + + # Bottom Up Grouper + + # Bottom Up FS + + # Bottom Up Exception + + # Bottom Up Codes + + final_dicts.append(dict) + + return final_dicts + + +def bottom_up(file_object): + #### PREPROCESS #### + filename, contract_text = file_object + text_dict = preprocess.split_text(contract_text) + text_dict = preprocess.highlight_rates(text_dict) + + #### PROMPT #### + # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY + answer_strings = run_bottom_up_primary(text_dict, 8000) + answer_dicts = dict_operations.list_string_to_dict(answer_strings) + + # Bottom Up Secondary + results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) + + #### POSTPROCESS #### + #answer_dicts = append_secondary(answer_dicts) + + for dict in results_dicts: dict['Filename'] = filename - answer_df = pd.DataFrame(answer_dicts) + answer_df = pd.DataFrame(results_dicts) # Write output - if config.OUTPUT_MODE == '_INDIVIDUAL_': - answer_df.to_csv(f"results/{filename}_results.csv") - elif config.OUTPUT_MODE == '_CONSOLIDATED_': - if not os.path.exists('temp'): - os.makedirs('temp') - answer_df.to_csv(f"temp/{filename}_results.csv") + if config.WRITE_OUTPUT: + if config.OUTPUT_MODE == '_INDIVIDUAL_': + answer_df.to_csv(f"results/{filename}_results.csv") + elif config.OUTPUT_MODE == '_CONSOLIDATED_': + if not os.path.exists('temp'): + os.makedirs('temp') + answer_df.to_csv(f"temp/{filename}_results.csv") + else: + if config.VERBOSE: print(answer_df) diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py index de05c38..9b803bb 100644 --- a/src/bottom_up_main.py +++ b/src/bottom_up_main.py @@ -5,23 +5,31 @@ import concurrent.futures import utils import config import bottom_up_funcs - +import claude_funcs +import table_funcs def main(): - # Read contract txt - input_dict = utils.read_input() + if config.TEST: + print(claude_funcs.invoke_claude_3("Write 'test', nothing more.", max_tokens = 10)) + else: + # Read contract txt + input_dict = utils.read_input() - # Set up results folder - if not os.path.exists('results'): - os.makedirs('results') + # Set up results folder + if not os.path.exists('results'): + os.makedirs('results') - # Run bottom up primary with multithreading - with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: - executor.map(bottom_up_funcs.bottom_up_primary, input_dict.items()) - - # Write Output - if config.OUTPUT_MODE == '_CONSOLIDATED_': - consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file, then write output + # Extract all tables as dfs + #tables_dict = table_funcs.extract_all_tables(input_dict) + #print(tables_dict) + + # Run bottom up primary with multithreading + with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + executor.map(bottom_up_funcs.bottom_up, input_dict.items()) + + # Write Output + if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': + consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file, then write output if __name__ == "__main__": diff --git a/src/claude_funcs.py b/src/claude_funcs.py index 5dd2cde..db7ad06 100644 --- a/src/claude_funcs.py +++ b/src/claude_funcs.py @@ -3,9 +3,6 @@ import anthropic import config - - - # Claude calls def invoke_claude_2(prompt, max_tokens): body = json.dumps( @@ -30,7 +27,7 @@ def invoke_claude_2(prompt, max_tokens): response_text = response_body['completion'] return response_text -def invoke_claude_3(prompt, max_tokens = 0): +def invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_SONNET, max_tokens = 0): prompt = prompt body = json.dumps({ "anthropic_version": "bedrock-2023-05-31", @@ -52,7 +49,7 @@ def invoke_claude_3(prompt, max_tokens = 0): response = config.BEDROCK_RUNTIME.invoke_model( body=body, - modelId=config.MODEL_ID_CLAUDE3, + modelId=model_id, accept="application/json", contentType="application/json" ) diff --git a/src/config.py b/src/config.py index 8b14dd1..d7c70cb 100644 --- a/src/config.py +++ b/src/config.py @@ -3,7 +3,8 @@ from datetime import datetime import boto3 # General Settings -WRITE_OUTPUT = True +TEST = False # True to run test prompt - just for testing model connection +WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written VERBOSE = True CLIENT_NAME = 'CareSource' TODAY = datetime.now().strftime("%Y%m%d") @@ -12,13 +13,13 @@ TODAY = datetime.now().strftime("%Y%m%d") MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="replace daily" -AWS_SECRET_ACCESS_KEY="replace daily" -AWS_SESSION_TOKEN="replace daily" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXBQFAP56Y" +AWS_SECRET_ACCESS_KEY="joivdLt+8EiERncyYSzpLST9gG95DOjLlED+II3Y" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEMr//////////wEaCXVzLWVhc3QtMiJHMEUCIQC8Mk/Uur4dOGNlq54o0I5Q8nEmsHoLqWCxNZHwkD7R+wIgAx1d3QXoxUROMAgeja1F89WHH1IcgW4eFjwUQE1yt7sqgwMIIxAAGgw2NjAxMzEwNjg3ODIiDPQa/Ly9yjtsuV17TyrgAsTpSFsih3AuFRzYCQzJ/b6F7MCtIdYaOTLXi5L1T2PCAqaDPlBEdypHIjZoO+cl93pDpy/ON4VMS4FCgC1Gpse3dZMKhPh9e60u2XZu6J9uvHDgmJ3HimUKYoMN4C3zkvVQR+fDPkG2ZY/H1vNq49iXyGra6zXf7+qICY99Om/E/cNl2OPAILY4Dr/nC5LmNr19MhO+37oaBjjkIwgJNseiuAoamSCawJXwjxliFjImUA58oUmVP8BD44NOseYVrBGbA3BK03cnK/unC+pWPbBF+mM1ZTHt1JzlAG/VtgE0YKh59/PLVbynHKRWbRJeUz0Gs05ZFdjdJHPnz8g7nAQYDl1vRX482+OQ3kTgFrmcU1s4G1GB2OTADXTtnxyrNw6hdqw4UUUWjE0iS2mVWLZnHCf0Bg+MgKJLXsALpGcdBLGc9Wg2M9ykHc9Ejsh39Ygeq1YS2mOPsZrICFmXyd4wg6PBsQY6pgFsWA5ItfTU/BIUD0IOo0GzijlMsmcv335YS16yW5HIcwmdSKdjHro28eLJ+AI6VZxlq9o0kORA3E9qhk4wdXy/wQ0Gy5KAPePowgtDfC6SMv6vzLmrkNcFcWW0AG+rwcaZFe5iK3ncEHOUA96Irl2J4iAQSubKViXPIxpDz2ONsmafK4DfjRA9PhictplsMNQL3mVhl2HckEG7625835N++T6qD+sK" # I/O Options READ_MODE = '_LOCAL_' # OR '_S3_' -OUTPUT_MODE = '_CONSOLIDATED_' # or '_INDIVIDUAL_' +OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' # File Paths LOCAL_PATH = 'docs/test/' # Replace with local @@ -40,6 +41,7 @@ BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime", aws_secret_access_key=AWS_SECRET_ACCESS_KEY, aws_session_token=AWS_SESSION_TOKEN ) -MODEL_ID_CLAUDE3 = 'anthropic.claude-3-sonnet-20240229-v1:0' +MODEL_ID_CLAUDE3_HAIKU = 'anthropic.claude-3-haiku-20240307-v1:0' +MODEL_ID_CLAUDE3_SONNET = 'anthropic.claude-3-sonnet-20240229-v1:0' MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1' diff --git a/src/dict_operations.py b/src/dict_operations.py new file mode 100644 index 0000000..67a063c --- /dev/null +++ b/src/dict_operations.py @@ -0,0 +1,47 @@ +import re +from collections import defaultdict + +def list_string_to_dict(string_dict): + # Regular expression to capture key-value pairs + regex_pattern = r'\{\s*([^}]*)\s*\}' + key_value_pattern = r'"([^"]+)":\s*"([^"]+)"' + data = [] + for page in string_dict.keys(): + dict_string = string_dict[page] + groups = re.findall(regex_pattern, dict_string) + for group in groups: + key_values = re.findall(key_value_pattern, group) + key_value_dict = {key: value for key, value in key_values} + key_value_dict['page_num'] = page + data.append(key_value_dict) + return data + +def dict_string_to_dict(dict_string): + key_value_pattern = r"'([^']+)':\s*'([^']+)'" + key_values = re.findall(key_value_pattern, dict_string) + key_value_dict = {key: value for key, value in key_values} + return key_value_dict + +def append_secondary(result_dicts): + # Helper function to create a key from dict excluding certain keys + def create_key(d): + return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT')) + + # Group dictionaries by non-REIMBURSEMENT fields + grouped = defaultdict(list) + for d in result_dicts: + grouped[create_key(d)].append(d) + + # Process groups to create combined entries + combined_results = [] + for key, group in grouped.items(): + # Create a new dictionary based on non-REIMBURSEMENT fields + new_dict = {k: v for k, v in key} + # Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary + for i, entry in enumerate(group[:3]): # Process only the first three entries + for k, v in entry.items(): + if k.startswith('REIMBURSEMENT'): + new_key = f'{["PRIMARY", "SECONDARY", "TERTIARY"][i]}_{k}' + new_dict[new_key] = v + combined_results.append(new_dict) + return combined_results diff --git a/src/preprocess.py b/src/preprocess.py new file mode 100644 index 0000000..ed252dc --- /dev/null +++ b/src/preprocess.py @@ -0,0 +1,46 @@ +from itertools import groupby, count + +def split_text(text): + text_list = text.split('Start of Page No. = ') + text_dict = {page.split()[0] : page for page in text_list} + return text_dict + + +def highlight_rates(text_dict): + for page, text in text_dict.items(): + words = text.split() + highlighted_words = [] + for word in words: + if '%' in word or '$' in word: + word = f'>>>{word}<<<' + highlighted_words.append(word) + text_dict[page] = ' '.join(highlighted_words) + return text_dict + +def chunk_text(text_dict): + special_pages = {int(page): text for page, text in text_dict.items() if '%' in text or '$' in text} + page_numbers = sorted(special_pages.keys()) + chunk_page_numbers = [] + for page in page_numbers: + chunk_page_numbers.append(page-1) + chunk_page_numbers.append(page) + chunk_page_numbers.append(page+1) + chunk_page_numbers = list(set(chunk_page_numbers)) + chunk_page_numbers = [page for page in chunk_page_numbers if str(page) in text_dict.keys()] + chunks = [list(group) for key, group in groupby(chunk_page_numbers, lambda x, c=count(): x - next(c))] + chunk_dict = {} + for item in chunks: + dict_key = f'{min(item)}-{max(item)}' + text = "".join([text_dict[str(page)] for page in item]) + chunk_dict[dict_key] = text + return chunk_dict + +def clean_billed_charges(text): + def replace(match): + start_pos = match.start() + pre_text = text[max(0, start_pos-15):start_pos] + if '%' in pre_text: + return match.group() + else: + return '100% of billed charges' + return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) \ No newline at end of file diff --git a/src/prompts.py b/src/prompts.py index b02a25b..7d948e9 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -1,40 +1,145 @@ +# def BOTTOM_UP_PRIMARY(page, payer): +# return f""" +# ### PAGE START ### {page} ### PAGE END + +# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. + +# Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more. + +# If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. + +# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. +# It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list. + +# Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: + +# 'IS_CARVEOUT' : A Primary Reimbursement is a generalized rate for which a specialty or service is reimbursed. A Carveout reimbursement is a specific rate that is different or more detailed than the primary service. If the rate is a carveout, write 'Y'. If not, write, 'N'. +# 'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional. +# 'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. +# 'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. +# 'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement. +# 'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. +# 'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program. +# 'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'. +# 'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'. +# 'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. +# 'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based. +# 'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable. +# 'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. +# 'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met. +# 'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar. +# 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. +# 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. + +# When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of. +# As an example, if you see that a reimbursement says something like "specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step. + +# Only return the list, with no other commentary or explanation. +# """ + + def BOTTOM_UP_PRIMARY(page, payer): return f""" ### PAGE START ### {page} ### PAGE END The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more. +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen. -If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. +If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. -It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list. + +Note that some of the answers may be found in tables indicated by -------Table Start-------- and -------Table End--------. Use knowledge of json format to extract relevant table information. Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: - -'IS_CARVEOUT' : A Primary Reimbursement is a generalized rate for which a specialty or service is reimbursed. A Carveout reimbursement is a specific rate that is different or more detailed than the primary service. If the rate is a carveout, write 'Y'. If not, write, 'N'. -'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional. -'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. -'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. -'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement. -'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. -'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program. -'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'. -'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'. -'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. -'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based. -'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable. -'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. -'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met. -'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar. +'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. - -When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of. -As an example, if you see that a reimbursement says something like "specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step. +'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. Only return the list, with no other commentary or explanation. """ +def BOTTOM_UP_LOB(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text is one page of a contract. The page states that {d['SERVICE']} is reimbursed with the following methodology: {d['FULL_METHODOLOGY']}. + +Your job is to identify attributes related to this reimbursement term and return them in a JSON-formatted dictionary. + +If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. + + Return a dictionary object with the following attributes: +'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all. +'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. This will likely only be listed once per page, if at all. +'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the name of a hospital. This will likely only be listed once per page, if at all. +'PROV_TYPE' : For what provider type or place of service is this fee schedule for? It could be Hospital, Professional, Ancillary, or similar. This will likely only be listed once per page, if at all. + +Only return the dictionary, with no other commentary or explanation. +""" + +def BOTTOM_UP_LESSER(d, page): + return f""" +### PAGE START ### {page} ### PAGE END + +The above text is one page of a contract. + +Your job is to identify attributes related to this reimbursement term and return them in a JSON-formatted dictionary. Specifically, the attributes are related to the presence of 'Lesser of' or 'Greater of' language in the text that applies to the specific reimbursement term. + +Specifically, we are interested only in: +Service: '{d['SERVICE']}' +Methodology: {d['FULL_METHODOLOGY']}. + +For this term, return a dictionary object with the following attributes: +'LESSER_OF_LANGUAGE_IND' : Is the Service paid as the lesser of the above Methodology and some other methodology? If so, return 'Y'. If not, return 'N' +'GREATER_OF_LANGUAGE_IND' : Is the Service paid as the greater of the above Methodology and some other methodology? If so, return 'Y'. If not, return 'N' +'FULL_METHODOOGY' : If the answer for either of the above is 'Y', what is the other Methodology in the lesser of/greater of condition? If the methodology is a percent of something, what is it the percent of? If the methodology is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. If both lesser of and greater of are N, leave this field N/A. + +Only return the dictionary, with no other commentary or explanation. +""" + +def BOTTOM_UP_INDICATORS(d, page): + return f""" + +""" + +def BOTTOM_UP_FS(d): + return f""" + +""" + + + +def BOTTOM_UP_CODES(codes, page): + return f"""CODE_LIST: {codes} + +The above strings were identified as some type of code in a health insurance contract. Your job is to assign the codes to their code type. Return a JSON dictionary with the following attributes: + +'REIMBURSEMENT_ADMITTYPE_CODES' : +'REIMBURSEMENT_DIAG_CODES' : +'REIMBURSEMENT_GROUPER_CODES' : +'REIMBURSEMENT_PLACEOFSERVICE_CODES' : +'REIMBURSEMENT_PROC_CODES' : +'REIMBURSEMENT_REVENUE_CODES' : +'REIMBURSEMENT_STATUS_INDICATOR_CODES' : + +Assign each code in the list to its proper code type based on the descriptions above. The codes seen will most often be just one type of code. + +For the code types that are not in the list, leave the field N/A. + +Return only the JSON dictionary, with no other commentary or explanation. +""" + + + + + + + + + + + + diff --git a/src/table_funcs.py b/src/table_funcs.py new file mode 100644 index 0000000..3598852 --- /dev/null +++ b/src/table_funcs.py @@ -0,0 +1,23 @@ + +import re +import pandas as pd +import ast + +import bottom_up_funcs + + +def extract_tables(text): + pattern = re.compile(r"-------Table Start--------(.*?)-------Table End--------", re.DOTALL) + tables = pattern.findall(text) + table_list = [table.strip() for table in tables] + table_list = ['{'+table.split('{')[1] for table in table_list] + df_list = [pd.DataFrame(ast.literal_eval(table)) for table in table_list] + for df in df_list: + print(df.columns) + return df_list + +def extract_all_tables(input_dict): + tables_dict = {} + for filename, filetext in input_dict.items(): + tables_dict[filename] = extract_tables(filetext) + return tables_dict \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index a8ff78b..5845493 100644 --- a/src/utils.py +++ b/src/utils.py @@ -24,13 +24,7 @@ def read_local(file_path): print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.") def read_s3(): - - s3_client = boto3.client('s3', - region_name="us-east-2", - aws_access_key_id=config.AWS_ACCESS_KEY_ID, - aws_secret_access_key=config.AWS_SECRET_ACCESS_KEY, - aws_session_token=config.AWS_SESSION_TOKEN - ) + s3_client = config.S3_CLIENT objects = s3_client.list_objects_v2(Bucket=config.BUCKET, Prefix=config.PREFIX) file_list = [] for obj in objects['Contents']: From 18ca91ac288b50d2b9e19eb7d8104d163504749e Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 29 Apr 2024 22:51:42 -0700 Subject: [PATCH 06/78] Update config --- src/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config.py b/src/config.py index d7c70cb..860bcef 100644 --- a/src/config.py +++ b/src/config.py @@ -13,9 +13,9 @@ TODAY = datetime.now().strftime("%Y%m%d") MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXBQFAP56Y" -AWS_SECRET_ACCESS_KEY="joivdLt+8EiERncyYSzpLST9gG95DOjLlED+II3Y" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEMr//////////wEaCXVzLWVhc3QtMiJHMEUCIQC8Mk/Uur4dOGNlq54o0I5Q8nEmsHoLqWCxNZHwkD7R+wIgAx1d3QXoxUROMAgeja1F89WHH1IcgW4eFjwUQE1yt7sqgwMIIxAAGgw2NjAxMzEwNjg3ODIiDPQa/Ly9yjtsuV17TyrgAsTpSFsih3AuFRzYCQzJ/b6F7MCtIdYaOTLXi5L1T2PCAqaDPlBEdypHIjZoO+cl93pDpy/ON4VMS4FCgC1Gpse3dZMKhPh9e60u2XZu6J9uvHDgmJ3HimUKYoMN4C3zkvVQR+fDPkG2ZY/H1vNq49iXyGra6zXf7+qICY99Om/E/cNl2OPAILY4Dr/nC5LmNr19MhO+37oaBjjkIwgJNseiuAoamSCawJXwjxliFjImUA58oUmVP8BD44NOseYVrBGbA3BK03cnK/unC+pWPbBF+mM1ZTHt1JzlAG/VtgE0YKh59/PLVbynHKRWbRJeUz0Gs05ZFdjdJHPnz8g7nAQYDl1vRX482+OQ3kTgFrmcU1s4G1GB2OTADXTtnxyrNw6hdqw4UUUWjE0iS2mVWLZnHCf0Bg+MgKJLXsALpGcdBLGc9Wg2M9ykHc9Ejsh39Ygeq1YS2mOPsZrICFmXyd4wg6PBsQY6pgFsWA5ItfTU/BIUD0IOo0GzijlMsmcv335YS16yW5HIcwmdSKdjHro28eLJ+AI6VZxlq9o0kORA3E9qhk4wdXy/wQ0Gy5KAPePowgtDfC6SMv6vzLmrkNcFcWW0AG+rwcaZFe5iK3ncEHOUA96Irl2J4iAQSubKViXPIxpDz2ONsmafK4DfjRA9PhictplsMNQL3mVhl2HckEG7625835N++T6qD+sK" +AWS_ACCESS_KEY_ID="update here" +AWS_SECRET_ACCESS_KEY="update here" +AWS_SESSION_TOKEN="update here" # I/O Options READ_MODE = '_LOCAL_' # OR '_S3_' From 0d8cbf42696d8411526517e1972fb7694b82726d Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 30 Apr 2024 06:53:44 -0700 Subject: [PATCH 07/78] Moved LOB prompt to just primary --- src/bottom_up_funcs.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 66caadd..408d606 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -18,7 +18,7 @@ def run_bottom_up_primary(text_dict, tokens): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): - if page_number == '24':#'%' in text_dict[page_number] or '$' in text_dict[page_number]: + if '%' in text_dict[page_number] or '$' in text_dict[page_number]: prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) @@ -33,6 +33,12 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): for dict in answer_dicts: temp_dicts.append(dict) + # Bottom Up LOB + # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[dict['page_num']]) + # lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # lob_dict = dict_operations.dict_string_to_dict(lob_answer) + # dict = dict.update(lob_dict) + # Bottom Up Lesser prompt = prompts.BOTTOM_UP_LESSER(dict, text_dict[dict['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) @@ -44,11 +50,7 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # Add additional fields to each row final_dicts = [] for dict in temp_dicts: - # Bottom Up LOB - prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[dict['page_num']]) - lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - lob_dict = dict_operations.dict_string_to_dict(lob_answer) - dict = dict.update(lob_dict) + # Bottom Up Methodology # Bottom Up Escalator From e2a6ee5a4a3d56217b2049d1e6f2340303f719b0 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 30 Apr 2024 18:52:39 -0700 Subject: [PATCH 08/78] Added lesser of prompt --- src/bottom_up_funcs.py | 69 +++++++++++++++++++++++++++++++----------- src/bottom_up_main.py | 2 +- src/config.py | 10 +++--- src/preprocess.py | 1 + src/prompts.py | 26 ++++++++++------ src/utils.py | 10 +++--- 6 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 408d606..8616312 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -9,19 +9,35 @@ import preprocess import dict_operations +def run_bottom_up_lesser(d_, text_dict, tokens=8000): + d = d_.copy() + prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) + lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + print(lesser_of_answer) + lesser_of_dict = dict_operations.dict_string_to_dict(lesser_of_answer) + if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': + lesser_of_dict['SERVICE'] = d['SERVICE'] + return (lesser_of_dict, d) + else: + d['LESSER_OF_LANGUAGE_IND'] == 'N' + d['GREATER_OF_LANGUAGE_IND'] == 'N' + return ({}, d) + + + def run_bottom_up_primary(text_dict, tokens): #chunk_dict = preprocess.chunk_text(text_dict) + if config.VERBOSE: print("Running Bottom Up Primary") answer_dict = {} - #for page_number in chunk_dict.keys(): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): if '%' in text_dict[page_number] or '$' in text_dict[page_number]: prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + print(answer) answer_dict[page_number] = answer return answer_dict @@ -30,8 +46,8 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): temp_dicts = [] # New rows for lesser of - for dict in answer_dicts: - temp_dicts.append(dict) + for d in answer_dicts: + # Bottom Up LOB # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[dict['page_num']]) @@ -40,29 +56,45 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # dict = dict.update(lob_dict) # Bottom Up Lesser - prompt = prompts.BOTTOM_UP_LESSER(dict, text_dict[dict['page_num']]) - lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - lesser_of_dict = dict_operations.dict_string_to_dict(lesser_of_answer) - lesser_of_dict['SERVICE'] = dict['SERVICE'] - lesser_of_dict['Filename'] = dict['Filename'] - temp_dicts.append(lesser_of_dict) - + if config.VERBOSE: print("Running Bottom Up Lesser") + lesser_object = run_bottom_up_lesser(d, text_dict, tokens) + temp_dicts.append(lesser_object[1]) + if lesser_object[0]: temp_dicts.append(lesser_object[0]) + + print(len(temp_dicts)) # Add additional fields to each row final_dicts = [] - for dict in temp_dicts: - # Bottom Up Methodology + for d in temp_dicts: + # # Bottom Up Methodology + # prompt = prompts.BOTTOM_UP_METHODOLOGY(dict) + # methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # dict['REIMBURSEMENT_METHODOLOGY'] = methodology_answer # Bottom Up Escalator + # Skip for now # Bottom Up Grouper + # Skip for now - # Bottom Up FS + # # Bottom Up FS + # prompt = prompts.BOTTOM_UP_FS(dict) + # fs_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # fs_dict = dict_operations.dict_string_to_dict(fs_answer) + # dict = dict.update(fs_dict) - # Bottom Up Exception + # # Bottom Up Exception + # prompt = prompts.BOTTOM_UP_EXCEPTION(dict, text_dict[dict['page_num']]) + # exc_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # exc_dict = dict_operations.dict_string_to_dict(exc_answer) + # dict = dict.update(exc_dict) - # Bottom Up Codes + # # Bottom Up Codes + # prompt = prompts.BOTTOM_UP_CODES(dict, text_dict[dict['page_num']]) + # codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # codes_dict = dict_operations.dict_string_to_dict(codes_answer) + # dict = dict.update(codes_dict) - final_dicts.append(dict) + final_dicts.append(d) return final_dicts @@ -70,6 +102,7 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): def bottom_up(file_object): #### PREPROCESS #### filename, contract_text = file_object + if config.VERBOSE: print(f"Processing {filename}...") text_dict = preprocess.split_text(contract_text) text_dict = preprocess.highlight_rates(text_dict) @@ -77,9 +110,11 @@ def bottom_up(file_object): # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) answer_dicts = dict_operations.list_string_to_dict(answer_strings) + print("Answer dicts ", answer_dicts) # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) + print("Results dicts ", results_dicts) #### POSTPROCESS #### #answer_dicts = append_secondary(answer_dicts) diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py index 9b803bb..509e7ca 100644 --- a/src/bottom_up_main.py +++ b/src/bottom_up_main.py @@ -29,7 +29,7 @@ def main(): # Write Output if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': - consolidated_df = utils.consolidate_individual() # Consolidate temp results to one file, then write output + consolidated_df = utils.consolidate_individual(folder='temp') # Consolidate temp results to one file, then write output if __name__ == "__main__": diff --git a/src/config.py b/src/config.py index 860bcef..58fea7b 100644 --- a/src/config.py +++ b/src/config.py @@ -4,18 +4,18 @@ import boto3 # General Settings TEST = False # True to run test prompt - just for testing model connection -WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written +WRITE_OUTPUT = False # True writes csvs, False prints result in console but no output written VERBOSE = True -CLIENT_NAME = 'CareSource' +CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") # Multithread Settings MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="update here" -AWS_SECRET_ACCESS_KEY="update here" -AWS_SESSION_TOKEN="update here" +AWS_ACCESS_KEY_ID="update" +AWS_SECRET_ACCESS_KEY="update" +AWS_SESSION_TOKEN="update" # I/O Options READ_MODE = '_LOCAL_' # OR '_S3_' diff --git a/src/preprocess.py b/src/preprocess.py index ed252dc..47d30aa 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -1,4 +1,5 @@ from itertools import groupby, count +import re def split_text(text): text_list = text.split('Start of Page No. = ') diff --git a/src/prompts.py b/src/prompts.py index 7d948e9..f8391b4 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -84,23 +84,25 @@ def BOTTOM_UP_LESSER(d, page): return f""" ### PAGE START ### {page} ### PAGE END -The above text is one page of a contract. +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} -Your job is to identify attributes related to this reimbursement term and return them in a JSON-formatted dictionary. Specifically, the attributes are related to the presence of 'Lesser of' or 'Greater of' language in the text that applies to the specific reimbursement term. +Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above. -Specifically, we are interested only in: -Service: '{d['SERVICE']}' -Methodology: {d['FULL_METHODOLOGY']}. +Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply. -For this term, return a dictionary object with the following attributes: -'LESSER_OF_LANGUAGE_IND' : Is the Service paid as the lesser of the above Methodology and some other methodology? If so, return 'Y'. If not, return 'N' -'GREATER_OF_LANGUAGE_IND' : Is the Service paid as the greater of the above Methodology and some other methodology? If so, return 'Y'. If not, return 'N' -'FULL_METHODOOGY' : If the answer for either of the above is 'Y', what is the other Methodology in the lesser of/greater of condition? If the methodology is a percent of something, what is it the percent of? If the methodology is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. If both lesser of and greater of are N, leave this field N/A. +Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): +'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. +'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. +'REIMBURSEMENT_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. Do not repeat the Methodology from above. +'REIMBURSEMENT_RATE' : If the REIMBURSEMENT_METHODOLOGY found above is a percent of something, write just the numeric percent value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. +'REIMBURSEMENT_FLAT_FEE' : If the REIMBURSEMENT_METHODOLOGY found above is a dollar value, write just the numeric value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. Only return the dictionary, with no other commentary or explanation. """ -def BOTTOM_UP_INDICATORS(d, page): +def BOTTOM_UP_EXCEPTION(d, page): return f""" """ @@ -110,6 +112,10 @@ def BOTTOM_UP_FS(d): """ +def BOTTOM_UP_METHODOLOGY(d): + return f""" + +""" def BOTTOM_UP_CODES(codes, page): diff --git a/src/utils.py b/src/utils.py index 5845493..754a04b 100644 --- a/src/utils.py +++ b/src/utils.py @@ -51,16 +51,16 @@ def read_input(mode=config.READ_MODE): elif mode == '_S3_': return read_s3() -def consolidate_individual(): +def consolidate_individual(folder): dfs = [] - for filename in os.listdir('temp'): + for filename in os.listdir(folder): if filename.endswith('.csv'): - filepath = os.path.join('temp', filename) + filepath = os.path.join(folder, filename) dfs.append(pd.read_csv(filepath)) # Remove temp folder - if os.path.exists('temp'): - shutil.rmtree('temp') + if folder=='temp' and os.path.exists(folder): + shutil.rmtree(folder) consolidated_df = pd.concat(dfs, ignore_index=True) From 98844625a1455c43ddf0a8e284932312c1384cf3 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 1 May 2024 17:57:50 -0700 Subject: [PATCH 09/78] Added all secondary prompts --- src/bottom_up_funcs.py | 89 ++++++++++++++++++++---------------------- src/bottom_up_main.py | 8 ++-- src/config.py | 19 ++++----- src/postprocess.py | 13 ++++++ src/prompts.py | 59 ++++++++++++++++++++-------- src/table_funcs.py | 3 -- src/utils.py | 1 - 7 files changed, 111 insertions(+), 81 deletions(-) create mode 100644 src/postprocess.py diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 8616312..f8f0367 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -7,20 +7,21 @@ import prompts import claude_funcs import preprocess import dict_operations +import postprocess -def run_bottom_up_lesser(d_, text_dict, tokens=8000): - d = d_.copy() +def run_bottom_up_lesser(d, text_dict, tokens=8000): + print(d) prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - print(lesser_of_answer) lesser_of_dict = dict_operations.dict_string_to_dict(lesser_of_answer) + lesser_of_dict['page_num'] = d['page_num'] if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': lesser_of_dict['SERVICE'] = d['SERVICE'] return (lesser_of_dict, d) else: - d['LESSER_OF_LANGUAGE_IND'] == 'N' - d['GREATER_OF_LANGUAGE_IND'] == 'N' + d['LESSER_OF_LANGUAGE_IND'] = 'N' + d['GREATER_OF_LANGUAGE_IND'] = 'N' return ({}, d) @@ -34,67 +35,60 @@ def run_bottom_up_primary(text_dict, tokens): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): - if '%' in text_dict[page_number] or '$' in text_dict[page_number]: + if page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number]): prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - print(answer) answer_dict[page_number] = answer return answer_dict def run_bottom_up_secondary(answer_dicts, text_dict, tokens): - temp_dicts = [] - - # New rows for lesser of - for d in answer_dicts: - + if config.VERBOSE: print("Running Bottom Up Secondary") + # New rows for lesser of + temp_dicts = [] + for d in answer_dicts: + print(d) # Bottom Up LOB - # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[dict['page_num']]) + # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[d['page_num']]) # lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) # lob_dict = dict_operations.dict_string_to_dict(lob_answer) - # dict = dict.update(lob_dict) + # d = d.update(lob_dict) # Bottom Up Lesser - if config.VERBOSE: print("Running Bottom Up Lesser") - lesser_object = run_bottom_up_lesser(d, text_dict, tokens) - temp_dicts.append(lesser_object[1]) - if lesser_object[0]: temp_dicts.append(lesser_object[0]) + lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens) + temp_dicts.append(lesser_object[1]) # Add original object from d + if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object - print(len(temp_dicts)) # Add additional fields to each row final_dicts = [] for d in temp_dicts: - # # Bottom Up Methodology - # prompt = prompts.BOTTOM_UP_METHODOLOGY(dict) - # methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # dict['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + print(d) + if d: + # # Bottom Up Methodology + # prompt = prompts.BOTTOM_UP_METHODOLOGY(dict) + # methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + # dict['REIMBURSEMENT_METHODOLOGY'] = methodology_answer - # Bottom Up Escalator - # Skip for now + # # Bottom Up FS + prompt = prompts.BOTTOM_UP_FS(d) + fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) + fs_dict = dict_operations.dict_string_to_dict(fs_answer) + d = d.update(fs_dict) - # Bottom Up Grouper - # Skip for now + # # Bottom Up Exception/Escalator + prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[d['page_num']]) + exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) + exc_dict = dict_operations.dict_string_to_dict(exc_answer) + d = d.update(exc_dict) - # # Bottom Up FS - # prompt = prompts.BOTTOM_UP_FS(dict) - # fs_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # fs_dict = dict_operations.dict_string_to_dict(fs_answer) - # dict = dict.update(fs_dict) + # # Bottom Up Codes + prompt = prompts.BOTTOM_UP_CODES(d, text_dict[d['page_num']]) + codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + codes_dict = dict_operations.dict_string_to_dict(codes_answer) + d = d.update(codes_dict) - # # Bottom Up Exception - # prompt = prompts.BOTTOM_UP_EXCEPTION(dict, text_dict[dict['page_num']]) - # exc_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # exc_dict = dict_operations.dict_string_to_dict(exc_answer) - # dict = dict.update(exc_dict) - - # # Bottom Up Codes - # prompt = prompts.BOTTOM_UP_CODES(dict, text_dict[dict['page_num']]) - # codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # codes_dict = dict_operations.dict_string_to_dict(codes_answer) - # dict = dict.update(codes_dict) - - final_dicts.append(d) + final_dicts.append(d) return final_dicts @@ -103,6 +97,7 @@ def bottom_up(file_object): #### PREPROCESS #### filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") + text_dict = preprocess.split_text(contract_text) text_dict = preprocess.highlight_rates(text_dict) @@ -110,11 +105,11 @@ def bottom_up(file_object): # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) answer_dicts = dict_operations.list_string_to_dict(answer_strings) - print("Answer dicts ", answer_dicts) + answer_dicts = postprocess.primary_filter(answer_dicts) + print(len(answer_dicts)) # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) - print("Results dicts ", results_dicts) #### POSTPROCESS #### #answer_dicts = append_secondary(answer_dicts) diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py index 509e7ca..b464587 100644 --- a/src/bottom_up_main.py +++ b/src/bottom_up_main.py @@ -6,7 +6,6 @@ import utils import config import bottom_up_funcs import claude_funcs -import table_funcs def main(): if config.TEST: @@ -24,9 +23,10 @@ def main(): #print(tables_dict) # Run bottom up primary with multithreading - with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: - executor.map(bottom_up_funcs.bottom_up, input_dict.items()) - + # with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + # executor.map(bottom_up_funcs.bottom_up, input_dict.items()) + bottom_up_funcs.bottom_up(list(input_dict.items())[0]) + # Write Output if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': consolidated_df = utils.consolidate_individual(folder='temp') # Consolidate temp results to one file, then write output diff --git a/src/config.py b/src/config.py index 58fea7b..2bb36d3 100644 --- a/src/config.py +++ b/src/config.py @@ -1,25 +1,26 @@ -import os + from datetime import datetime import boto3 # General Settings TEST = False # True to run test prompt - just for testing model connection -WRITE_OUTPUT = False # True writes csvs, False prints result in console but no output written + VERBOSE = True CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") +# I/O Options +WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written +READ_MODE = '_LOCAL_' # OR '_S3_' +OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' + # Multithread Settings MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="update" -AWS_SECRET_ACCESS_KEY="update" -AWS_SESSION_TOKEN="update" - -# I/O Options -READ_MODE = '_LOCAL_' # OR '_S3_' -OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' +AWS_ACCESS_KEY_ID="update here" +AWS_SECRET_ACCESS_KEY="update here" +AWS_SESSION_TOKEN="update here" # File Paths LOCAL_PATH = 'docs/test/' # Replace with local diff --git a/src/postprocess.py b/src/postprocess.py new file mode 100644 index 0000000..a58109b --- /dev/null +++ b/src/postprocess.py @@ -0,0 +1,13 @@ +import config + +# Filter applied between primary and secondary +def primary_filter(answer_dict): + if config.VERBOSE: print("Applying filter") + filtered_dicts = [] + for d in answer_dict: + if 'SERVICE' in d.keys(): + if 'INSURANCE' not in d['SERVICE'].upper(): + filtered_dicts.append(d) + else: + filtered_dicts.append(d) + return filtered_dicts diff --git a/src/prompts.py b/src/prompts.py index f8391b4..08bce6a 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -95,21 +95,42 @@ Note that a 'lesser of'/'greater of' statement may not be directly written for a Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): 'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. 'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. -'REIMBURSEMENT_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. Do not repeat the Methodology from above. +'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. Do not repeat the Methodology from above. 'REIMBURSEMENT_RATE' : If the REIMBURSEMENT_METHODOLOGY found above is a percent of something, write just the numeric percent value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. 'REIMBURSEMENT_FLAT_FEE' : If the REIMBURSEMENT_METHODOLOGY found above is a dollar value, write just the numeric value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. Only return the dictionary, with no other commentary or explanation. """ -def BOTTOM_UP_EXCEPTION(d, page): - return f""" +def BOTTOM_UP_EXCEPT_ESC(d, page): + return f"""### PAGE START ### {page} ### PAGE END +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Your job is to identify if the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations) or an escalation + +Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): +REIMBRUSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N' +REIMBURSEMENT_DESCRIBE_EXCEPTION: If the listed reimbursement contains some sort of exception, describe it here. Try to use exact words from the text only. If there is none, just write 'N/A' +RATE_ESCALATOR_IND: If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. + +Only return the dictionary, with no other commentary or explanation. """ def BOTTOM_UP_FS(d): - return f""" + return f"""Analyze the reimbursement terms listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} +Your job is to identify if the reimbursement is based on a Fee Schedule. + +Read and analyze the Methodology, then populate a JSONdictionary with the following fields (descriptions provided): +REIMBURSEMENT_FEE_SCHEDULE : If the Methodology is based on a Fee Schedule, what is the name of the fee schedule on which it is based? It could be something like Medicaid, Medicare, CMS, DRG, or similar. Do not answer with a complete sentence. If the Methodology is not based on a Fee Schedule, just return 'N/A' +REIMBURSEMENT_FEE_SCHEDULE_VERSION : If the Methodology is based on a Fee Schedule, what specific version of the Fee Schedule is it based on. It could be something like 'Current', 'Prevailing', 'at the date of service', a specific year, or similar. If there is no answer, return 'N/A' + +Only return the dictionary, with no other commentary or explanation. """ def BOTTOM_UP_METHODOLOGY(d): @@ -118,24 +139,28 @@ def BOTTOM_UP_METHODOLOGY(d): """ -def BOTTOM_UP_CODES(codes, page): - return f"""CODE_LIST: {codes} +def BOTTOM_UP_CODES(d, page): + return f"""### PAGE START ### {page} ### PAGE END -The above strings were identified as some type of code in a health insurance contract. Your job is to assign the codes to their code type. Return a JSON dictionary with the following attributes: +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} -'REIMBURSEMENT_ADMITTYPE_CODES' : -'REIMBURSEMENT_DIAG_CODES' : -'REIMBURSEMENT_GROUPER_CODES' : -'REIMBURSEMENT_PLACEOFSERVICE_CODES' : -'REIMBURSEMENT_PROC_CODES' : -'REIMBURSEMENT_REVENUE_CODES' : -'REIMBURSEMENT_STATUS_INDICATOR_CODES' : +Your job is to identify codes associated with this specific term. -Assign each code in the list to its proper code type based on the descriptions above. The codes seen will most often be just one type of code. +Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided): +'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility. +'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes. +'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar. +'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options. +'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service. +'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures. +'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes. +'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status. -For the code types that are not in the list, leave the field N/A. +Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. -Return only the JSON dictionary, with no other commentary or explanation. +Only return the dictionary, with no other commentary or explanation. """ diff --git a/src/table_funcs.py b/src/table_funcs.py index 3598852..996fa6f 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -3,9 +3,6 @@ import re import pandas as pd import ast -import bottom_up_funcs - - def extract_tables(text): pattern = re.compile(r"-------Table Start--------(.*?)-------Table End--------", re.DOTALL) tables = pattern.findall(text) diff --git a/src/utils.py b/src/utils.py index 754a04b..d736fc0 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,7 +1,6 @@ import os import pandas as pd import shutil -import boto3 import config From a4b50d3d2a4a67dc72f341e7fe66477d861a2e9b Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 2 May 2024 10:27:10 -0700 Subject: [PATCH 10/78] Debugged multi-prompt layering --- src/bottom_up_funcs.py | 29 +- src/config.py | 4 - src/dict_operations.py | 38 +- src/prompts.py | 14 +- src/test.ipynb | 333 ++++++++++++++ test.ipynb | 971 ----------------------------------------- 6 files changed, 374 insertions(+), 1015 deletions(-) create mode 100644 src/test.ipynb delete mode 100644 test.ipynb diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index f8f0367..07e0f54 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -11,10 +11,9 @@ import postprocess def run_bottom_up_lesser(d, text_dict, tokens=8000): - print(d) prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - lesser_of_dict = dict_operations.dict_string_to_dict(lesser_of_answer) + lesser_of_dict = dict_operations.secondary_string_to_dict(lesser_of_answer) lesser_of_dict['page_num'] = d['page_num'] if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': lesser_of_dict['SERVICE'] = d['SERVICE'] @@ -47,8 +46,7 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # New rows for lesser of temp_dicts = [] - for d in answer_dicts: - print(d) + for d in answer_dicts: # Bottom Up LOB # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[d['page_num']]) # lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) @@ -63,8 +61,8 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # Add additional fields to each row final_dicts = [] for d in temp_dicts: - print(d) - if d: + if d is not None: + page_num = d['page_num'] # # Bottom Up Methodology # prompt = prompts.BOTTOM_UP_METHODOLOGY(dict) # methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) @@ -73,20 +71,20 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # # Bottom Up FS prompt = prompts.BOTTOM_UP_FS(d) fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) - fs_dict = dict_operations.dict_string_to_dict(fs_answer) - d = d.update(fs_dict) + fs_dict = dict_operations.secondary_string_to_dict(fs_answer) + d.update(fs_dict) # # Bottom Up Exception/Escalator - prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[d['page_num']]) + prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num]) exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) - exc_dict = dict_operations.dict_string_to_dict(exc_answer) - d = d.update(exc_dict) + exc_dict = dict_operations.secondary_string_to_dict(exc_answer) + d.update(exc_dict) # # Bottom Up Codes - prompt = prompts.BOTTOM_UP_CODES(d, text_dict[d['page_num']]) + prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num]) codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - codes_dict = dict_operations.dict_string_to_dict(codes_answer) - d = d.update(codes_dict) + codes_dict = dict_operations.secondary_string_to_dict(codes_answer) + d.update(codes_dict) final_dicts.append(d) @@ -104,9 +102,8 @@ def bottom_up(file_object): #### PROMPT #### # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) - answer_dicts = dict_operations.list_string_to_dict(answer_strings) + answer_dicts = dict_operations.primary_string_to_dict(answer_strings) answer_dicts = postprocess.primary_filter(answer_dicts) - print(len(answer_dicts)) # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) diff --git a/src/config.py b/src/config.py index 2bb36d3..83576c1 100644 --- a/src/config.py +++ b/src/config.py @@ -4,7 +4,6 @@ import boto3 # General Settings TEST = False # True to run test prompt - just for testing model connection - VERBOSE = True CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") @@ -18,9 +17,6 @@ OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="update here" -AWS_SECRET_ACCESS_KEY="update here" -AWS_SESSION_TOKEN="update here" # File Paths LOCAL_PATH = 'docs/test/' # Replace with local diff --git a/src/dict_operations.py b/src/dict_operations.py index 67a063c..6ef4c04 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -1,26 +1,30 @@ import re +import json from collections import defaultdict -def list_string_to_dict(string_dict): - # Regular expression to capture key-value pairs - regex_pattern = r'\{\s*([^}]*)\s*\}' - key_value_pattern = r'"([^"]+)":\s*"([^"]+)"' +def primary_string_to_dict(string_dict): data = [] - for page in string_dict.keys(): - dict_string = string_dict[page] - groups = re.findall(regex_pattern, dict_string) - for group in groups: - key_values = re.findall(key_value_pattern, group) - key_value_dict = {key: value for key, value in key_values} - key_value_dict['page_num'] = page - data.append(key_value_dict) + pattern = r'\{.*?\}' + for page_num in string_dict.keys(): + primary_list = string_dict[page_num] + dicts = primary_list.split('[')[1] # Strip front + dicts = dicts.split(']')[0] # Strip back + dicts = dicts.replace('\n', '') # Remove new lines + dicts = re.sub(r"(?>>(100%)<<< of current year CMS Home Health Prospective Payment System (PPS) for each specific Home Health Facility.\"\\n }\\n]'}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "primary_dict" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "text_dict = preprocess.split_text(input_dict[filename])\n", + "text_dict = preprocess.highlight_rates(text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [], + "source": [ + "def BOTTOM_UP_CODES(d, page):\n", + " return f\"\"\"\n", + "### PAGE START ### {page} ### PAGE END\n", + "\n", + "The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\n", + "Service: {d['SERVICE']}\n", + "Methodology: {d['FULL_METHODOLOGY']}\n", + "\n", + "Your job is to identify codes associated with this specific term.\n", + "\n", + "Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\n", + "'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility.\n", + "'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\n", + "'REIMBURSEMENT_GROUPER_CODES' : 2- or 3-digit codes that may be labelled DRG or MS-DRG.\n", + "'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service.\n", + "'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\n", + "'REIMBURSEMENT_REVENUE_CODES' : 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes.\n", + "'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status.\n", + "\n", + "Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\n", + "\n", + "Only return the dictionary, with no other commentary or explanation.\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": {}, + "outputs": [], + "source": [ + "prompt = BOTTOM_UP_CODES(primary_dict[1], text_dict['3'])" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n### PAGE START ### 3 Provider\\'s Name and Address Provider\\'s Tax ID Number (or UPIN) NPI Written referral, if applicable Clean Claim shall not include those claims which require Coordination of Benefits and Third Party Liability issues until receipt of Explanation of Benefits from primary carrier or claims, which are being reviewed by the Medical Director or Peer Review Committee for Medical Necessity. 1.3 \"CMS\" means the Centers for Medicare and Medicaid Services. 1.4 \"Covered Services\" means those medically necessary health care services covered under a Health Benefits Plan, as determined under the terms and conditions of the applicable Health Benefits Plan. 1.5 \"Downstream Entity\" means any party that enters into an acceptable written arrangement below the level of the arrangement between ODS and a first-tier entity. These written arrangements continue down to the level of the ultimate provider of health and/or administrative services. 1.6 \"Emergency Medical Condition\" means a medical condition manifesting itself by acute symptoms of sufficient severity (including severe pain) such that a prudent layperson, with an average knowledge of health and medicine, could reasonably expect the absence of immediate medical attention to result in (a) serious jeopardy to the health of the individual or, in the case of a pregnant woman, the health of a woman or her unborn child; (b) serious impairment to bodily functions; or (c) serious dysfunction of any bodily organ or part. 1.7 \"Emergency Services\" means covered inpatient and outpatient services furnished by a provider qualified to furnish emergency services; and needed to evaluate or stabilize an emergency medical condition. 1.8 \"First Tier Entity\" means any party that enters into a written agreement with ODS to provide administrative or health care services for a Medicare Advantage eligible individual. 1.9 \"Health Benefits Plan\" means a health benefits plan or a group health benefits plan, including individual or group health insurance policies, offering the services of approved health care Facilities and Providers participating in the ODS Medicare Advantage and Medicare Advantage PPO Plans-underwritten or administered by ODS and which describe the Covered Services, applicable co-payments (if any) and deductibles (if any) and other information pertinent to the provision of services. 1.10 \"Hospital\" means a facility licensed by the State of Oregon to provide inpatient and outpatient health care and who has entered into an agreement with ODS to provide services as described herein to Medicare Advantage Members. 1.11 \"Medical Emergency\" means a situation in which a \"medical emergency\" is present and when the Medicare Advantage Member reasonably believes that the Medicare Advantage Member\\'s health is in serious danger. A medical emergency includes severe pain, a bad injury, a serious illness, or a medical condition that is quickly getting much worse. 1.12 \"Medicare Advantage Member\" means a Medicare beneficiary entitled to receive coverage for certain health care services under the terms of the Medicare Advantage Evidence of Coverage, 2012 ODS Medicare Advantage Hospital Agreement Peace Health Southwest Medical Center 2 of 27 ### PAGE END\\n\\nThe above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\\nService: Pneumococcal and influenza vaccinations\\nMethodology: For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83\\n\\nYour job is to identify codes associated with this specific term.\\n\\nRead and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\\n\\'REIMBURSEMENT_ADMITTYPE_CODES\\' : Single-digit codes that indicate the type of admission to a facility.\\n\\'REIMBURSEMENT_DIAG_CODES\\' : Multi-digit alphanumeric codes related to diagnosis. They may contain a \\'.\\' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\\n\\'REIMBURSEMENT_GROUPER_CODES\\' : 2- or 3-digit codes that may be labelled DRG or MS-DRG.\\n\\'REIMBURSEMENT_PLACEOFSERVICE_CODES\\' : 2-digit codes related to the place of service.\\n\\'REIMBURSEMENT_PROC_CODES\\' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\\n\\'REIMBURSEMENT_REVENUE_CODES\\' : 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as \\'Rev\\' codes.\\n\\'REIMBURSEMENT_STATUS_INDICATOR_CODES\\' : 1- or 2-digit alphanumeric codes indicating the payment status.\\n\\nReturn a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\\n\\nOnly return the dictionary, with no other commentary or explanation.\\n'" + ] + }, + "execution_count": 58, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "prompt" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"{\\n 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\\n 'REIMBURSEMENT_DIAG_CODES': 'N/A',\\n 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\\n 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\\n 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\\n 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\\n 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'\\n}\"" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "answer" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\n", + " 'REIMBURSEMENT_DIAG_CODES': 'N/A',\n", + " 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\n", + " 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\n", + " 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\n", + " 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\n", + " 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "\n", + "def primary_string_to_dict(string_dict):\n", + " data = []\n", + " pattern = r'\\{.*?\\}'\n", + " for page_num in string_dict.keys():\n", + " primary_list = string_dict[page_num]\n", + " dicts = primary_list.split('[')[1] # Strip front\n", + " dicts = dicts.split(']')[0] # Strip back\n", + " dicts = dicts.replace('\\n', '') # Remove new lines\n", + " dict_list = re.findall(pattern, dicts)\n", + " dict_list = [json.loads(d) for d in dict_list]\n", + " for dict_ in dict_list:\n", + " dict_['page_num'] = page_num\n", + " data.append(dict_)\n", + " return data\n", + "\n", + "def secondary_string_to_dict(dict_string):\n", + "# Find the start and end positions of the dictionary substring\n", + " start_index = dict_string.find('{')\n", + " end_index = dict_string.rfind('}') + 1\n", + "\n", + " # Extract the dictionary substring\n", + " dict_substring = dict_string[start_index:end_index]\n", + "\n", + " # Replace single quotes with double quotes to make it valid JSON\n", + " dict_substring = dict_substring.replace(\"'\", '\"')\n", + "\n", + " # Convert the substring to a dictionary\n", + " result_dict = json.loads(dict_substring)\n", + "\n", + " return result_dict\n", + "\n", + "\n", + "#primary_string_to_dict(primary_dict)\n", + "\n", + "secondary_dict = \"{\\n 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\\n 'REIMBURSEMENT_DIAG_CODES': 'N/A',\\n 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\\n 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\\n 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\\n 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\\n 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'\\n}\"\n", + "secondary_string_to_dict(secondary_dict)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'4': '[\\n {\\n \"SERVICE\": \"Primary Care Physician\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Primary Care Physician means a participating physician who is a family physician, nurse practitioner, physicians assistant, pediatrician or internist, and whose billings for primary care services are at least 50 percent of the physician\\'s total billings.\"\\n }\\n]',\n", + " '18': 'Here is the list of JSON objects with the relevant attributes extracted from the given text:\\n\\n[\\n {\\n \"SERVICE\": \"Professional Liability Insurance\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"1,000,000\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"During the term of this Agreement, Hospital shall maintain professional liability insurance in an amount not less than $1,000,000 per claim/$3,000,000 aggregate.\"\\n },\\n {\\n \"SERVICE\": \"General Liability Insurance\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"1,000,000\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"During the term of this Agreement, Hospital shall maintain general liability insurance in an amount not less than $1,000,000 per claim/$3,000,000 aggregate.\"\\n }\\n]',\n", + " '24': '[\\n {\\n \"SERVICE\": \"Hospital Inpatient Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"103%\",\\n \"FULL_METHODOLOGY\": \"103% of Medicare Allowance using the CMS Prospective Payment System in effect at the time of services provided.\"\\n },\\n {\\n \"SERVICE\": \"Hospital Outpatient Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\", \\n \"REIMBURSEMENT_RATE\": \"103%\",\\n \"FULL_METHODOLOGY\": \"103% of Medicare Allowance using the CMS Prospective Payment System in effect at the time of services provided.\"\\n },\\n {\\n \"SERVICE\": \"Surgery Secondary procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of the allowed amount for the procedure.\"\\n },\\n {\\n \"SERVICE\": \"Subsequent procedures after primary and secondary on same day\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of the allowed amount for those procedures.\"\\n },\\n {\\n \"SERVICE\": \"Primary procedure for some self-funded plans\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"100% of allowance\"\\n },\\n {\\n \"SERVICE\": \"Secondary procedure for some self-funded plans\", \\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of allowance\"\\n },\\n {\\n \"SERVICE\": \"Remaining procedures after primary and secondary for some self-funded plans\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"25%\",\\n \"FULL_METHODOLOGY\": \"25% of allowance\"\\n }\\n]',\n", + " '25': '[\\n {\\n \"SERVICE\": \"Medicine\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Surgery\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Lab/Pathology\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Radiology\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Pharmaceuticals\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Anesthesia\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"DME\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Unlisted Procedures and/or Supplies\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\", \\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Allow at fifty percent (50%) of billed charges for medically necessary supplies or unlisted procedures (a procedure without a Relative Value Unit).\"\\n },\\n {\\n \"SERVICE\": \"Secondary procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Secondary procedures performed on the same day as primary procedure will be reimbursed at 50% of the allowed amount for that procedure.\"\\n },\\n {\\n \"SERVICE\": \"Subsequent procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Any subsequent procedures performed on the same day as the primary procedure will be reimbursed at 50% of the allowed amount for those procedures.\"\\n },\\n {\\n \"SERVICE\": \"Primary procedure\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Secondary code\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Remaining codes\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"25%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Ambulatory Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"250.00\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"Ambulatory services administered to patient will be reimbursed at a rate of $250.00 per diem.\"\\n }\\n]',\n", + " '28': '[\\n {\\n \"SERVICE\": \"Home Health\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"ODS will reimburse the facility according to the one hundred percent >>>(100%)<<< of current year CMS Home Health Prospective Payment System (PPS) for each specific Home Health Facility.\"\\n }\\n]'}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "primary_dict" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/test.ipynb b/test.ipynb deleted file mode 100644 index f15d5e2..0000000 --- a/test.ipynb +++ /dev/null @@ -1,971 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "# Imports\n", - "import json\n", - "import boto3\n", - "import os\n", - "import dotenv\n", - "import anthropic\n", - "import re\n", - "import pandas as pd\n", - "\n", - "pd.set_option('display.max_rows', None)\n", - "pd.set_option('display.max_columns', None)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Globals\n", - "\n", - "# Load environment variables\n", - "dotenv.load_dotenv()\n", - "\n", - "# Constants for model IDs\n", - "#MODEL_ID_CLAUDE3 = 'anthropic.claude-3-haiku-20240307-v1:0'\n", - "MODEL_ID_CLAUDE3 = 'anthropic.claude-3-sonnet-20240229-v1:0'\n", - "MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1'\n", - "\n", - "# Initialize Bedrock runtime client\n", - "bedrock_runtime = boto3.client(\n", - " service_name=\"bedrock-runtime\",\n", - " region_name=\"us-east-1\",\n", - " aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),\n", - " aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),\n", - " aws_session_token=os.getenv('AWS_SESSION_TOKEN')\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Claude calls\n", - "def invoke_claude_2(prompt, max_tokens):\n", - " body = json.dumps(\n", - " {\"prompt\": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, \n", - " \"max_tokens_to_sample\": max_tokens,\n", - " \"temperature\":0.0,\n", - " \"top_p\":1,\n", - " \"top_k\":250,\n", - " \"stop_sequences\":[anthropic.HUMAN_PROMPT]\n", - " }\n", - " )\n", - "\n", - " response = bedrock_runtime.invoke_model(\n", - " body=body, \n", - " modelId=MODEL_ID_CLAUDE2, \n", - " accept=\"application/json\", \n", - " contentType=\"application/json\"\n", - " )\n", - "\n", - " response_body = json.loads(response.get(\"body\").read())\n", - "\n", - " response_text = response_body['completion']\n", - " return response_text\n", - "\n", - "def invoke_claude_3(prompt, max_tokens = 0):\n", - " prompt = prompt\n", - " body = json.dumps({\n", - " \"anthropic_version\": \"bedrock-2023-05-31\",\n", - " \"max_tokens\": max_tokens,\n", - " \"temperature\": 0.0,\n", - " \"messages\": [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\n", - " \"type\": \"text\",\n", - " \"text\":prompt\n", - " }\n", - " ]\n", - " }\n", - " ]\n", - " }\n", - " )\n", - "\n", - " response = bedrock_runtime.invoke_model(\n", - " body=body, \n", - " modelId=MODEL_ID_CLAUDE3, \n", - " accept=\"application/json\", \n", - " contentType=\"application/json\"\n", - " )\n", - "\n", - " response_body = json.loads(response.get(\"body\").read())\n", - "\n", - " return (response_body['content'][0]['text'])" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'test'" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Test\n", - "invoke_claude_3(\"Write 'test', nothing more.\", max_tokens=100)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# read texts\n", - "# Define the path to the file\n", - "folder_path = r'C:\\\\Users\\\\kminhas\\\\Documents\\\\Doczy\\\\doczy.ai\\\\texts\\\\training-data\\\\contract-text-file\\\\test1\\\\'\n", - "\n", - "file_contents = {}\n", - "\n", - "for filename in os.listdir(folder_path):\n", - " # Create full file path\n", - " file_path = os.path.join(folder_path, filename)\n", - " # Check if the file is a text file\n", - " if os.path.isfile(file_path) and file_path.endswith('.txt'):\n", - " try:\n", - " # First attempt to open the file with UTF-8 encoding\n", - " with open(file_path, 'r', encoding='utf-8') as file:\n", - " file_contents[filename] = file.read()\n", - " except UnicodeDecodeError:\n", - " # If UTF-8 fails, try reading the file with ANSI encoding\n", - " try:\n", - " with open(file_path, 'r', encoding='cp1252') as file:\n", - " file_contents[filename] = file.read()\n", - " except UnicodeDecodeError:\n", - " # If ANSI also fails, log an error message or handle it accordingly\n", - " print(f\"Failed to decode {file_path} with UTF-8 and cp1252 encodings.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [], - "source": [ - "def BOTTOM_UP_MAIN(page, payer):\n", - " return f\"\"\"\n", - "### PAGE START ### {page} ### PAGE END\n", - "\n", - "The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. \n", - "\n", - "Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more.\n", - "\n", - "If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words.\n", - "\n", - "Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. \n", - "It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list.\n", - "\n", - "Here are the attributes to be included in each dictionary, and instructions on how to correctly answer:\n", - "\n", - "'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional.\n", - "'PLAN' : What Plan is the reimbursement schedule for? This the brand name of the health plan.\n", - "'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar.\n", - "'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement.\n", - "'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A.\n", - "'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program.\n", - "'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'.\n", - "'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'.\n", - "'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. \n", - "'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based.\n", - "'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable.\n", - "'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'.\n", - "'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met.\n", - "'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar.\n", - "'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them.\n", - "'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value.\n", - "\n", - "When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of.\n", - "As an example, if you see that a reimbursement says something like \"specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule\", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step.\n", - "\n", - "Only return the list, with no other commentary or explanation.\n", - "\"\"\"\n", - "\n", - "test_prompt = \"\"\"\n", - "Definitions:\n", - "Primary Reimbursement - The generalized rate for which a specialty or service is reimbursed.\n", - "Carveout Reimbursement - A specific rate that is different or specific than the primary service\n", - "\n", - "Text: \n", - "Services:\n", - "Professional Services\n", - "Physician agrees to participate in the Benefit Plan/Program described in this Exhibit and authorizes, through its\n", - "signature below, the transfer of all payment/reimbursement terms and obligations under the Agreement to Payors\n", - "as set forth in this Agreement.\n", - "Subject to allowed reductions that Community may apply as a result of Physician's non-compliance with any\n", - "applicable credentialing, billing, Prior Authorization, Referral, or Utilization Management guidelines, or other\n", - "Community Protocols referenced in this Agreement, Community shall compensate Physician for Covered Services\n", - "rendered by Physician or Healthcare Provider according to the lessor of Contracted Physician's Billed Charges or\n", - "the following schedule, less any Member Expense:\n", - "One hundred and five percent (105%) of the then current Texas Medicaid Fee Schedule unless otherwise\n", - "stated herein, in accordance with the Texas Medicaid reimbursement methodology.\n", - "Clinical Laboratory services (Current Procedural Coding (CPT) codes 80000 through 87999) shall be\n", - "reimbursed at one hundred percent (100%) of the then current Texas Medicaid Clinical Laboratory Fee\n", - "Schedule, in accordance with Texas Medicaid reimbursement methodology. Physician agrees to only bill\n", - "for and Community shall be obligated to only pay for clinical laboratory services for which Physician holds\n", - "CLIA waiver.\n", - "Radiology services (CPT codes 70000 through 79999) shall be reimbursed at one hundred percent (100%)\n", - "of the Texas Medicaid Fee Schedule, in accordance with Texas Medicaid reimbursement methodology.\n", - "Drugs dispensed and administered by Physician that are Covered Services under this Agreement shall be\n", - "reimbursed at one hundred percent (100%) of the Texas Medicaid reimbursement, in accordance with Texas\n", - "Medicaid reimbursement methodology.\n", - "Compensation Notes:\n", - "If Physieian bills for a Covered Service for which no reimbursement is defined by any of the methods listed above,\n", - "Community shall compensate Physician according to thirty percent (30%) of Physician's Billed Charges.\n", - "\n", - "\n", - "Based on the definitions provided, in the text above, which are Primary Reimbursements and which are Carveout Reimbursements?\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Based on the definitions and the text provided, here is how I would categorize the reimbursements mentioned:\n", - "\n", - "Primary Reimbursement:\n", - "- 105% of the then current Texas Medicaid Fee Schedule (for most professional services)\n", - "\n", - "Carveout Reimbursements (specific rates different than the primary):\n", - "- 100% of the then current Texas Medicaid Clinical Laboratory Fee Schedule (for clinical laboratory services CPT 80000-87999)\n", - "- 100% of the Texas Medicaid Fee Schedule (for radiology services CPT 70000-79999) \n", - "- 100% of the Texas Medicaid reimbursement (for drugs dispensed and administered by the physician)\n", - "\n", - "The text states that most professional services will be reimbursed at 105% of the Texas Medicaid Fee Schedule, which appears to be the generalized or primary reimbursement rate. However, it then carves out specific reimbursement rates for clinical laboratory services, radiology services, and drugs that are different from the primary 105% rate.\n" - ] - } - ], - "source": [ - "print(invoke_claude_3(test_prompt, max_tokens=1000))" - ] - }, - { - "cell_type": "code", - "execution_count": 40, - "metadata": {}, - "outputs": [], - "source": [ - "# Add 100% to billed charges when there isn't already a percentage\n", - "def clean_billed_charges(text):\n", - " # This function finds all occurrences of 'billed charges' and checks the preceding 15 characters for '%'\n", - " def replace(match):\n", - " start_pos = match.start() # Get the start position of the match\n", - " pre_text = text[max(0, start_pos-15):start_pos] # Extract up to 15 characters before the match\n", - " if '%' in pre_text:\n", - " return match.group() # If '%' is found, return the original 'billed charges'\n", - " else:\n", - " return '100% of billed charges' # If '%' is not found, prepend '100% of'\n", - "\n", - " # Use regular expression to replace 'billed charges' with the modified version if needed, using re.IGNORECASE for case insensitivity\n", - " return re.sub(r'\\bbilled charges\\b', replace, text, flags=re.IGNORECASE)\n", - "\n", - "def split_text(text):\n", - " text_list = text.split('Start of Page No. = ')\n", - " text_dict = {page.split()[0] : page for page in text_list}\n", - " return text_dict\n", - "\n", - "def string_to_dict(string_dict):\n", - " # Regular expression to capture key-value pairs\n", - " regex_pattern = r'\\{\\s*([^}]*)\\s*\\}'\n", - " key_value_pattern = r'\"([^\"]+)\":\\s*\"([^\"]+)\"'\n", - " data = []\n", - " for page in string_dict.keys():\n", - " dict_string = string_dict[page]\n", - " groups = re.findall(regex_pattern, dict_string)\n", - " for group in groups:\n", - " key_values = re.findall(key_value_pattern, group)\n", - " key_value_dict = {key: value for key, value in key_values}\n", - " key_value_dict['page_num'] = page\n", - " data.append(key_value_dict)\n", - " return data\n", - " \n", - "def run_bottom_up_prompt(text_dict, client_name, tokens):\n", - " answer_dict = {}\n", - " for page_number in text_dict.keys():\n", - " if page_number.isdigit():\n", - " if int(page_number) in (22, 23):\n", - " prompt = BOTTOM_UP_MAIN(text_dict[page_number], client_name)\n", - " #print(len(prompt))\n", - " answer = invoke_claude_3(prompt, max_tokens=tokens)\n", - " answer_dict[page_number] = answer\n", - " return answer_dict\n", - "\n", - "def run_lesser_of_prompt(text_dict, result_dicts):\n", - " valid_dicts = [dict for dict in result_dicts if dict['LESSER_OF_LANGUAGE_IND'] == 'Y']\n", - " pages_to_check = list(set([dict['page_num'] for dict in valid_dicts]))\n", - " for page_num in pages_to_check:\n", - " page_text = text_dict[page_num]\n", - " dicts_on_page = [dict for dict in valid_dicts if dict['page_num'] == page_num]\n", - " print(dicts_on_page)\n", - "\n", - "\n", - "def append_secondary(result_dicts):\n", - " from collections import defaultdict\n", - "\n", - " # Helper function to create a key from dict excluding certain keys\n", - " def create_key(d):\n", - " return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT'))\n", - "\n", - " # Group dictionaries by non-REIMBURSEMENT fields\n", - " grouped = defaultdict(list)\n", - " for d in result_dicts:\n", - " grouped[create_key(d)].append(d)\n", - "\n", - " # Process groups to create combined entries\n", - " combined_results = []\n", - " for key, group in grouped.items():\n", - " # Create a new dictionary based on non-REIMBURSEMENT fields\n", - " new_dict = {k: v for k, v in key}\n", - " # Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary\n", - " for i, entry in enumerate(group[:3]): # Process only the first three entries\n", - " for k, v in entry.items():\n", - " if k.startswith('REIMBURSEMENT'):\n", - " new_key = f'{[\"PRIMARY\", \"SECONDARY\", \"TERTIARY\"][i]}_{k}'\n", - " new_dict[new_key] = v\n", - " combined_results.append(new_dict)\n", - "\n", - " return combined_results\n", - " \n", - "\n", - "def bottom_up_extract(filename, client_name, contract_text):\n", - " # add '100% of ' to billed charges\n", - " contract_text = clean_billed_charges(contract_text)\n", - "\n", - " # Split text into pages\n", - " text_dict = split_text(contract_text)\n", - "\n", - " # Extract for each page\n", - " answer_dicts = run_bottom_up_prompt(text_dict, client_name, 8000)\n", - "\n", - " # Convert results to list of dictionary\n", - " answer_dicts = string_to_dict(answer_dicts)\n", - "\n", - " # combine dicts to primary, secondary, tertiary fields\n", - " answer_dicts = append_secondary(answer_dicts)\n", - "\n", - " # Add filename\n", - " for dict in answer_dicts:\n", - " dict['Filename'] = filename\n", - " \n", - " return answer_dicts\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "all_dicts = []\n", - "for filename, file_text in file_contents.items():\n", - " file_results_list = bottom_up_extract(filename, 'CareSource', file_text)\n", - " all_dicts += file_results_list" - ] - }, - { - "cell_type": "code", - "execution_count": 76, - "metadata": {}, - "outputs": [], - "source": [ - "all_results_df = pd.DataFrame(all_dicts)" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [], - "source": [ - "all_results_df.to_csv('test_output_20240417.csv')" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [], - "source": [ - "filename = '2017 01 01 Raul A. Rivera & Associates, P.A. (Master)PHY Executed MU.txt'\n", - "result = bottom_up_extract(filename, 'Community Health Choice', file_contents[filename])" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
CONTRACT_LOBPLANPROGRAMPROV_TYPEPROV_SPECIALTYAPPLICABLE_TOGREATER_OF_LANGUAGE_INDLESSER_OF_LANGUAGE_INDpage_numPRIMARY_REIMBURSEMENT_METHODOLOGYPRIMARY_REIMBURSEMENT_FEE_SCHEDULEPRIMARY_REIMBURSEMENT_CODESPRIMARY_REIMBURSEMENT_RATE_ESCALATOR_INDPRIMARY_REIMBURSEMENT_EXCEPTION_INDPRIMARY_REIMBURSEMENT_GROUPERPRIMARY_REIMBURSEMENT_FLAT_FEEPRIMARY_REIMBURSEMENT_RATESECONDARY_REIMBURSEMENT_METHODOLOGYSECONDARY_REIMBURSEMENT_FEE_SCHEDULESECONDARY_REIMBURSEMENT_CODESSECONDARY_REIMBURSEMENT_RATE_ESCALATOR_INDSECONDARY_REIMBURSEMENT_EXCEPTION_INDSECONDARY_REIMBURSEMENT_GROUPERSECONDARY_REIMBURSEMENT_FLAT_FEESECONDARY_REIMBURSEMENT_RATEFilename
0MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Family Practice, Internal Medicine, General Pr...N/ANY22Billed ChargesN/AN/ANNN/AN/A100%Texas Medicaid Fee ScheduleTexas Medicaid Fee ScheduleN/ANNN/AN/A105%2017 01 01 Raul A. Rivera & Associates, P.A. (...
1MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Clinical Laboratory servicesN/ANN22Texas Medicaid Clinical Laboratory Fee ScheduleTexas Medicaid Clinical Laboratory Fee Schedule80000 through 87999NYN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
2MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Radiology servicesN/ANN22Texas Medicaid Fee ScheduleTexas Medicaid Fee Schedule70000 through 79999NNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
3MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Drugs dispensed and administeredN/ANN22Texas Medicaid reimbursementN/AN/ANNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
4MedicaidN/ASTARPhysician and/or Mid-Level Practitioners - Pri...Professional ServicesN/ANN22Billed ChargesN/AN/ANYN/AN/A30%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
5MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Professional ServicesN/ANY23Medicare Fee ScheduleMedicare Physician Fee ScheduleN/ANNN/AN/A100%Billed ChargesN/AN/ANNN/AN/A100%2017 01 01 Raul A. Rivera & Associates, P.A. (...
6MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Clinical Laboratory servicesN/ANN23Medicare Fee ScheduleMedicare Clinical Laboratory Fee Schedule80000 through 87999NYN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
7MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Radiology servicesN/ANN23Medicare Fee ScheduleMedicare Professional Fee Schedule70000 through 79999NNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
8MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...Drugs dispensed and administeredN/ANN23APS reimbursementN/AN/ANNN/AN/A100%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
9MarketplaceN/AN/AAll Physician and Mid-level Healthcare Profess...N/AN/ANN23Billed ChargesN/AN/ANNN/AN/A30%NaNNaNNaNNaNNaNNaNNaNNaN2017 01 01 Raul A. Rivera & Associates, P.A. (...
\n", - "
" - ], - "text/plain": [ - " CONTRACT_LOB PLAN PROGRAM \\\n", - "0 Medicaid N/A STAR \n", - "1 Medicaid N/A STAR \n", - "2 Medicaid N/A STAR \n", - "3 Medicaid N/A STAR \n", - "4 Medicaid N/A STAR \n", - "5 Marketplace N/A N/A \n", - "6 Marketplace N/A N/A \n", - "7 Marketplace N/A N/A \n", - "8 Marketplace N/A N/A \n", - "9 Marketplace N/A N/A \n", - "\n", - " PROV_TYPE \\\n", - "0 Physician and/or Mid-Level Practitioners - Pri... \n", - "1 Physician and/or Mid-Level Practitioners - Pri... \n", - "2 Physician and/or Mid-Level Practitioners - Pri... \n", - "3 Physician and/or Mid-Level Practitioners - Pri... \n", - "4 Physician and/or Mid-Level Practitioners - Pri... \n", - "5 All Physician and Mid-level Healthcare Profess... \n", - "6 All Physician and Mid-level Healthcare Profess... \n", - "7 All Physician and Mid-level Healthcare Profess... \n", - "8 All Physician and Mid-level Healthcare Profess... \n", - "9 All Physician and Mid-level Healthcare Profess... \n", - "\n", - " PROV_SPECIALTY APPLICABLE_TO \\\n", - "0 Family Practice, Internal Medicine, General Pr... N/A \n", - "1 Clinical Laboratory services N/A \n", - "2 Radiology services N/A \n", - "3 Drugs dispensed and administered N/A \n", - "4 Professional Services N/A \n", - "5 Professional Services N/A \n", - "6 Clinical Laboratory services N/A \n", - "7 Radiology services N/A \n", - "8 Drugs dispensed and administered N/A \n", - "9 N/A N/A \n", - "\n", - " GREATER_OF_LANGUAGE_IND LESSER_OF_LANGUAGE_IND page_num \\\n", - "0 N Y 22 \n", - "1 N N 22 \n", - "2 N N 22 \n", - "3 N N 22 \n", - "4 N N 22 \n", - "5 N Y 23 \n", - "6 N N 23 \n", - "7 N N 23 \n", - "8 N N 23 \n", - "9 N N 23 \n", - "\n", - " PRIMARY_REIMBURSEMENT_METHODOLOGY \\\n", - "0 Billed Charges \n", - "1 Texas Medicaid Clinical Laboratory Fee Schedule \n", - "2 Texas Medicaid Fee Schedule \n", - "3 Texas Medicaid reimbursement \n", - "4 Billed Charges \n", - "5 Medicare Fee Schedule \n", - "6 Medicare Fee Schedule \n", - "7 Medicare Fee Schedule \n", - "8 APS reimbursement \n", - "9 Billed Charges \n", - "\n", - " PRIMARY_REIMBURSEMENT_FEE_SCHEDULE \\\n", - "0 N/A \n", - "1 Texas Medicaid Clinical Laboratory Fee Schedule \n", - "2 Texas Medicaid Fee Schedule \n", - "3 N/A \n", - "4 N/A \n", - "5 Medicare Physician Fee Schedule \n", - "6 Medicare Clinical Laboratory Fee Schedule \n", - "7 Medicare Professional Fee Schedule \n", - "8 N/A \n", - "9 N/A \n", - "\n", - " PRIMARY_REIMBURSEMENT_CODES PRIMARY_REIMBURSEMENT_RATE_ESCALATOR_IND \\\n", - "0 N/A N \n", - "1 80000 through 87999 N \n", - "2 70000 through 79999 N \n", - "3 N/A N \n", - "4 N/A N \n", - "5 N/A N \n", - "6 80000 through 87999 N \n", - "7 70000 through 79999 N \n", - "8 N/A N \n", - "9 N/A N \n", - "\n", - " PRIMARY_REIMBURSEMENT_EXCEPTION_IND PRIMARY_REIMBURSEMENT_GROUPER \\\n", - "0 N N/A \n", - "1 Y N/A \n", - "2 N N/A \n", - "3 N N/A \n", - "4 Y N/A \n", - "5 N N/A \n", - "6 Y N/A \n", - "7 N N/A \n", - "8 N N/A \n", - "9 N N/A \n", - "\n", - " PRIMARY_REIMBURSEMENT_FLAT_FEE PRIMARY_REIMBURSEMENT_RATE \\\n", - "0 N/A 100% \n", - "1 N/A 100% \n", - "2 N/A 100% \n", - "3 N/A 100% \n", - "4 N/A 30% \n", - "5 N/A 100% \n", - "6 N/A 100% \n", - "7 N/A 100% \n", - "8 N/A 100% \n", - "9 N/A 30% \n", - "\n", - " SECONDARY_REIMBURSEMENT_METHODOLOGY SECONDARY_REIMBURSEMENT_FEE_SCHEDULE \\\n", - "0 Texas Medicaid Fee Schedule Texas Medicaid Fee Schedule \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN \n", - "4 NaN NaN \n", - "5 Billed Charges N/A \n", - "6 NaN NaN \n", - "7 NaN NaN \n", - "8 NaN NaN \n", - "9 NaN NaN \n", - "\n", - " SECONDARY_REIMBURSEMENT_CODES SECONDARY_REIMBURSEMENT_RATE_ESCALATOR_IND \\\n", - "0 N/A N \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN \n", - "4 NaN NaN \n", - "5 N/A N \n", - "6 NaN NaN \n", - "7 NaN NaN \n", - "8 NaN NaN \n", - "9 NaN NaN \n", - "\n", - " SECONDARY_REIMBURSEMENT_EXCEPTION_IND SECONDARY_REIMBURSEMENT_GROUPER \\\n", - "0 N N/A \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN \n", - "4 NaN NaN \n", - "5 N N/A \n", - "6 NaN NaN \n", - "7 NaN NaN \n", - "8 NaN NaN \n", - "9 NaN NaN \n", - "\n", - " SECONDARY_REIMBURSEMENT_FLAT_FEE SECONDARY_REIMBURSEMENT_RATE \\\n", - "0 N/A 105% \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN \n", - "4 NaN NaN \n", - "5 N/A 100% \n", - "6 NaN NaN \n", - "7 NaN NaN \n", - "8 NaN NaN \n", - "9 NaN NaN \n", - "\n", - " Filename \n", - "0 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "1 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "2 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "3 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "4 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "5 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "6 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "7 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "8 2017 01 01 Raul A. Rivera & Associates, P.A. (... \n", - "9 2017 01 01 Raul A. Rivera & Associates, P.A. (... " - ] - }, - "execution_count": 43, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pd.DataFrame(result)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From f6beacf79f7652eb83ef3cd00cb4d3980cf37ce3 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 3 May 2024 07:32:18 -0700 Subject: [PATCH 11/78] 5/2 test update --- src/bottom_up_funcs.py | 54 ++++--- src/bottom_up_main.py | 28 +++- src/config.py | 6 +- src/dict_operations.py | 27 ++-- src/postprocess.py | 1 - src/prompts.py | 30 ++-- src/test.ipynb | 348 +++++++++++++---------------------------- src/utils.py | 20 +-- 8 files changed, 228 insertions(+), 286 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 07e0f54..874fe5c 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -24,17 +24,15 @@ def run_bottom_up_lesser(d, text_dict, tokens=8000): return ({}, d) - def run_bottom_up_primary(text_dict, tokens): #chunk_dict = preprocess.chunk_text(text_dict) - if config.VERBOSE: print("Running Bottom Up Primary") - answer_dict = {} #for page_number in chunk_dict.keys(): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): if page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number]): + # Run Primary prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) answer_dict[page_number] = answer @@ -42,17 +40,9 @@ def run_bottom_up_primary(text_dict, tokens): def run_bottom_up_secondary(answer_dicts, text_dict, tokens): - if config.VERBOSE: print("Running Bottom Up Secondary") - # New rows for lesser of temp_dicts = [] for d in answer_dicts: - # Bottom Up LOB - # prompt = prompts.BOTTOM_UP_LOB(dict, text_dict[d['page_num']]) - # lob_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # lob_dict = dict_operations.dict_string_to_dict(lob_answer) - # d = d.update(lob_dict) - # Bottom Up Lesser lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens) temp_dicts.append(lesser_object[1]) # Add original object from d @@ -63,10 +53,10 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): for d in temp_dicts: if d is not None: page_num = d['page_num'] - # # Bottom Up Methodology - # prompt = prompts.BOTTOM_UP_METHODOLOGY(dict) - # methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - # dict['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + # Bottom Up Methodology + prompt = prompts.BOTTOM_UP_METHODOLOGY(d) + methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer # # Bottom Up FS prompt = prompts.BOTTOM_UP_FS(d) @@ -90,6 +80,28 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts +def run_bottom_up_lob(answer_dicts, text_dict, tokens): + lob_dict = {} + for page_num in set(d['page_num'] for d in answer_dicts): + try: + text_section = text_dict[str(int(page_num)-1)] + text_dict[page_num] + except: + text_section = text_dict[page_num] + prompt = prompts.BOTTOM_UP_LOB(text_section) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + answer_dict = dict_operations.secondary_string_to_dict(answer) + lob_dict[page_num] = answer_dict + return lob_dict + +def consolidate_lob(lob_dicts, results_dicts): + final_dicts = [] + for d in results_dicts: + try: + d.update(lob_dicts[d['page_num']]) + final_dicts.append(d) + except: + final_dicts.append(d) + return final_dicts def bottom_up(file_object): #### PREPROCESS #### @@ -105,16 +117,22 @@ def bottom_up(file_object): answer_dicts = dict_operations.primary_string_to_dict(answer_strings) answer_dicts = postprocess.primary_filter(answer_dicts) + # Bottom Up LOB - CONTRACT_LOB, CONTRACT_PROGRAM, CONTRACT_PRODUCT, PROV_TYPE + lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000) + # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) + # Append LOB + final_dicts = consolidate_lob(lob_dicts, results_dicts) + #### POSTPROCESS #### #answer_dicts = append_secondary(answer_dicts) - for dict in results_dicts: - dict['Filename'] = filename + for d in final_dicts: + d['Filename'] = filename - answer_df = pd.DataFrame(results_dicts) + answer_df = pd.DataFrame(final_dicts) # Write output if config.WRITE_OUTPUT: diff --git a/src/bottom_up_main.py b/src/bottom_up_main.py index b464587..636c4b1 100644 --- a/src/bottom_up_main.py +++ b/src/bottom_up_main.py @@ -1,6 +1,7 @@ # Imports import os import concurrent.futures +import traceback import utils import config @@ -13,6 +14,8 @@ def main(): else: # Read contract txt input_dict = utils.read_input() + already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('results/')] + input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} # Set up results folder if not os.path.exists('results'): @@ -22,10 +25,31 @@ def main(): #tables_dict = table_funcs.extract_all_tables(input_dict) #print(tables_dict) - # Run bottom up primary with multithreading + # # Run bottom up primary with multithreading # with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: # executor.map(bottom_up_funcs.bottom_up, input_dict.items()) - bottom_up_funcs.bottom_up(list(input_dict.items())[0]) + + def process_item(item): + key, value = item + try: + bottom_up_funcs.bottom_up(item) + except Exception as e: + # Print the error message and traceback + print(f"Error processing item {key}: {e}") + traceback.print_exc() + + with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: + # Process each item individually + futures = [executor.submit(process_item, item) for item in input_dict.items()] + + # Wait for all futures to complete + for future in concurrent.futures.as_completed(futures): + # Retrieve any exceptions raised by the thread + try: + future.result() + except Exception as e: + print(f"Error: {e}") + # Write Output if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': diff --git a/src/config.py b/src/config.py index 83576c1..2494200 100644 --- a/src/config.py +++ b/src/config.py @@ -17,9 +17,12 @@ OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' MAX_WORKERS = 20 # AWS Keys +AWS_ACCESS_KEY_ID="update here" +AWS_SECRET_ACCESS_KEY="update here" +AWS_SESSION_TOKEN="update here" # File Paths -LOCAL_PATH = 'docs/test/' # Replace with local +LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', @@ -41,4 +44,3 @@ BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime", MODEL_ID_CLAUDE3_HAIKU = 'anthropic.claude-3-haiku-20240307-v1:0' MODEL_ID_CLAUDE3_SONNET = 'anthropic.claude-3-sonnet-20240229-v1:0' MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1' - diff --git a/src/dict_operations.py b/src/dict_operations.py index 6ef4c04..6eb5b88 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -2,6 +2,21 @@ import re import json from collections import defaultdict +def secondary_string_to_dict(dict_string): + dict_string = dict_string.replace('<<<', '') + dict_string = dict_string.replace('>>>', '') + dict_string = dict_string.replace('\n', '') # Remove new line + + start_index = dict_string.find('{') + end_index = dict_string.rfind('}') + 1 + dict_substring = dict_string[start_index:end_index] + try: + result_dict = json.loads(dict_substring) + except: + dict_substring = dict_substring.replace("'", '"') + result_dict = json.loads(dict_substring) + return result_dict + def primary_string_to_dict(string_dict): data = [] pattern = r'\{.*?\}' @@ -10,22 +25,16 @@ def primary_string_to_dict(string_dict): dicts = primary_list.split('[')[1] # Strip front dicts = dicts.split(']')[0] # Strip back dicts = dicts.replace('\n', '') # Remove new lines + dicts = dicts.replace('<<<', '') + dicts = dicts.replace('>>>', '') dicts = re.sub(r"(?>>(100%)<<< of current year CMS Home Health Prospective Payment System (PPS) for each specific Home Health Facility.\"\\n }\\n]'}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "primary_dict" + "def BOTTOM_UP_LOB(page):\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + "\n", + "The above text represents two pages from a contract between a healthcare Payer and Provider.\n", + "\n", + "Your job is to identify what LOBs, Products, Plans, and Provider Types the reimbursement terms on this page apply to.\n", + "\n", + "Return a dictionary object with the following attributes:\n", + "'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all.\n", + "'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. This will likely only be listed once per page, if at all.\n", + "'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the same as LOB. It is NOT the name of a hospital. This will likely only be listed once per page, if at all.\n", + "'PROV_TYPE' : For what provider type or place of service is this fee schedule for? It could be Hospital, Professional, Ancillary, or similar. This will likely only be listed once per page, if at all.\n", + "\n", + "If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages.\n", + "If there are multiple correct answers for the second page, return them both in a comma-separated list. However, in most instances there will be only one.\n", + "\n", + "Return N/A for any answers not found.\n", + "\n", + "Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.\n", + "\"\"\"\n", + "\n", + "def run_bottom_up_lob(answer_dicts, text_dict, tokens):\n", + " lob_dict = {}\n", + " for page_num in set(d['page_num'] for d in answer_dicts):\n", + " try:\n", + " text_section = text_dict[str(int(page_num)-1)] + text_dict[page_num]\n", + " except:\n", + " text_section = text_dict[page_num]\n", + " prompt = BOTTOM_UP_LOB(text_section)\n", + " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " answer_dict = dict_operations.secondary_string_to_dict(answer)\n", + " lob_dict[page_num] = answer_dict\n", + " return lob_dict\n", + "\n", + "result_dict = run_bottom_up_lob(answer_dicts, text_dict, 8000)" ] }, { @@ -121,184 +99,77 @@ "metadata": {}, "outputs": [], "source": [ - "text_dict = preprocess.split_text(input_dict[filename])\n", + "import config\n", + "import os\n", + "import utils\n", + "def read_input(path=config.LOCAL_PATH, mode=config.READ_MODE):\n", + " if mode == '_LOCAL_':\n", + " files = {}\n", + " for file in os.listdir(path):\n", + " full_path = os.path.join(path, file)\n", + " file_text = utils.read_local(full_path)\n", + " files[file] = file_text\n", + " return files\n", + " elif mode == '_S3_':\n", + " return read_s3()\n", + "\n", + "\n", + "already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('../results/')]\n", + "input_dict = read_input(path='../docs/all_txt_updated')\n", + "input_dict_filtered = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed}" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['050540697 Deepak Nanda 2017 PHSP MU.txt', '133923495 CAIPA 2015 PHSP.txt', '2001-11-01 California Emergency Physicians PSA_MU.txt', '2013-09-01 Fresno Dental Surgery Center PPA_MU.txt', '2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof MU.txt', '2014-01-01 COMM PPO - PeaceHealth Multi TIN - Facility and Prof.txt', '2014-01-01 United Health Centers San Joaquin AMD MU.txt', '2015-01-01 COMM PPO FE - PeaceHealth Multi TIN - Facility and Prof Amendment .txt', '2015-01-01 OHSU Agmt FE (2).txt', '2015-01-01 OHSU Agmt FE.txt', '2017 01 01 Raul A. Rivera & Associates, P.A. (Master)PHY Executed MU.txt', '2017-01-01 COMM Amend FE - PeaceHealth (2).txt', '2017-07-01 DH-Dominican Hospital CDM MU.txt', '2017-07-01 St. Agnes Medical Center AMD MU.txt', '2017-11-01 MA Agmt FE - OHSU (2).txt', '2018 09 01 UTMB (Amendment 10 change in reimbursement) MU.txt', '2018-01-01 COMM Amend FE - OHSU (2).txt', '2018-01-01 COMM Amend FE - PeaceHealth (2).txt', '2019 01 01 Gulf Coast Division (Agreement_STAR, CHIP, CHIP P, STAR_PLUS) MU.txt', '2021 01 01 Houston Methodist (Amendment 7_change in reimbursement) MU.txt', '2021-01-01 COMM Amend FE - Yakima Valley Farm Workers Clinic.txt', '2021-01-01 MA Amend FE - Yakima Valley Farm Workers Clinic - MU.txt', '2021-07-15 COMM Agmt FE - Samaritan Health Services MU.txt', '2021-08-01 Steward Health Care System Hospital Services Agreement MU.txt', '2022 01 20 West Park Primary Care_Master (Master) PHY Executed.pdf.filepart MU.txt', '2022-01-01 Orlando Health Network_Medicare_Language & Reimbursement MU.txt', '2022-10-01 COMM Amend FE - Salem Health Hospitals and Clinics MU.txt', '2023 09 18 MHMD (Amendment 5_update reimbursement HHSC) executed MU.txt', '2023 10 26_Pediatrix Medical Group of T...ment CHIP_STAR_Removed HIM_executed MU.txt', '2023-01-01 MA Amend FE - Salem Health Hospitals and Clinics MU.txt', '221487173_EnglewoodHospitalandMedicalCenter_ICMSupportingDocuments_566705 MU.txt', 'Adena Health System National Template Agreement C19460229AA.txt', 'AGMT - HSA - TEXAS HEALTH RESOURCES - 75-6001743 MU.txt', 'Akron General Health System_Eleventh Amendment_20171001.txt', 'Anc10_040121 Nationwide Vision Center Care 1st Amendment.txt', 'Anc2_060117 Barnet Dulaney Perkins Eye Cntr Care1st Anc - 562589722 MU.txt', 'Anc4_White Mountain Phys Contract MU.txt', 'Anc5_Tipton MU.txt', 'Anc6_090119 SONORA QUEST LABORATORIES LLC Care1st MU.txt', 'Anc8_Hanger Pros & Ortho West Contract.txt', 'App Regional Healthcare_Hospital Agreement_20160414_Dually Executed WV MCD-C13439125AA.txt', 'ASMG MU.txt', 'Bay Ridge Medical Imaging Universal (113045841) MU.txt', 'Baz Allergy Asthma & Sinus_MU.txt', 'BoilerplateFac_Bertrand Chaffee Hospital - Amendment 4.17.15 MU.txt', 'Boilerplate_2016-01-01 17th BayCare Amendment Adding Bartow Regional MU.txt', 'Boilerplate_2020-01-01 COMM Amend FE (Beacon) - Portland Adventist Medical Center MU.txt', 'Boilerplate_AMD - MKPL - NBHD dba Broward Health - Copy_MU.txt', 'Boilerplate_Baptist-Provider Service Agreement_MU.txt', 'Boilerplate_Benjamin Bieber_MU.txt', 'Boilerplate_Bethpage Medical, PLLC - Agreement_MU.txt', \"Boilerplate_HSA_EFF03012012_Children's Medical Center MU.txt\", 'Boilerplate_PSA_Houston Eye Associates_76-0512625_05012007 MU.txt', 'Bon Secours Mercy Health_20190101_Base Contract.txt', 'Bon_Secours_Mercy_Third_Amendment MU.txt', 'BSW _ Original Contract _ 1.1.2000 MU.txt', 'Cabell Huntington_OH MCD Compensation MU.txt', 'ComEMS_ICMProviderAgreement_ICMProviderAgreement_278127_5.txt', 'Community Health Network_Base Contract MU.txt', 'CustomFac_AMD - AMD - UNIVERSITY MEDICAL CENTER OF EL PASO - 74-6000756 MU.txt', 'Custom_060121 HonorHealth and its Affiliates Care 1st Physician Agreement.txt', 'Custom_2013-10-01 AdventHealth Amendment.txt', 'Custom_2014 Slocum Surgery Center PPO Agreement - scan MU .txt', 'Custom_2015-08-01 COMM Amend FE - Slocum Surgery Center MU.txt', 'Custom_2017-04-05 Lee Memorial Amendment Exhibits A-B MU.txt', 'Custom_2017-09-15 COMM Agmt FE - Plaza Ambulatory Surgery Center, LLC MU.txt', 'Custom_2018-01-1 Adentist adding SOF Rates MU.txt', 'Custom_2018-09-15 COMM Agmt FE - Bandon Community Health Center dba Coast Community Health Center.txt', 'Custom_2018-09-15 COMM Amend FE Plaza Ambulatory Surgery Center, LLC MU.txt', 'Custom_2019-10-01 COMM Amend FE (CCM) - Bandon Community Health Center.txt', 'Custom_2020-01-01 Com Affinity Amend FE - St Charles Health System MU.txt', 'Custom_2020-02-01 COMM Amend FE (IPOP) - Portland Adventist Medical Center MU.txt', 'Custom_2022-07-01 COMM Amend #28 FE - Legacy Health Hospital MU.txt', 'Custom_AncProf_Keith C Chang MD PLLC - Agreement Provider Signed 1.1.16.txt', 'Custom_CHI ST. LUKES HEALTH SYSTEM-MP_MU.txt', 'Custom_Community Medical Associates_20150217_Dually Executed_IN MP-ID C12324214AA MU.txt', 'Custom_Dallas County Hospital Parkland 75-6004221 - Amendment 5 MU.txt', \"Custom_HSA_AMD 1_EFF02012012_Driscoll Children's Hospital MU.txt\", \"Custom_HSA_EFF07012010_Driscoll Children's Hospital MU.txt\", 'Custom_KentuckyOne Group Practice Provider Agreement_20140806_Dually Executed (1) MU.txt', 'Custom_Molina Healthcare of Texas, Inc. Amendment 5 - HIX ACA__EFF 09012017 MU.txt', 'Custom_MP AMD - ONCOLIGY CONSULTANTS PA 76-0605200 (003)_MU.txt', 'Custom_Norton Community Medical Associates-IN MCD-ID C13931037AA MU.txt', 'Custom_Parkview Health Systems-OH MP, IN MP-ID C15656222AA MU.txt', 'Custom_Prof_Anna Suponya MD PC - Agreement Provider Signed_MU.txt', 'Custom_Provider Services Agreement 7 24 2006 BMC MU.txt', 'Custom_PSA - Advanced Gastroenterology of Texas PLLC -Executed 47-4543923 MU.txt', 'Custom_PSA - TX - Fresenius BIOMEDICAL APPLICATIONS OF TEXAS INC - 11-2226275 - 06012010 (1) (2) MU.txt', 'Custom_SOUTHWEST BEHAVIORAL & HEALTH SERVICES INC.txt', 'Custom_The Eye Center of Columbus LLC_J4M Deeming Letter MU.txt', 'Custom_TriHealth_Group Practice_Sixth Amendment_20140101 MU.txt', 'Custom_TX - AMD 1 - Davita - 27-0888875 eff 04012015 MU.txt', 'Custom_TX - AMD 2 - MMP - DAVITA 27-0888875 eff 11012015 MU.txt', 'Custom_TX - MP AMENDMENT - MISSION HEALTH NETWORK - MU.txt', 'Custom_ULP Inc_ULRF_20160311_Dually Executed KY J4M_MA-C13371728AA_MU.txt', 'Delaware First Health_First State Homecare Agency_212260_7 MU.txt', 'Fac1_Bullhead City_Western Arizona Med Center - 860982071 MU.txt', 'Fac3_Phoenix Children Hosp_2 MU.txt', 'Genesis_ICMProviderAgreement_ICMProviderAgreement_153537_5.txt', 'Heartland_ICMProviderAgreement_ICMProviderAgreement_144644_5.txt', 'Hospital Authority of Valdosta and Lowndes County Georgia PA_20180817_ Dually Executed Agreement (1).txt', 'HSA_EFF TBD_PrimeHealthcare_MissionRegionalMedicalCenter MU.txt', 'IASIS FULLY EXECUTED MP_0001_NEW 9.7.16 MU.txt', 'ICMProviderAgreementAmendment_ICMProviderAgreementAmendment_169834_6 - Hope Community MU.txt', 'ICMProviderAgreement_101eDelawareAveOperations_211248_9 MU.txt', 'ICMProviderAgreement_AlisonUnitisLPCMH_227710_5.txt', 'ICMProviderAgreement_AmyRiceMA_223388_5.txt', 'ICMProviderAgreement_AnnEWhite-TheCenterForHealingConversations_227589_4.txt', 'ICMProviderAgreement_AvenueMedicalAssociatesPA_212330_5.txt', 'ICMProviderAgreement_AzarEyeSurgeryCenter_232594_6.txt', 'ICMProviderAgreement_BalancedMindCounselingCenter_217525_6.txt', 'ICMProviderAgreement_BanyanDelaware_210080_5.txt', 'ICMProviderAgreement_ChampionsForChildrensMentalHealth_219129_5.txt', 'ICMProviderAgreement_CorasWellnessAndBehavioralHealth_212482_8.txt', 'ICMProviderAgreement_CostalCarePhysicalTherapy_211707_6.txt', 'ICMProviderAgreement_Delaware4TheSeniors_212439_6.txt', 'ICMProviderAgreement_GuardianAngelHomeHealthCareAgency_210054_9.txt', 'ICMProviderAgreement_GuardianAngelHomeHealthCareAgency_210054_9_MU.txt', 'ICMProviderAgreement_ICMProviderAgreement_143342_10_Landmark of Midwest City Rehab and Nursing Center.txt', 'ICMProviderAgreement_ICMProviderAgreement_295973_15.txt', 'ICMProviderAgreement_ICMProviderAgreement_34251_10_Summit Medical Center MU.txt', 'ICMProviderAgreement_ICMProviderAgreement_35455_5.txt', 'ICMProviderAgreement_ICMProviderAgreement_35970_5.txt', 'ICMProviderAgreement_ICMProviderAgreement_49186_5 - Hope Community MU.txt', 'ICMProviderAgreement_Medi-Rents&Sales_212293_10_MU.txt', 'ICMProviderAgreement_MichaelBaer-FirstStateDME_212259_7MU.txt', 'ICMProviderAgreement_NewCastleHealthandRehabilitationCenter_212261_13 MU.txt', 'ICMProviderAgreement_Orthologix_227580_6.txt', 'ICMProviderAgreement_PinkRibbonBoutique_212369_7.txt', 'ICMProviderAgreement_RecoveryInnovations_223416_5.txt', 'JD Added_2010-06-01 Lee Memorial Amendment MU.txt', 'JD Added_2020-10-01 HCA WFD EO HOS CDM Amendment w Rate Sheets 10-1-20 Countersigned MU.txt', 'JD Added_2022-01-01 Orlando Health 42nd Amendment MU.txt', 'JD Added_2023-01-01 UF Health 48th Amend.Hospital.Archer Rehab.GNV HH Addendums Select and Exchange Networks with Checklist - signed MU.txt', 'Kidney_ICMProviderAgreement_ICMProviderAgreement_104832_5_MU.txt', 'Laboratory Corporation of America Holdings_2023.06.16.Fifth Amendment_OH MCD Comp Chg and TINs add_MU.txt', 'Laboratory Corporation of America Holdings_Second Amendment_20150101_MU.txt', 'MarionCounty_Caresource Exchange Hosp 110121 Partial_MP.txt', 'Mult1_Arizona Oncology_1 - 860938204 MU.txt', 'Mult5_22-3612265_ICMSupportingDocuments_553333_1 MU.txt', 'Muskogee_ICMProviderAgreement_ICMProviderAgreement_159776_5_MU.txt', 'OhioHealth Twenty- Sixth Amendment_20190101.txt', 'Omni Point Health_Physician Agreement MU.txt', 'Orange Co Radiation Oncology MU.txt', 'Physician - FQHC COMPLETE TEMPLATE_ MU.txt', 'Prof1_ABAN Care Contract - 562592781 MU.txt', 'Prof1_ICM135705_ICMProviderAgreement_135705 IPA of NJ MU.txt', 'Prof2_Black Canyon Medical_Malin Medical PLLC_2 - 811754238 MU .txt', 'Prof3_Arizona Children Association - 860096772 MU.txt', 'Prof4_West Yavapai Guidance Clinic Inc. 11.01.2021 Care1st Practitioner Amendment Template 1.22.2020.1_MU.txt', \"Progressive Women's Health, PLLC_Physician Agreement MU.txt\", 'Roussel Clement, MD_Physician Agreement MU (2).txt', 'Sierra Hematology & Oncology MU.txt', 'Southeast Georgia Health System Inc_20170517_Dually Executed MU.txt', \"TX - AGMT - HSA - Texas Children's Hospital - 74-1100555 -Executed eff 10152022 MU.txt\", 'TX - AMD 11 - HSA UTMB 74-6000949 executed MU.txt'])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input_dict_filtered.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "import preprocess\n", + "\n", + "filename, contract_text = list(input_dict.keys())[0], list(input_dict.values())[0]\n", + "text_dict = preprocess.split_text(contract_text)\n", "text_dict = preprocess.highlight_rates(text_dict)" ] }, { "cell_type": "code", - "execution_count": 56, - "metadata": {}, - "outputs": [], - "source": [ - "def BOTTOM_UP_CODES(d, page):\n", - " return f\"\"\"\n", - "### PAGE START ### {page} ### PAGE END\n", - "\n", - "The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\n", - "Service: {d['SERVICE']}\n", - "Methodology: {d['FULL_METHODOLOGY']}\n", - "\n", - "Your job is to identify codes associated with this specific term.\n", - "\n", - "Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\n", - "'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility.\n", - "'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\n", - "'REIMBURSEMENT_GROUPER_CODES' : 2- or 3-digit codes that may be labelled DRG or MS-DRG.\n", - "'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service.\n", - "'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\n", - "'REIMBURSEMENT_REVENUE_CODES' : 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes.\n", - "'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status.\n", - "\n", - "Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\n", - "\n", - "Only return the dictionary, with no other commentary or explanation.\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 57, - "metadata": {}, - "outputs": [], - "source": [ - "prompt = BOTTOM_UP_CODES(primary_dict[1], text_dict['3'])" - ] - }, - { - "cell_type": "code", - "execution_count": 58, + "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'\\n### PAGE START ### 3 Provider\\'s Name and Address Provider\\'s Tax ID Number (or UPIN) NPI Written referral, if applicable Clean Claim shall not include those claims which require Coordination of Benefits and Third Party Liability issues until receipt of Explanation of Benefits from primary carrier or claims, which are being reviewed by the Medical Director or Peer Review Committee for Medical Necessity. 1.3 \"CMS\" means the Centers for Medicare and Medicaid Services. 1.4 \"Covered Services\" means those medically necessary health care services covered under a Health Benefits Plan, as determined under the terms and conditions of the applicable Health Benefits Plan. 1.5 \"Downstream Entity\" means any party that enters into an acceptable written arrangement below the level of the arrangement between ODS and a first-tier entity. These written arrangements continue down to the level of the ultimate provider of health and/or administrative services. 1.6 \"Emergency Medical Condition\" means a medical condition manifesting itself by acute symptoms of sufficient severity (including severe pain) such that a prudent layperson, with an average knowledge of health and medicine, could reasonably expect the absence of immediate medical attention to result in (a) serious jeopardy to the health of the individual or, in the case of a pregnant woman, the health of a woman or her unborn child; (b) serious impairment to bodily functions; or (c) serious dysfunction of any bodily organ or part. 1.7 \"Emergency Services\" means covered inpatient and outpatient services furnished by a provider qualified to furnish emergency services; and needed to evaluate or stabilize an emergency medical condition. 1.8 \"First Tier Entity\" means any party that enters into a written agreement with ODS to provide administrative or health care services for a Medicare Advantage eligible individual. 1.9 \"Health Benefits Plan\" means a health benefits plan or a group health benefits plan, including individual or group health insurance policies, offering the services of approved health care Facilities and Providers participating in the ODS Medicare Advantage and Medicare Advantage PPO Plans-underwritten or administered by ODS and which describe the Covered Services, applicable co-payments (if any) and deductibles (if any) and other information pertinent to the provision of services. 1.10 \"Hospital\" means a facility licensed by the State of Oregon to provide inpatient and outpatient health care and who has entered into an agreement with ODS to provide services as described herein to Medicare Advantage Members. 1.11 \"Medical Emergency\" means a situation in which a \"medical emergency\" is present and when the Medicare Advantage Member reasonably believes that the Medicare Advantage Member\\'s health is in serious danger. A medical emergency includes severe pain, a bad injury, a serious illness, or a medical condition that is quickly getting much worse. 1.12 \"Medicare Advantage Member\" means a Medicare beneficiary entitled to receive coverage for certain health care services under the terms of the Medicare Advantage Evidence of Coverage, 2012 ODS Medicare Advantage Hospital Agreement Peace Health Southwest Medical Center 2 of 27 ### PAGE END\\n\\nThe above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\\nService: Pneumococcal and influenza vaccinations\\nMethodology: For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83\\n\\nYour job is to identify codes associated with this specific term.\\n\\nRead and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\\n\\'REIMBURSEMENT_ADMITTYPE_CODES\\' : Single-digit codes that indicate the type of admission to a facility.\\n\\'REIMBURSEMENT_DIAG_CODES\\' : Multi-digit alphanumeric codes related to diagnosis. They may contain a \\'.\\' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\\n\\'REIMBURSEMENT_GROUPER_CODES\\' : 2- or 3-digit codes that may be labelled DRG or MS-DRG.\\n\\'REIMBURSEMENT_PLACEOFSERVICE_CODES\\' : 2-digit codes related to the place of service.\\n\\'REIMBURSEMENT_PROC_CODES\\' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\\n\\'REIMBURSEMENT_REVENUE_CODES\\' : 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as \\'Rev\\' codes.\\n\\'REIMBURSEMENT_STATUS_INDICATOR_CODES\\' : 1- or 2-digit alphanumeric codes indicating the payment status.\\n\\nReturn a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\\n\\nOnly return the dictionary, with no other commentary or explanation.\\n'" + "'050540697 Deepak Nanda 2017 PHSP MU.txt'" ] }, - "execution_count": 58, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "prompt" - ] - }, - { - "cell_type": "code", - "execution_count": 59, - "metadata": {}, - "outputs": [], - "source": [ - "answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)" - ] - }, - { - "cell_type": "code", - "execution_count": 60, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "\"{\\n 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\\n 'REIMBURSEMENT_DIAG_CODES': 'N/A',\\n 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\\n 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\\n 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\\n 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\\n 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'\\n}\"" - ] - }, - "execution_count": 60, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "answer" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\n", - " 'REIMBURSEMENT_DIAG_CODES': 'N/A',\n", - " 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\n", - " 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\n", - " 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\n", - " 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\n", - " 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import json\n", - "\n", - "def primary_string_to_dict(string_dict):\n", - " data = []\n", - " pattern = r'\\{.*?\\}'\n", - " for page_num in string_dict.keys():\n", - " primary_list = string_dict[page_num]\n", - " dicts = primary_list.split('[')[1] # Strip front\n", - " dicts = dicts.split(']')[0] # Strip back\n", - " dicts = dicts.replace('\\n', '') # Remove new lines\n", - " dict_list = re.findall(pattern, dicts)\n", - " dict_list = [json.loads(d) for d in dict_list]\n", - " for dict_ in dict_list:\n", - " dict_['page_num'] = page_num\n", - " data.append(dict_)\n", - " return data\n", - "\n", - "def secondary_string_to_dict(dict_string):\n", - "# Find the start and end positions of the dictionary substring\n", - " start_index = dict_string.find('{')\n", - " end_index = dict_string.rfind('}') + 1\n", - "\n", - " # Extract the dictionary substring\n", - " dict_substring = dict_string[start_index:end_index]\n", - "\n", - " # Replace single quotes with double quotes to make it valid JSON\n", - " dict_substring = dict_substring.replace(\"'\", '\"')\n", - "\n", - " # Convert the substring to a dictionary\n", - " result_dict = json.loads(dict_substring)\n", - "\n", - " return result_dict\n", - "\n", - "\n", - "#primary_string_to_dict(primary_dict)\n", - "\n", - "secondary_dict = \"{\\n 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A',\\n 'REIMBURSEMENT_DIAG_CODES': 'N/A',\\n 'REIMBURSEMENT_GROUPER_CODES': 'N/A',\\n 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A',\\n 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'],\\n 'REIMBURSEMENT_REVENUE_CODES': 'N/A',\\n 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'\\n}\"\n", - "secondary_string_to_dict(secondary_dict)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'4': '[\\n {\\n \"SERVICE\": \"Primary Care Physician\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Primary Care Physician means a participating physician who is a family physician, nurse practitioner, physicians assistant, pediatrician or internist, and whose billings for primary care services are at least 50 percent of the physician\\'s total billings.\"\\n }\\n]',\n", - " '18': 'Here is the list of JSON objects with the relevant attributes extracted from the given text:\\n\\n[\\n {\\n \"SERVICE\": \"Professional Liability Insurance\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"1,000,000\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"During the term of this Agreement, Hospital shall maintain professional liability insurance in an amount not less than $1,000,000 per claim/$3,000,000 aggregate.\"\\n },\\n {\\n \"SERVICE\": \"General Liability Insurance\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"1,000,000\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"During the term of this Agreement, Hospital shall maintain general liability insurance in an amount not less than $1,000,000 per claim/$3,000,000 aggregate.\"\\n }\\n]',\n", - " '24': '[\\n {\\n \"SERVICE\": \"Hospital Inpatient Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"103%\",\\n \"FULL_METHODOLOGY\": \"103% of Medicare Allowance using the CMS Prospective Payment System in effect at the time of services provided.\"\\n },\\n {\\n \"SERVICE\": \"Hospital Outpatient Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\", \\n \"REIMBURSEMENT_RATE\": \"103%\",\\n \"FULL_METHODOLOGY\": \"103% of Medicare Allowance using the CMS Prospective Payment System in effect at the time of services provided.\"\\n },\\n {\\n \"SERVICE\": \"Surgery Secondary procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of the allowed amount for the procedure.\"\\n },\\n {\\n \"SERVICE\": \"Subsequent procedures after primary and secondary on same day\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of the allowed amount for those procedures.\"\\n },\\n {\\n \"SERVICE\": \"Primary procedure for some self-funded plans\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"100% of allowance\"\\n },\\n {\\n \"SERVICE\": \"Secondary procedure for some self-funded plans\", \\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"50% of allowance\"\\n },\\n {\\n \"SERVICE\": \"Remaining procedures after primary and secondary for some self-funded plans\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"25%\",\\n \"FULL_METHODOLOGY\": \"25% of allowance\"\\n }\\n]',\n", - " '25': '[\\n {\\n \"SERVICE\": \"Medicine\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Surgery\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Lab/Pathology\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Radiology\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Pharmaceuticals\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Anesthesia\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"DME\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"105%\",\\n \"FULL_METHODOLOGY\": \"For services billed on a CMS 1500 or successor form, the Fee Schedule will be set at one hundred five percent (105%) of the Medicare rates in place on January 1 of each year.\"\\n },\\n {\\n \"SERVICE\": \"Unlisted Procedures and/or Supplies\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\", \\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Allow at fifty percent (50%) of billed charges for medically necessary supplies or unlisted procedures (a procedure without a Relative Value Unit).\"\\n },\\n {\\n \"SERVICE\": \"Secondary procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Secondary procedures performed on the same day as primary procedure will be reimbursed at 50% of the allowed amount for that procedure.\"\\n },\\n {\\n \"SERVICE\": \"Subsequent procedures\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Any subsequent procedures performed on the same day as the primary procedure will be reimbursed at 50% of the allowed amount for those procedures.\"\\n },\\n {\\n \"SERVICE\": \"Primary procedure\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Secondary code\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"50%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Remaining codes\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"25%\",\\n \"FULL_METHODOLOGY\": \"Some self-funded plans will continue to consider the primary procedure at 100% of allowance, the secondary code at 50% of allowance, and the remaining codes at 25% of allowance.\"\\n },\\n {\\n \"SERVICE\": \"Ambulatory Services\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"250.00\",\\n \"REIMBURSEMENT_RATE\": \"N/A\",\\n \"FULL_METHODOLOGY\": \"Ambulatory services administered to patient will be reimbursed at a rate of $250.00 per diem.\"\\n }\\n]',\n", - " '28': '[\\n {\\n \"SERVICE\": \"Home Health\",\\n \"REIMBURSEMENT_FLAT_FEE\": \"N/A\",\\n \"REIMBURSEMENT_RATE\": \"100%\",\\n \"FULL_METHODOLOGY\": \"ODS will reimburse the facility according to the one hundred percent >>>(100%)<<< of current year CMS Home Health Prospective Payment System (PPS) for each specific Home Health Facility.\"\\n }\\n]'}" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "primary_dict" + "filename" ] }, { @@ -306,7 +177,14 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "d = \"\"\"{\n", + " \"REIMBRUSEMENT_EXCEPTION_IND\": \"Y\",\n", + " \"REIMBURSEMENT_DESCRIBE_EXCEPTION\": \"any Change in Control (as defined in clauses (i) through (iv) of this Section (9.11) with respect to any entity that owns 50% or more of Provider's stock or other ownership interests\",\n", + " \"RATE_ESCALATOR_IND\": \"N\"\n", + "}\"\"\"\n", + "\n" + ] } ], "metadata": { diff --git a/src/utils.py b/src/utils.py index d736fc0..42388ae 100644 --- a/src/utils.py +++ b/src/utils.py @@ -39,39 +39,39 @@ def read_s3(): files[filename] = context return files -def read_input(mode=config.READ_MODE): +def read_input(path=config.LOCAL_PATH, mode=config.READ_MODE): if mode == '_LOCAL_': files = {} - for file in os.listdir(config.LOCAL_PATH): - full_path = os.path.join(config.LOCAL_PATH, file) + for file in os.listdir(path): + full_path = os.path.join(path, file) file_text = read_local(full_path) files[file] = file_text return files elif mode == '_S3_': return read_s3() -def consolidate_individual(folder): +def consolidate_individual(input_folder='results', output_folder='output'): dfs = [] - for filename in os.listdir(folder): + for filename in os.listdir(input_folder): if filename.endswith('.csv'): - filepath = os.path.join(folder, filename) + filepath = os.path.join(input_folder, filename) dfs.append(pd.read_csv(filepath)) # Remove temp folder - if folder=='temp' and os.path.exists(folder): - shutil.rmtree(folder) + if input_folder=='temp' and os.path.exists(input_folder): + shutil.rmtree(input_folder) consolidated_df = pd.concat(dfs, ignore_index=True) # Write output version = 1 - existing_files = [filename for filename in os.listdir('results') if filename.startswith(f'consolidated_results_{config.TODAY}')] + existing_files = [filename for filename in os.listdir(output_folder) if filename.startswith(f'consolidated_results_{config.TODAY}')] if existing_files: versions = [int(file.split('_v')[1].split('.')[0]) for file in existing_files if '_v' in file] if versions: version = max(versions) + 1 filename = f'consolidated_results_{config.TODAY}_v{version}.csv' - consolidated_df.to_csv(os.path.join('results', filename)) + consolidated_df.to_csv(os.path.join(output_folder, filename)) From 3b6c92285fb33116cc2f69c51be03c9f4a1e60dc Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 3 May 2024 09:48:13 -0700 Subject: [PATCH 12/78] Minor prompt modifications --- src/prompts.py | 41 +---------------------------------------- src/test.ipynb | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/src/prompts.py b/src/prompts.py index ed5c147..d0a9a2d 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -1,43 +1,4 @@ -# def BOTTOM_UP_PRIMARY(page, payer): -# return f""" -# ### PAGE START ### {page} ### PAGE END - -# The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. - -# Analyze the page text. First, identify if the page contains information related to reimbursement for certain services or specialties. If yes, return a list of json-formatted dictionaries with the attributes. If no, return just an empty list ('[]'), nothing more. - -# If any of the above attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. - -# Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. -# It is also possible (and likely) that there will be multiple reimbursement values corresponding to the same specialty/provider type. When that is the case, they must be returned in a separate dictionary in the list. - -# Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: - -# 'IS_CARVEOUT' : A Primary Reimbursement is a generalized rate for which a specialty or service is reimbursed. A Carveout reimbursement is a specific rate that is different or more detailed than the primary service. If the rate is a carveout, write 'Y'. If not, write, 'N'. -# 'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be either Medicaid, Medicare, or Marketplace. If a PLAN is found, this field is optional. -# 'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. -# 'PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. -# 'PROV_TYPE' : What is the type of provider in agreement to be reimbursed. This might not be listed separately for each occurrence and instead be shown at the top of the page or earlier in the text. It could be 'Provider', 'Provider Group', or other types of doctors and medical entities. Use your best judgement. -# 'PROV_SPECIALTY' : What is the Specialty or Service that is being reimbursed. This could be something general, like 'Professional Services', 'Physician Services, or similar. It could also be something more specific, like 'Anesthesiology', 'DME', or similar. Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. -# 'APPLICABLE_TO' : What type or types of patients are the reimbursement terms applicable for? This will NOT be a Plan or Program. -# 'GREATER_OF_LANGUAGE_IND' : If the listed reimbursement is the greater of two or more values, return 'Y'. Otherwise, return 'N'. -# 'LESSER_OF_LANGUAGE_IND': If the listed reimbursement is the lesser of two or more values, return 'Y'. Otherwise, return 'N'. -# 'REIMBURSEMENT_METHODOLOGY' : What is the method of reimbursement? This might be something like 'Medicare Fee Schedule' or 'Medicaid Fee Schedule'. It might also be 'Billed Charges', 'Per Diem', 'Per Case', or similar. -# 'REIMBURSEMENT_FEE_SCHEDULE' : If the reimbursement is based on a Fee Schedule, return the name of the exact fee schedule on which it is based. -# 'REIMBURSEMENT_CODES' : List any codes to which the reimbursement applies. Codes are typically 3-6 uppercase alphanumeric characters. Return a list or a range, if applicable. -# 'REIMBURSEMENT_RATE_ESCALATOR_IND' : If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. -# 'REIMBURSEMENT_EXCEPTION_IND' : If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, return 'N'. This could also be presented as a conditional - the reimbursement value is only valid if some condition is met. -# 'REIMBURSEMENT_GROUPER' : If the listed reimbursement is based on a grouper rate, provide the applicable grouper methodology. This may be something like 'DRG, 'MSDRG', 'APC' or similar. -# 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. -# 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. - -# When LESSER_OF_LANGUAGE_IND = 'Y' OR GREATER_OF_LANGUAGE_IND = 'Y', that means that there must be TWO dictionaries in the list, one for each methodology and rate that the reimbursement is the lesser/greater of. -# As an example, if you see that a reimbursement says something like "specialty will be paid as the lesser of billed charges or 100% of the medicare fee schedule", then there needs to be TWO dictionary objects with the same specialty, one of which has REIMBURSEMENT_METHODOLOGY='Billed Charges' and the other of which has REIMBURSEMENT_METHODOLOGY='Medicare Fee Schedule'. Do not forget this step. - -# Only return the list, with no other commentary or explanation. -# """ - def BOTTOM_UP_PRIMARY(page, payer): return f""" @@ -115,7 +76,7 @@ Methodology: {d['FULL_METHODOLOGY']} Your job is to identify if the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations) or an escalation Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): -REIMBRUSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N' +REIMBURSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N' REIMBURSEMENT_DESCRIBE_EXCEPTION: If the listed reimbursement contains some sort of exception, describe it here. Try to use exact words from the text only. If there is none, just write 'N/A' RATE_ESCALATOR_IND: If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. diff --git a/src/test.ipynb b/src/test.ipynb index 9329195..d56a14c 100644 --- a/src/test.ipynb +++ b/src/test.ipynb @@ -174,17 +174,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'REIMBRUSEMENT_EXCEPTION_IND': 'Y',\n", + " 'REIMBURSEMENT_DESCRIBE_EXCEPTION': \"any Change in Control (as defined in clauses (i) through (iv) of this Section (9.11) with respect to any entity that owns 50% or more of Provider's stock or other ownership interests\",\n", + " 'RATE_ESCALATOR_IND': 'N'}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ + "import json\n", + "\n", "d = \"\"\"{\n", " \"REIMBRUSEMENT_EXCEPTION_IND\": \"Y\",\n", " \"REIMBURSEMENT_DESCRIBE_EXCEPTION\": \"any Change in Control (as defined in clauses (i) through (iv) of this Section (9.11) with respect to any entity that owns 50% or more of Provider's stock or other ownership interests\",\n", " \"RATE_ESCALATOR_IND\": \"N\"\n", "}\"\"\"\n", - "\n" + "\n", + "json.loads(d)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From f0f5f0ad345ed49e1779aaf6fe743dea0bbb486f Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 6 May 2024 14:26:48 -0700 Subject: [PATCH 13/78] Prompt modifications --- src/prompts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/prompts.py b/src/prompts.py index d0a9a2d..8cd8061 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -6,7 +6,7 @@ def BOTTOM_UP_PRIMARY(page, payer): The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen. +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. @@ -104,9 +104,10 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Billed Charges, Per X (where X is some criteria like Visit, Hour, Unit, Diem, etc) +Allowed Amount, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. +For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. Return only the answer, without using complete sentences. If there is nothing even close to the above options in the full methodology text, simply return 'N/A' """ From b4a50e52d93772118b26ed3171d241f5085ebc64 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 7 May 2024 11:36:32 -0700 Subject: [PATCH 14/78] Realign tables in preprocessing --- src/bottom_up_funcs.py | 2 + src/config.py | 5 +- src/preprocess.py | 16 ++ src/test.ipynb | 335 +++++++++++++++++++++-------------------- 4 files changed, 192 insertions(+), 166 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 874fe5c..3205d1c 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -108,7 +108,9 @@ def bottom_up(file_object): filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") + contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) + text_dict = preprocess.align_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) #### PROMPT #### diff --git a/src/config.py b/src/config.py index 2494200..ee57efd 100644 --- a/src/config.py +++ b/src/config.py @@ -17,12 +17,9 @@ OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' MAX_WORKERS = 20 # AWS Keys -AWS_ACCESS_KEY_ID="update here" -AWS_SECRET_ACCESS_KEY="update here" -AWS_SESSION_TOKEN="update here" # File Paths -LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local +LOCAL_PATH = 'docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/preprocess.py b/src/preprocess.py index 47d30aa..8ae48d7 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -1,11 +1,27 @@ from itertools import groupby, count import re +def clean_newlines(contract_text): + cleaned_text = re.sub(r'(?>>THIS IS PRETABLE TEXT>>>\\n IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above.\\n<<<<<<{\\'Managed Health, Inc.\\': [\\'By:\\', \\'Name: Print Name\\', \\'Title:\\', \\'Date:\\'], \\'Provider: Chinese American Independent Practice Association, Inc.\\': [\\'By:\\', \\'Name: Print Name\\', \\'Title:\\', \\'Date:\\']}\\n-------Table End--------\\n\\nStart of Page No. = 4\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT C\\nCompensation Schedule\\nReimbursement to IPA Providers\\nI.\\nMedicare and Commercial Enrollees\\n1.\\nExcept for the Covered Services set forth below, MHI shall directly reimburse IPA Providers\\nwho provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum\\nallowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\")\\nfor payments under the Medicare program.\\n2.\\nMHI shall reimburse IPA Providers for the following Covered Services as follows:\\na. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a\\nmulti-specialty office or facility. If provided by an individually-operated Physical Therapist office\\n(solely physical therapy services), physical therapy services set forth below shall be reimbursed at a\\nrate of $65 per service. These services, which shall be updated from time to time, shall include:\\nb. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a\\nfee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee\\nschedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program, except for physical therapy services which will be reimbursed as\\nnoted in \"a.\" above.\\nC. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office\\n(as mandated by the New York State Department of Health), will be reimbursed on a fee-for-\\nservice basis in an amount equal to 100% of the maximum allowable Region I fee schedule\\nestablished by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the\\nMedicare program.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include:\\n<<<{\\'Code Range\\': [\\'97001-97006\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program.\\n<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsies\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 5\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following preventive care services will be reimbursed at the rates noted below:\\ne.\\nHearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent\\n(80%) of the manufacturer\\'s suggested retail price.\\nII.\\nWhere rates in any fee schedule described in this Agreement make reference to Medicare or\\nMedicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and\\nMedicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates,\\nshall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon\\nforty five calendar days following the date on which CMS or SDOH publishes such rates.\\nIPA Network Access and Administrative Fees.\\nFor Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month\\nfor the following administrative services,:\\nAccess to IPA\\'s network of Participating IPA Providers\\nCredentialing of Participating IPA Providers\\nReporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst\\nAdministrative services relating to quality improvement and Enrollee satisfaction\\nEducation of Participating IPA Providers regarding Healthfirst\\'s policies and procedures and\\nquality improvement programs.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following preventive care services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code\\': [\\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 6\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Sharing with IPA\\n1\\nDefinitions. For the purposes of this Agreement and Compensation Schedule, the following\\nterms shall have the indicated meanings:\\n1.1\\n\"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees,\\nadjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such\\nadjustment is reasonably applied by Healthfirst.\\n1.2\\n\"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable\\nreserve for incurred but not received (\"IBNR\") claims for Risk Services.\\n1.3\\n\"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target\\nfollowing Reconciliation.\\n1.4\\n\"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target.\\n1.5\\n\"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine\\nSurpluses and Deficits.\\n1.6\\n\"Risk Enrollee\" shall mean Enrollees in Healthfirst\\'s Medicare Advantage plans who\\nselect an IPA Provider as their Primary Care Provider.\\n1.7\\n\"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of\\nwhether such Services were provided by IPA Providers or other health care providers.\\n1.8\\n\"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums\\n1.9\\n\"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target\\nfollowing Reconciliation.\\n1.10\\n\"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target.\\n2\\nRisk Sharing.\\n2.1\\nRisk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk\\nServices rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk\\nServices provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall\\nassume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst\\na portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration\\nof Healthfirst\\'s utilization and case management programs, the sharing of utilization data, and education\\nof IPA Providers regarding utilization trends and the provision of quality and medically appropriate\\nservices.\\n2.2\\nLimited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services\\nother than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 7\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nbe required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or\\nhealth care provider. In all instances, including any termination or expiration of this Agreement,\\nHealthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk\\nServices, to IPA Providers and other health care providers.\\n3\\nRegulatory Approval.\\nThe parties understand and acknowledge that this risk sharing\\narrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency\\nhaving jurisdiction over this Agreement as required by applicable statutes and regulation. No payments\\nshall be made by either party, prior to receiving such regulatory approval. The failure to obtain\\nregulatory shall not be grounds for either party to terminate this Agreement which shall otherwise\\nremain in effect.\\n4\\nQuarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each\\ncalendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding\\nquarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the\\nmeasurement period shall commence on the later of a) January 1, 2010 or b) the date of any required\\nregulatory approval.\\n5\\nRisk Sharing; Distribution of Surplus and Payment of Deficits.\\n5.1\\nSurpluses. In the event that Healthfirst determines that a Surplus exists following\\nReconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal\\nto the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Upside Risk Limit.\\n5.2\\nDeficits.\\n5.2.1 In the event that Healthfirst determines that a Deficit exists following\\nReconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal\\nto the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Downside Risk Limit.\\n5.2.2\\nIn the event that IPA fails to timely pay any amount due under Section 5.2.1, any\\nsuch amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due\\nto IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against\\nfuture amounts due to IPA in the succeeding calendar year quarter.\\n5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless\\nof the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such\\ntermination, expiration or non-renewal.\\n6\\nCompliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may\\nuse any compensation methodology to reimburse IPA Providers for Health Care Services; provided,\\nhowever, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or\\namount, either separately or in combination with any compensation formulas which would place IPA\\nProviders at substantial financial risk for the provision of referral services, as determined by the\\napplicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 8\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts\\npaid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA\\'s compliance with the\\nfederal PIP regulations.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 9\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nSecond Amendment to the Agreement\\nby and between\\nHealthfirst PHSP, Inc. and Independent Practice Association\\nThis Amendment shall be effective January 1, 2015 by and between Chinese American\\nIndependent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\").\\nWITNESSETH:\\nWHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of\\nCovered Services to certain Healthfirst Enrollees; and\\nWHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation\\nwhich Healthfirst pays to IPA.\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and IPA hereby agree to amend the Agreement as follows:\\n1.\\nSection 2.1 of the Agreement titled Provision of Health Care Services is deleted in its\\nentirety and replaced Section 2.1.\\n2.1\\nProvision of Health Care Services. IPA shall arrange for IPA Providers to provide Health\\nCare Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the\\napplicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer\\nparticipation with Healthfirst pursuant to this Agreement to any health care provider who is a\\nmember of IPA or has a contract with IPA, nor disclose share the terms and conditions of this\\nAgreement included rates of reimbursement, without first obtaining the written consent of\\nHealthfirst. Healthfirst shall have the right to refuse the participation of any particular health care\\nprovider under this Agreement. Healthfirst\\'s determination to offer participation to any health care\\nprovider pursuant to this Agreement or to refuse to accept any health care provider for participation\\nshall be based on Healthfirst\\'s business needs as determined solely by Healthfirst. Healthfirst\\'s\\nVice-President, Network Management or Executive Vice-President/Chief Operating Officer shall\\nhave sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the\\nname, addresses, telephone numbers and other information regarding IPA Providers reasonably\\nrequested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use\\nsuch information in Healthfirst advertising and provider directories.\\n2.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n1 of 10\\n\\nStart of Page No. = 10\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective Date or at the time of Healthfirst\\'s final approval of the credentialing\\nprocess, whichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n2 of 10\\n\\nStart of Page No. = 11\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted\\nor\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n3\\nA new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA\\nProviders for the provision of Health Care Services to Enrollees and IPA for the provision of certain\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n3 of 10\\n\\nStart of Page No. = 12\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nadministrative services, a copy of which is attached to this Amendment, is added to the Agreement.\\nThe reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of\\nservice on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service\\nprior to that date.\\n4\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n4 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n 4 All other terms and conditions of the Agreement shall remain in full force and effect.\\n<<<<<<{\\'Healthfirst PHSP, Inc.\\': [\\'By: Name: Print Name\\', \\'Title:\\', \\'Date: -\\'], \\'Chinese American Independent Practice Association, Inc.\\': [\\'By: Name: Print Name\\', \\'Title:\\', \\'Date:\\']}\\n-------Table End--------\\n\\nStart of Page No. = 13\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT 4.1\\nCOMPENSATION SCHEDULE\\nReimbursement to IPA Providers\\n1.\\nExcept for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA\\nProviders who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the\\nmaximum allowable Region I fee schedule established by the Center for Medicaid and Medicare\\nServices (\"CMS\") for payments under the Medicare program.\\n2.\\nHealthfirst shall reimburse IPA Providers for the Health Care Services set forth below as\\nfollows:\\na. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of\\n$45 per service if provided in a multi-specialty office or facility. If provided by an individually-\\noperated Physical Therapist office (solely physical therapy services), physical therapy services set\\nforth below shall be reimbursed at a rate of $60 per service.\\nb.\\nPhysical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall\\nbe reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable\\nRegion I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for\\npayments under the Medicare program, except for physical therapy services which will be\\nreimbursed as noted in \"a.\" above.\\nC.\\nGastroenterology Services. The following services, when rendered in an \"Office Based Surgery\"\\n(OBS) accredited office (as mandated by the New York State Department of Health), will be\\nreimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region\\nI fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n5 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'Code Range\\': [\\'97001-97004\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsy\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 14\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%)\\nof the manufacturer\\'s suggested retail price.\\nIPA Network Access and Administrative Fees.\\nHealthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative\\nservices relating to quality improvement and education of Participating IPA Providers regarding\\nHealthfirst\\'s quality improvement programs. IPA and Healthfirst understand and agree that IPA shall\\nnot provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant\\nto this Agreement. In the event that IPA or an affiliated of IPA provides management services, the\\nparties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n6 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code/Modifier\\': [\\'99205-P1\\', \\'99212-P2\\', \\'99215-P3\\', \\'59409\\', \\'59514\\', \\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'PCAP Initial Visit\\', \\'PCAP Prenatal Visit\\', \\'PCAP Post partum Visit\\', \\'Vaginal Delivery\\', \\'Cesarean Delivery\\', \\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$196.07\\', \\'$130.00\\', \\'$163.00\\', \\'$1500.00\\', \\'$1500.00\\', \\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 15\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nAdditional Compensation to IPA in the form of Surplus Distribution.\\n1.\\nLimited Risk Transfer/No Transfer of Financial Risk\\n1.1.\\nIPA shall be entitled to Additional Compensation under this Agreement on an annual\\nbasis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus\\nfunds exist, as specified below.\\n1.2.\\nUse of Additional Compensation. IPA represents and warrants that Additional\\nCompensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a\\nmanner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On\\nrequest by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds\\nreceived from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d).\\n2.\\nDefinitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this\\nExhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the\\nAgreement to which this Exhibit is attached.\\n2.1.\\n\"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant\\nto Section 5 of this Exhibit 4.1(b).\\n2.2.\\n\"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk\\nEnrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan\\nContract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst.\\n2.3.\\n\"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to\\nHealthfirst\\'s administrative costs as reasonably determined by Healthfirst.\\n2.4.\\n\"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of\\nMedical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or\\nDeficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit.\\n2.5.\\n\"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the\\nAdministrative Deduction and any Quality Incentive Deduction.\\n2.6.\\n\"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for\\nprograms in which IPA and IPA Providers participate to improve the quality of Health Care Services\\nwhich Risk Enrollees receive.\\n2.7.\\n\"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical\\nRevenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits.\\n2.8.\\n\"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary\\ncare provider who is a Participating IPA Provider.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n7 of 10\\n\\nStart of Page No. = 16\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.9.\\n\"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following\\nReconciliation\\n3.\\nReconciliation of Medical Revenue to Determine Surpluses and Deficits. During each\\nReconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine\\nthe amount of any Surplus of Deficit:\\n3.1.\\nthe costs of claims paid for Health Care Services rendered to Risk Enrollees by\\nParticipating IPA Providers, Participating Providers and all other health care providers.\\n3.2.\\na\\nreasonable reserve for the cost of claims for Health Care Services rendered to Risk\\nEnrollees but not yet received;\\n3.3.\\nother costs associated with the provision of Health Care Services to Risk Enrollees\\nincluding but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs.\\n3.4.\\naccruals for any risk adjustment applicable to Adjusted Premiums.\\n4.\\nReconciliation Schedule.\\n4.1.\\nInterim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days\\n(120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus\\nor Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines\\nthat a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall\\nmake an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the\\nevent that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the\\ncalendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the\\nDeficit shall be handled as set forth in Section 6 below.\\n4.2.\\nAnnual Reconciliation and Distribution. Within one-hundred and twenty days (120) after\\nthe end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for\\nthat calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that\\ncalendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in\\nSection 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that\\nCalendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit\\nshall be handled as set forth in Section 6 below.\\n5.\\nPayment of Additional Compensation. In the event that Healthfirst determines that a Surplus has\\nbeen achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which\\nshall be the lesser of the following:\\n5.1.\\nTen percent (10%) of the Surplus for the calendar year, or\\n5.2.\\nAn amount such that the Additional Compensation equals twenty-five percent (25%) of\\nthe sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n8 of 10\\n\\nStart of Page No. = 17\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and\\niii) the Additional Compensation.\\n5.3.\\nHealthfirst shall pay any Additional Compensation, within thirty days of each\\nReconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation.\\n6.\\nTreatment of Deficits.\\n6.1.\\nDeficits Rolled Forward. In the event that a Deficit remains following any Reconciliation\\nor a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be\\nrolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar\\nyear.\\n6.2.\\nDeficits at Termination, Expiration or Non-Renewal. In the event that this Agreement\\nterminates, is not renewed or expires, regardless of the reason, Provider shall not be required to\\nreimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or\\nnon-renewal.\\n7.\\nCompliance with Regulatory Provisions Regarding Risk Transfer\\n7.1.\\nSDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with\\nthe\\napplicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH\\'s\\nProvider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount\\nfinancial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting\\nGuidelines, IPA shall demonstrate financial viability and establish a financial security deposit as\\nrequired by SDOH.\\n7.2.\\nPhysician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the\\napplicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the\\nplacement of stop loss insurance if applicable.\\n7.3.\\nFinancial Stability Monitoring and Reporting. Notwithstanding any other provision of\\nthis Agreement Healthfirst shall regularly monitor Provider\\'s financial condition to ensure that Provider\\nremains financially able to perform its obligations under this Agreement and meet any financial\\nsolvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material\\ndevelopments which may impair its ability to perform its obligations under this Agreement. In addition,\\nIPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e),\\nwhich shall include but not be limited to IPA\\'s quarterly and annual financial statements. IPA shall also\\nmaintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received\\nfrom Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and\\ndisbursements to Healthfirst on request.\\n8.\\nTermination or Amendment of Additional Compensation\\nHealthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to\\nwhich Additional Compensation is determined and paid to IPA as well as eliminate the payment of\\nAdditional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n9 of 10\\n\\nStart of Page No. = 18\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nthe Agreement such amendment or termination shall not change any Additional Compensation\\ncalculated or paid prior to the effective date of such amendment or termination.\\n9.\\nTermination, Non-Renewal or Expiration of the Agreement.\\nFor a period of twelve months following any termination, non-renewal or expiration of this Agreement,\\nregardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this\\nAgreement until the Final Reconciliation; provided, however, that payments to IPA of Additional\\nCompensation shall be suspended. These Reconciliations shall be on based Health Care Services\\nrendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement.\\nHealthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty\\ndays of the Final Reconciliation.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n10 of 10\\n\\nStart of Page No. = 19\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nFifth Amendment to the Agreement by and between\\nManaged Health, Inc. and Independent Practice Association\\nThis Amendment to the Agreement between Managed Health, Inc. (\"Healthfirst\"), and Chinese\\nAmerican Independent Practice Association, Inc. (\"Provider\") is effective January 1, 2015.\\nWITNESSETH:\\nWHEREAS, Healthfirst and Provider have entered into an Agreement (the \"Agreement\") for the\\nprovision of Health Care Services to Enrollees; and\\nWHEREAS, Healthfirst and Provider desire to amend that Agreement to delegate credentialing\\nto Provider :\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and Provider hereby agree to amend the Agreement as follows:\\n1.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective or at the time of Healthfirst\\'s final approval of the credentialing process,\\nwhichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 20\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 21\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n2.\\nA new Exhibit C setting forth the reimbursement to be paid by MHI to IPA and IPA\\nProviders for the provision of Covered Services to Enrollees, a copy of which is attached to this\\nAmendment, added to the Agreement. The reimbursement set forth in the Exhibit C attached to this\\namendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits C and the Letter\\nof Agreement effective July 1, 2006 shall continue to apply to dates of service prior to that date. This\\nAmendment supersedes the July 1, 2006 Letter of Agreement setting forth an administrative fee.\\n3.\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nIN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective\\nDate above.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above.\\n<<<<<<{\\'Managed Health, Inc.\\': [\\'DocuSigned by: By: Paul E. Portsmore\\', \\'441CEAD8E502476 Name: Paul E. Portsmore Print Name\\', \\'Title: SVP\\', \\'Date: 12/31/2014 18:32:28 ET\\'], \\'Provider: Chinese American Independent Practice Association, Inc.\\': [\\'By: family\\', \\'Name: PEGGY SHENG Print Name\\', \\'Chief Operating Officer Title:\\', \\'Date: Dec. 26, 2015\\']}\\n-------Table End--------\\n\\nStart of Page No. = 22\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT C\\nCompensation Schedule\\nReimbursement to IPA Providers\\nI.\\nMedicare and Commercial Enrollees\\n1.\\nExcept for the Covered Services set forth below, MHI shall directly reimburse IPA Providers\\nwho provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum\\nallowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\")\\nfor payments under the Medicare program.\\n2.\\nMHI shall reimburse IPA Providers for the following Covered Services as follows:\\na. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a\\nmulti-specialty office or facility. If provided by an individually-operated Physical Therapist office\\n(solely physical therapy services), physical therapy services set forth below shall be reimbursed at a\\nrate of $65 per service. These services, which shall be updated from time to time, shall include:\\nb. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a\\nfee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee\\nschedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program, except for physical therapy services which will be reimbursed as\\nnoted in \"a.\" above.\\nC. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office\\n(as mandated by the New York State Department of Health), will be reimbursed on a fee-for-\\nservice basis in an amount equal to 100% of the maximum allowable Region I fee schedule\\nestablished by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the\\nMedicare program.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include:\\n<<<{\\'Code Range\\': [\\'97001-97006\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program.\\n<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsies\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 23\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following preventive care services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent\\n(80%) of the manufacturer\\'s suggested retail price.\\nII.\\nWhere rates in any fee schedule described in this Agreement make reference to Medicare or\\nMedicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and\\nMedicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates,\\nshall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon\\nforty five calendar days following the date on which CMS or SDOH publishes such rates.\\nIPA Network Access and Administrative Fees.\\nFor Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month\\nfor the following administrative services,:\\nAccess to IPA\\'s network of Participating IPA Providers\\nCredentialing of Participating IPA Providers\\nReporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst\\nAdministrative services relating to quality improvement and Enrollee satisfaction\\nEducation of Participating IPA Providers regarding Healthfirst\\'s policies and procedures and\\nquality improvement programs.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following preventive care services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code\\': [\\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 24\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Sharing with IPA\\n1\\nDefinitions. For the purposes of this Agreement and Compensation Schedule, the following\\nterms shall have the indicated meanings:\\n1.1\\n\"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees,\\nadjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such\\nadjustment is reasonably applied by Healthfirst.\\n1.2\\n\"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable\\nreserve for incurred but not received (\"IBNR\") claims for Risk Services.\\n1.3\\n\"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target\\nfollowing Reconciliation.\\n1.4\\n\"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target.\\n1.5\\n\"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine\\nSurpluses and Deficits.\\n1.6\\n\"Risk Enrollee\" shall mean Enrollees in Healthfirst\\'s Medicare Advantage plans who\\nselect an IPA Provider as their Primary Care Provider.\\n1.7\\n\"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of\\nwhether such Services were provided by IPA Providers or other health care providers.\\n1.8\\n\"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums\\n1.9\\n\"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target\\nfollowing Reconciliation.\\n1.10\\n\"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target.\\n2\\nRisk Sharing.\\n2.1\\nRisk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk\\nServices rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk\\nServices provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall\\nassume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst\\na portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration\\nof Healthfirst\\'s utilization and case management programs, the sharing of utilization data, and education\\nof IPA Providers regarding utilization trends and the provision of quality and medically appropriate\\nservices.\\n2.2\\nLimited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services\\nother than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 25\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nbe required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or\\nhealth care provider. In all instances, including any termination or expiration of this Agreement,\\nHealthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk\\nServices, to IPA Providers and other health care providers.\\n3\\nRegulatory Approval.\\nThe parties understand and acknowledge that this risk sharing\\narrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency\\nhaving jurisdiction over this Agreement as required by applicable statutes and regulation. No payments\\nshall be made by either party, prior to receiving such regulatory approval. The failure to obtain\\nregulatory shall not be grounds for either party to terminate this Agreement which shall otherwise\\nremain in effect.\\n4\\nQuarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each\\ncalendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding\\nquarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the\\nmeasurement period shall commence on the later of a) January 1, 2010 or b) the date of any required\\nregulatory approval.\\n5\\nRisk Sharing; Distribution of Surplus and Payment of Deficits.\\n5.1\\nSurpluses. In the event that Healthfirst determines that a Surplus exists following\\nReconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal\\nto the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Upside Risk Limit.\\n5.2\\nDeficits.\\n5.2.1 In the event that Healthfirst determines that a Deficit exists following\\nReconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal\\nto the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Downside Risk Limit.\\n5.2.2\\nIn the event that IPA fails to timely pay any amount due under Section 5.2.1, any\\nsuch amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due\\nto IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against\\nfuture amounts due to IPA in the succeeding calendar year quarter.\\n5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless\\nof the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such\\ntermination, expiration or non-renewal.\\n6\\nCompliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may\\nuse any compensation methodology to reimburse IPA Providers for Health Care Services; provided,\\nhowever, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or\\namount, either separately or in combination with any compensation formulas which would place IPA\\nProviders at substantial financial risk for the provision of referral services, as determined by the\\napplicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 26\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts\\npaid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA\\'s compliance with the\\nfederal PIP regulations.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 27\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nSecond Amendment to the Agreement\\nby and between\\nHealthfirst PHSP, Inc. and Independent Practice Association\\nThis Amendment shall be effective January 1, 2015 by and between Chinese American\\nIndependent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\").\\nWITNESSETH:\\nWHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of\\nCovered Services to certain Healthfirst Enrollees; and\\nWHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation\\nwhich Healthfirst pays to IPA.\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and IPA hereby agree to amend the Agreement as follows:\\n1.\\nSection 2.1 of the Agreement titled Provision of Health Care Services is deleted in its\\nentirety and replaced Section 2.1.\\n2.1\\nProvision of Health Care Services. IPA shall arrange for IPA Providers to provide Health\\nCare Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the\\napplicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer\\nparticipation with Healthfirst pursuant to this Agreement to any health care provider who is a\\nmember of IPA or has a contract with IPA, nor disclose share the terms and conditions of this\\nAgreement included rates of reimbursement, without first obtaining the written consent of\\nHealthfirst. Healthfirst shall have the right to refuse the participation of any particular health care\\nprovider under this Agreement. Healthfirst\\'s determination to offer participation to any health care\\nprovider pursuant to this Agreement or to refuse to accept any health care provider for participation\\nshall be based on Healthfirst\\'s business needs as determined solely by Healthfirst. Healthfirst\\'s\\nVice-President, Network Management or Executive Vice-President/Chief Operating Officer shall\\nhave sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the\\nname, addresses, telephone numbers and other information regarding IPA Providers reasonably\\nrequested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use\\nsuch information in Healthfirst advertising and provider directories.\\n2.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n1 of 10\\n\\nStart of Page No. = 28\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective Date or at the time of Healthfirst\\'s final approval of the credentialing\\nprocess, whichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n2 of 10\\n\\nStart of Page No. = 29\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted\\nor\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n3\\nA new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA\\nProviders for the provision of Health Care Services to Enrollees and IPA for the provision of certain\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n3 of 10\\n\\nStart of Page No. = 30\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nadministrative services, a copy of which is attached to this Amendment, is added to the Agreement.\\nThe reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of\\nservice on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service\\nprior to that date.\\n4\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n4 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n 4 All other terms and conditions of the Agreement shall remain in full force and effect.\\n<<<<<<{\\'Healthfirst PHSP, Inc.\\': [\\'DocuSigned by: By: Paul E. Portsmore\\', \\'441CEAD8E502476. Name: Paul E. Portsmore Print Name\\', \\'Title: SVP -\\', \\'Date: 12/31/2014 I 18:32:28 ET -\\'], \\'Chinese American Independent Practice Association, Inc.\\': [\\'By: f Sear\\', \\'Name: PEGGY SHENG Print Name\\', \\'Title: Chief Operating Officer\\', \\'Date: December 26, 2014\\']}\\n-------Table End--------\\n\\nStart of Page No. = 31\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT 4.1\\nCOMPENSATION SCHEDULE\\nReimbursement to IPA Providers\\n1.\\nExcept for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA\\nProviders who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the\\nmaximum allowable Region I fee schedule established by the Center for Medicaid and Medicare\\nServices (\"CMS\") for payments under the Medicare program.\\n2.\\nHealthfirst shall reimburse IPA Providers for the Health Care Services set forth below as\\nfollows:\\na. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of\\n$45 per service if provided in a multi-specialty office or facility. If provided by an individually-\\noperated Physical Therapist office (solely physical therapy services), physical therapy services set\\nforth below shall be reimbursed at a rate of $60 per service.\\nb.\\nPhysical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall\\nbe reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable\\nRegion I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for\\npayments under the Medicare program, except for physical therapy services which will be\\nreimbursed as noted in \"a.\" above.\\nC.\\nGastroenterology Services. The following services, when rendered in an \"Office Based Surgery\"\\n(OBS) accredited office (as mandated by the New York State Department of Health), will be\\nreimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region\\nI fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n5 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'Code Range\\': [\\'97001-97004\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsy\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 32\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%)\\nof the manufacturer\\'s suggested retail price.\\nIPA Network Access and Administrative Fees.\\nHealthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative\\nservices relating to quality improvement and education of Participating IPA Providers regarding\\nHealthfirst\\'s quality improvement programs. IPA and Healthfirst understand and agree that IPA shall\\nnot provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant\\nto this Agreement. In the event that IPA or an affiliated of IPA provides management services, the\\nparties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n6 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code/Modifier\\': [\\'99205-P1\\', \\'99212-P2\\', \\'99215-P3\\', \\'59409\\', \\'59514\\', \\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'PCAP Initial Visit\\', \\'PCAP Prenatal Visit\\', \\'PCAP Post partum Visit\\', \\'Vaginal Delivery\\', \\'Cesarean Delivery\\', \\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$196.07\\', \\'$130.00\\', \\'$163.00\\', \\'$1500.00\\', \\'$1500.00\\', \\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 33\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nAdditional Compensation to IPA in the form of Surplus Distribution.\\n1.\\nLimited Risk Transfer/No Transfer of Financial Risk\\n1.1.\\nIPA shall be entitled to Additional Compensation under this Agreement on an annual\\nbasis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus\\nfunds exist, as specified below.\\n1.2.\\nUse of Additional Compensation. IPA represents and warrants that Additional\\nCompensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a\\nmanner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On\\nrequest by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds\\nreceived from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d).\\n2.\\nDefinitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this\\nExhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the\\nAgreement to which this Exhibit is attached.\\n2.1.\\n\"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant\\nto Section 5 of this Exhibit 4.1(b).\\n2.2.\\n\"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk\\nEnrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan\\nContract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst.\\n2.3.\\n\"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to\\nHealthfirst\\'s administrative costs as reasonably determined by Healthfirst.\\n2.4.\\n\"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of\\nMedical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or\\nDeficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit.\\n2.5.\\n\"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the\\nAdministrative Deduction and any Quality Incentive Deduction.\\n2.6.\\n\"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for\\nprograms in which IPA and IPA Providers participate to improve the quality of Health Care Services\\nwhich Risk Enrollees receive.\\n2.7.\\n\"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical\\nRevenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits.\\n2.8.\\n\"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary\\ncare provider who is a Participating IPA Provider.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n7 of 10\\n\\nStart of Page No. = 34\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.9.\\n\"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following\\nReconciliation\\n3.\\nReconciliation of Medical Revenue to Determine Surpluses and Deficits. During each\\nReconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine\\nthe amount of any Surplus of Deficit:\\n3.1.\\nthe costs of claims paid for Health Care Services rendered to Risk Enrollees by\\nParticipating IPA Providers, Participating Providers and all other health care providers.\\n3.2.\\na\\nreasonable reserve for the cost of claims for Health Care Services rendered to Risk\\nEnrollees but not yet received;\\n3.3.\\nother costs associated with the provision of Health Care Services to Risk Enrollees\\nincluding but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs.\\n3.4.\\naccruals for any risk adjustment applicable to Adjusted Premiums.\\n4.\\nReconciliation Schedule.\\n4.1.\\nInterim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days\\n(120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus\\nor Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines\\nthat a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall\\nmake an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the\\nevent that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the\\ncalendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the\\nDeficit shall be handled as set forth in Section 6 below.\\n4.2.\\nAnnual Reconciliation and Distribution. Within one-hundred and twenty days (120) after\\nthe end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for\\nthat calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that\\ncalendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in\\nSection 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that\\nCalendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit\\nshall be handled as set forth in Section 6 below.\\n5.\\nPayment of Additional Compensation. In the event that Healthfirst determines that a Surplus has\\nbeen achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which\\nshall be the lesser of the following:\\n5.1.\\nTen percent (10%) of the Surplus for the calendar year, or\\n5.2.\\nAn amount such that the Additional Compensation equals twenty-five percent (25%) of\\nthe sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n8 of 10\\n\\nStart of Page No. = 35\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and\\niii) the Additional Compensation.\\n5.3.\\nHealthfirst shall pay any Additional Compensation, within thirty days of each\\nReconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation.\\n6.\\nTreatment of Deficits.\\n6.1.\\nDeficits Rolled Forward. In the event that a Deficit remains following any Reconciliation\\nor a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be\\nrolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar\\nyear.\\n6.2.\\nDeficits at Termination, Expiration or Non-Renewal. In the event that this Agreement\\nterminates, is not renewed or expires, regardless of the reason, Provider shall not be required to\\nreimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or\\nnon-renewal.\\n7.\\nCompliance with Regulatory Provisions Regarding Risk Transfer\\n7.1.\\nSDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with\\nthe\\napplicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH\\'s\\nProvider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount\\nfinancial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting\\nGuidelines, IPA shall demonstrate financial viability and establish a financial security deposit as\\nrequired by SDOH.\\n7.2.\\nPhysician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the\\napplicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the\\nplacement of stop loss insurance if applicable.\\n7.3.\\nFinancial Stability Monitoring and Reporting. Notwithstanding any other provision of\\nthis Agreement Healthfirst shall regularly monitor Provider\\'s financial condition to ensure that Provider\\nremains financially able to perform its obligations under this Agreement and meet any financial\\nsolvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material\\ndevelopments which may impair its ability to perform its obligations under this Agreement. In addition,\\nIPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e),\\nwhich shall include but not be limited to IPA\\'s quarterly and annual financial statements. IPA shall also\\nmaintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received\\nfrom Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and\\ndisbursements to Healthfirst on request.\\n8.\\nTermination or Amendment of Additional Compensation\\nHealthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to\\nwhich Additional Compensation is determined and paid to IPA as well as eliminate the payment of\\nAdditional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n9 of 10\\n\\nStart of Page No. = 36\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nthe Agreement such amendment or termination shall not change any Additional Compensation\\ncalculated or paid prior to the effective date of such amendment or termination.\\n9.\\nTermination, Non-Renewal or Expiration of the Agreement.\\nFor a period of twelve months following any termination, non-renewal or expiration of this Agreement,\\nregardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this\\nAgreement until the Final Reconciliation; provided, however, that payments to IPA of Additional\\nCompensation shall be suspended. These Reconciliations shall be on based Health Care Services\\nrendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement.\\nHealthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty\\ndays of the Final Reconciliation.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n10 of 10'" ] }, - "execution_count": 11, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "input_dict_filtered.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - "import preprocess\n", - "\n", - "filename, contract_text = list(input_dict.keys())[0], list(input_dict.values())[0]\n", - "text_dict = preprocess.split_text(contract_text)\n", - "text_dict = preprocess.highlight_rates(text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'050540697 Deepak Nanda 2017 PHSP MU.txt'" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "filename" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'REIMBRUSEMENT_EXCEPTION_IND': 'Y',\n", - " 'REIMBURSEMENT_DESCRIBE_EXCEPTION': \"any Change in Control (as defined in clauses (i) through (iv) of this Section (9.11) with respect to any entity that owns 50% or more of Provider's stock or other ownership interests\",\n", - " 'RATE_ESCALATOR_IND': 'N'}" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import json\n", - "\n", - "d = \"\"\"{\n", - " \"REIMBRUSEMENT_EXCEPTION_IND\": \"Y\",\n", - " \"REIMBURSEMENT_DESCRIBE_EXCEPTION\": \"any Change in Control (as defined in clauses (i) through (iv) of this Section (9.11) with respect to any entity that owns 50% or more of Provider's stock or other ownership interests\",\n", - " \"RATE_ESCALATOR_IND\": \"N\"\n", - "}\"\"\"\n", - "\n", - "json.loads(d)" + "contract_text" ] }, { From 0704041bae0bce7e18c2b67f8a9a966c44e21d30 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 7 May 2024 21:12:29 -0700 Subject: [PATCH 15/78] Table formatting for code improvement --- src/bottom_up_funcs.py | 67 +++-- src/config.py | 14 +- src/postprocess.py | 14 + src/preprocess.py | 13 - src/table_funcs.py | 43 +++- src/test.ipynb | 566 +++++++++++++++++++++++++++++------------ 6 files changed, 519 insertions(+), 198 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 3205d1c..461c9f8 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -8,6 +8,7 @@ import claude_funcs import preprocess import dict_operations import postprocess +import table_funcs def run_bottom_up_lesser(d, text_dict, tokens=8000): @@ -44,37 +45,45 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): temp_dicts = [] for d in answer_dicts: # Bottom Up Lesser - lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens) - temp_dicts.append(lesser_object[1]) # Add original object from d - if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object + if config.RUN_LESSER: + lesser_object = run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens) + temp_dicts.append(lesser_object[1]) # Add original object from d + if lesser_object[0]: temp_dicts.append(lesser_object[0]) # If lesser, add lesser object + else: + temp_dicts.append(d) # Add additional fields to each row final_dicts = [] for d in temp_dicts: if d is not None: page_num = d['page_num'] + # Bottom Up Methodology - prompt = prompts.BOTTOM_UP_METHODOLOGY(d) - methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) - d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + if config.RUN_METHODOLOGY: + prompt = prompts.BOTTOM_UP_METHODOLOGY(d) + methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer # # Bottom Up FS - prompt = prompts.BOTTOM_UP_FS(d) - fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) - fs_dict = dict_operations.secondary_string_to_dict(fs_answer) - d.update(fs_dict) + if config.RUN_FS: + prompt = prompts.BOTTOM_UP_FS(d) + fs_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) + fs_dict = dict_operations.secondary_string_to_dict(fs_answer) + d.update(fs_dict) # # Bottom Up Exception/Escalator - prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num]) - exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) - exc_dict = dict_operations.secondary_string_to_dict(exc_answer) - d.update(exc_dict) + if config.RUN_EXCEPTION: + prompt = prompts.BOTTOM_UP_EXCEPT_ESC(d, text_dict[page_num]) + exc_answer = claude_funcs.invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_HAIKU, max_tokens=tokens) + exc_dict = dict_operations.secondary_string_to_dict(exc_answer) + d.update(exc_dict) # # Bottom Up Codes - prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num]) - codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - codes_dict = dict_operations.secondary_string_to_dict(codes_answer) - d.update(codes_dict) + if config.RUN_CODES: + prompt = prompts.BOTTOM_UP_CODES(d, text_dict[page_num]) + codes_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + codes_dict = dict_operations.secondary_string_to_dict(codes_answer) + d.update(codes_dict) final_dicts.append(d) @@ -103,14 +112,15 @@ def consolidate_lob(lob_dicts, results_dicts): final_dicts.append(d) return final_dicts + def bottom_up(file_object): - #### PREPROCESS #### filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") + #### PREPROCESS #### contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) - text_dict = preprocess.align_tables(text_dict) + text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) #### PROMPT #### @@ -120,22 +130,27 @@ def bottom_up(file_object): answer_dicts = postprocess.primary_filter(answer_dicts) # Bottom Up LOB - CONTRACT_LOB, CONTRACT_PROGRAM, CONTRACT_PRODUCT, PROV_TYPE - lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000) - + if config.RUN_LOB: + lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000) + # Bottom Up Secondary results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) # Append LOB - final_dicts = consolidate_lob(lob_dicts, results_dicts) - - #### POSTPROCESS #### - #answer_dicts = append_secondary(answer_dicts) + if config.RUN_LOB: + final_dicts = consolidate_lob(lob_dicts, results_dicts) + else: + final_dicts = results_dicts for d in final_dicts: d['Filename'] = filename answer_df = pd.DataFrame(final_dicts) + #### POSTPROCESS #### + #answer_dicts = append_secondary(answer_dicts) + answer_df = postprocess.bottom_up_postprocess(answer_df) + # Write output if config.WRITE_OUTPUT: if config.OUTPUT_MODE == '_INDIVIDUAL_': diff --git a/src/config.py b/src/config.py index ee57efd..75bba8d 100644 --- a/src/config.py +++ b/src/config.py @@ -16,10 +16,22 @@ OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' # Multithread Settings MAX_WORKERS = 20 +# Prompt Debugging +RUN_PRIMARY = True # Always True +RUN_LOB = True +RUN_LESSER = True +RUN_METHODOLOGY = True +RUN_FS = True +RUN_EXCEPTION = True +RUN_CODES = True + # AWS Keys +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAOY7QOHS" +AWS_SECRET_ACCESS_KEY="U3zdt9cAgCFVdy48uwktZSx2owC4QXo4/mtWCwdQ" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIv//////////wEaCXVzLWVhc3QtMiJHMEUCIQCxyHC2X3koEVwj1v5GQCySf/crrB3a1muqvXsMEUOvjwIgbX9sZ25cw8Hc0zklioXs/5rq9hfXna4AeAynR3O0+eUqjAMI5P//////////ARAAGgw2NjAxMzEwNjg3ODIiDDZ45+avGABDdEaUiirgAkJzeXOOSyKxFQv6Mq3sCSFlAGkjzUuMAziu1RZdWcF99u0rQthauWdxJ6rU4kwfd/M2LoWRQgu/8VxsktWxcivbleRrxwGAtcayMOBDWzeCdCDnvWZgKxZULiefEeJK3BDwUsWRN541FgZ/xqsAoASQlMO98rpNZ6W8xNq961xSpVUItMRPjOuHAot9YhPi1xLl+haeIdpsxGhx4ztQuJ+/ag5KAVrKpEiSsrQqf2L/q4FHB/3/YYzzAgXqP7Wy4qoPsWxfARvCw1FAoOqqPGjcmFwD3kov8q0FyYG4cc7/0UySSHxl7qUHFAr8vPuF7i+BdAFxeZEpAFd4s00gIm0V4ne8ta0wELiIfbkULdKZu7UezROqhm4+q9nQByrog4d1+YRE7B6x3zA9koFogcHQizZtZyoBDYK6Q3p+eab+SjIW6NndyYhCT1diebVLfnpth0MDMnIHibMHetm+XGwwiMnrsQY6pgGKQdkRrc2lhFuoRtw4uh4GTTz9PTQap3Ts2UNe7sT/49TpWxtqSB5WUyFGn2GAqnh1sOBZON3jo1e8qjibVxfACfv38PUBlktj6bVI0izhKfcEdoSyYWW1Cs5CTt1wlF3sOu5ztkTB+Gq8+H20gYAH+BgctDy43H6OwCGbBcveRxOEVH9jn+9GC6I99GNmjQ0w3Is9jT7zaJf52lE9nAA4n/p64Rro" # File Paths -LOCAL_PATH = 'docs/test/' # Replace with local +LOCAL_PATH = '../docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/postprocess.py b/src/postprocess.py index fc46135..e890b87 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -10,3 +10,17 @@ def primary_filter(answer_dict): else: filtered_dicts.append(d) return filtered_dicts + + +def bottom_up_postprocess(df): + # Remove Unnamed columns + df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] + + # Reimbursement Flat Fee + df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True) + + # Reimbursement Rate + df.loc[:, 'REIMBURSEMENT_RATE'] = df['REIMBURSEMENT_RATE'].astype(str).str.replace(r'%|<<<|>>>|[^\d.]', '', regex=True) + + return df + diff --git a/src/preprocess.py b/src/preprocess.py index 8ae48d7..e76d580 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -10,19 +10,6 @@ def split_text(text): text_dict = {page.split()[0] : page for page in text_list} return text_dict -def align_tables(text_dict): - pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL) - for page, text in text_dict.items(): - tables = pattern.findall(text) - for table in tables: - pre_table = table.split('{')[0].strip() # Get the pretable text for alignment - text = text.replace(table, '') # Remove full table - text = text.replace(pre_table, table) # replace remaining pretable text with full table - text = text.replace('-------Table Start--------', '') - text = text.replace('-------Table End--------', '') - text_dict[page] = text - return text_dict - def highlight_rates(text_dict): for page, text in text_dict.items(): words = text.split() diff --git a/src/table_funcs.py b/src/table_funcs.py index 996fa6f..7ae937d 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -17,4 +17,45 @@ def extract_all_tables(input_dict): tables_dict = {} for filename, filetext in input_dict.items(): tables_dict[filename] = extract_tables(filetext) - return tables_dict \ No newline at end of file + return tables_dict + +def format_table(table): + table = table.replace('>>>', '').replace('<<<', '') + table_dict = eval(table) + + keys = list(table_dict.keys()) + # If values are lists + if isinstance(table_dict[keys[0]], list): + final_string = "" + for i in range(len(table_dict[keys[0]])): + for k in keys: + key_string = k + ': ' + table_dict[k][i] + ', ' + final_string += key_string + final_string += '|\n' + else: + # If values are not lists + final_string = "" + for k in keys: + key_string = k + ': ' + table_dict[k] + ', ' + final_string += key_string + final_string += '|\n' + return final_string + +def align_and_format_tables(text_dict): + pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL) + for page, text in text_dict.items(): + tables = pattern.findall(text) + for table in tables: + pre_table = table.split('{')[0].strip() # Get the pretable text for alignment + text = text.replace(table, '') # Remove full table + table_only = table.replace(pre_table, '') + try: + formatted_table = format_table(table_only) + except: + formatted_table = table + text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table + text = text.replace('-------Table Start--------', '') + text = text.replace('-------Table End--------', '') + text_dict[page] = text + return text_dict + diff --git a/src/test.ipynb b/src/test.ipynb index 2e51ada..56219a7 100644 --- a/src/test.ipynb +++ b/src/test.ipynb @@ -2,215 +2,467 @@ "cells": [ { "cell_type": "code", - "execution_count": 11, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import utils\n", "import preprocess\n", - "import re" + "import re\n", + "import pandas as pd\n", + "import bottom_up_funcs\n", + "import dict_operations\n", + "import postprocess\n", + "import prompts\n", + "import claude_funcs\n", + "\n", + "pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "input_dict = utils.read_input()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "dict_keys(['050540697 Deepak Nanda 2017 PHSP MU.txt', '133923495 CAIPA 2015 PHSP.txt'])" + "dict_keys(['133923495 CAIPA 2015 PHSP.txt'])" ] }, - "execution_count": 4, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "input_dict = utils.read_input()\n", "input_dict.keys()" ] }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ - "def clean_newlines(contract_text):\n", - " cleaned_text = re.sub(r'(?>>', '').replace('<<<', '')\n", + " table_dict = eval(table)\n", + " \n", + " keys = list(table_dict.keys())\n", + " # If values are lists\n", + " if isinstance(table_dict[keys[0]], list):\n", + " final_string = \"\"\n", + " for i in range(len(table_dict[keys[0]])):\n", + " for k in keys:\n", + " key_string = k + ': ' + table_dict[k][i] + ', '\n", + " final_string += key_string\n", + " final_string += '|\\n'\n", + " else:\n", + " # If values are not lists\n", + " final_string = \"\"\n", + " for k in keys:\n", + " key_string = k + ': ' + table_dict[k] + ', '\n", + " final_string += key_string\n", + " final_string += '|\\n'\n", + " return final_string\n", "\n", - "def align_tables(text):\n", + "def align_and_format_tables(text_dict):\n", " pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)\n", - " tables = pattern.findall(text)\n", - " for table in tables:\n", - " pre_table = table.split('{')[0].strip() # Get the pretable text for alignment\n", - " text = text.replace(table, '') # Remove full table\n", - " text = text.replace(pre_table, table) # replace remaining pretable text with full table\n", - " text = text.replace('-------Table Start--------', '')\n", - " text = text.replace('-------Table End--------', '') \n", - " return text" + " for page, text in text_dict.items():\n", + " tables = pattern.findall(text)\n", + " for table in tables:\n", + " pre_table = table.split('{')[0].strip() # Get the pretable text for alignment\n", + " text = text.replace(table, '') # Remove full table\n", + " table_only = table.replace(pre_table, '')\n", + " try:\n", + " formatted_table = format_table(table_only)\n", + " except:\n", + " formatted_table = table\n", + " text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table\n", + " text = text.replace('-------Table Start--------', '')\n", + " text = text.replace('-------Table End--------', '') \n", + " text_dict[page] = text\n", + " return text_dict" ] }, { "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "for filename, contract_text in input_dict.items():\n", - " if filename == '133923495 CAIPA 2015 PHSP.txt':\n", - " contract_text = clean_newlines(contract_text)\n", - " contract_text = align_tables(contract_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 55, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Document Index DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 1 Fifth Amendment to the Agreement by and between 1Managed Health, Inc. and Independent Practice Association 1 WITNESSETH: 1 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 3 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 4 EXHIBIT C 4 Compensation Schedule 4 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 5 5 IPA Network Access and Administrative Fees. 5 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 6 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 7 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 8 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 9 WITNESSETH: 9 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 10 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 11 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 12 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 13 COMPENSATION SCHEDULE 13 Reimbursement to IPA Providers 13 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 14 14 IPA Network Access and Administrative Fees. 14 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 15 Additional Compensation to IPA in the form of Surplus Distribution. 15 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 16 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 17 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 18 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 19 Fifth Amendment to the Agreement by and between 19Managed Health, Inc. and Independent Practice Association 19 WITNESSETH: 19 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 20 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 21 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 22 EXHIBIT C 22 Compensation Schedule 22 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 23 23 IPA Network Access and Administrative Fees. 23 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 24 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 25 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 26 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 27 Second Amendment to the Agreement 27 by and between 27Healthfirst PHSP, Inc. and Independent Practice Association 27 WITNESSETH: 27 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 28 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 29 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 30 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 31 COMPENSATION SCHEDULE 31 Reimbursement to IPA Providers 31 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 32 32 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 33 Additional Compensation to IPA in the form of Surplus Distribution. 33 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 34 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 35 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 36\n", - "\n", - "\n", - "\n", - "Start of Page No. = 1 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Fifth Amendment to the Agreement by and between Managed Health, Inc. and Independent Practice Association This Amendment to the Agreement between Managed Health, Inc. (\"Healthfirst\"), and Chinese American Independent Practice Association, Inc. (\"Provider\") is effective January 1, 2015. WITNESSETH: WHEREAS, Healthfirst and Provider have entered into an Agreement (the \"Agreement\") for the provision of Health Care Services to Enrollees; and WHEREAS, Healthfirst and Provider desire to amend that Agreement to delegate credentialing to Provider : NOW, THEREFORE, in consideration of the mutual covenants and agreements herein contained, Healthfirst and Provider hereby agree to amend the Agreement as follows: 1. A new Section 2.8 entitled Credentialing of Affiliated Providers is added to the Agreement as follows: 2.8 Credentialing of Affiliated Providers. 2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the responsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider; provided however, that such delegation shall be limited to i) those categories and types of providers which Provider credentials as part of Provider's regular credentialing process and ii) those providers who have or seek medical staff privileges with Provider. Provider shall not be required to credential Participating Providers which provider does not regularly credential or which do not have or seek medical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such Participating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider's credentialing process and that such process meets the requirements of this Agreement, including this Section 2.8 as of the Effective or at the time of Healthfirst's final approval of the credentialing process, whichever is later. 2.8.2 Provider shall have a formal application and review process for such Providers which includes, but is not limited to, a signed application and attestation. Such review shall be conducted by a Provider credentials committee in consultation with the Healthfirst credentials committee. Provider shall have Credentialing Policies and Procedures that defines the following: 2.8.2.1 scope of Affiliated Providers covered; 2.8.2.2 criteria for primary source verification of information; 2.8.2.3 the process used to make credentialing decisions; 2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements; FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 2 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2.8.2.5 the right of Affiliated Providers to review the information submitted in support of their credentialing application; 2.8.2.6 the process of notification to a Affiliated Provider of any information obtained during the credentialing process that varies substantially from the information provided by the Affiliated Provider; 2.8.2.7 the Affiliated Provider's right to correct erroneous information; 2.8.2.8 the Provider's medical director's direct responsibility and participation in the credentialing program; and 2.8.2.9 the process used to ensure confidentiality of all information obtained in the credentialing process, except otherwise provided by law. 2.8.3 Credentialing Criteria. The Provider's Credentials Committee shall determine the criteria for selection, which shall be consistent with Healthfirst's credentialing policies and procedures, as may be modified by Healthfirst from time to time. Provider's credentialing and recredentialing criteria and procedure, its procedure for selection and termination of Providers, and its Provider grievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be unreasonably withheld. At a minimum, Provider's credentialing and recredentialing criteria and procedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required under Article 28 of the New York State Public Health law and shall include requiring Affiliated Providers to be licensed and qualified to provide the services specified and to maintain such license in good standing. Provider shall include further credentialing requirements in addition to the TJC reasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision foregoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and his/her participation in the Healthfirst network. 2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the credentialing and recredentialing criteria and procedures used by Provider are consistent with Healthfirst's and TJC's credentialing and recredentialing criteria and procedures. In the event that Provider's credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst's credentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations with Provider, may adjust the compensation payable to Provider under this Agreement to compensate Healthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers employed by or affiliated with Provider. 2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the delegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an ongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and procedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than annually, rosters of Healthfirst Providers and such written verification or other substantiation as requested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are currently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to Healthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv) FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 3 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE and to provide Healthfirst with Affiliated Providers' initial appointment date, reappointment start date and reappointment end date. Provider shall retain the right to terminate immediately its agreements with, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated Provider is no longer a member of the Provider's medical staff, if the license to practice or DEA permit or any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or revoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid programs, or if said Affiliated Provider is not covered by insurance as provided pursuant to this Agreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify Healthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any Affiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA permit or any controlled substances registration suspended or otherwise restricted or revoked or is excluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance as required pursuant to this Agreement below or is convicted of any crime related to the provision of health care services. Notwithstanding any other provision of this Section to the contrary, it is expressly understood and agreed that Provider shall not be responsible for conducting individual site visits at provider sites other than at facilities operated by Provider. 2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions of this Agreement shall not require the release or other disclosure by any person of any records or other documents or information to the extent that such release or other disclosure is prohibited by or otherwise contrary to any applicable law. 2. A new Exhibit C setting forth the reimbursement to be paid by MHI to IPA and IPA Providers for the provision of Covered Services to Enrollees, a copy of which is attached to this Amendment, added to the Agreement. The reimbursement set forth in the Exhibit C attached to this amendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits C and the Letter of Agreement effective July 1, 2006 shall continue to apply to dates of service prior to that date. This Amendment supersedes the July 1, 2006 Letter of Agreement setting forth an administrative fee. 3. All other terms and conditions of the Agreement shall remain in full force and effect. IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above. {'Managed Health, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476 Name: Paul E. Portsmore Print Name', 'Title: SVP', 'Date: 12/31/2014 18:32:28 ET'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By: family', 'Name: PEGGY SHENG Print Name', 'Chief Operating Officer Title:', 'Date: Dec. 26, 2015']} {'Managed Health, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:']} FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 4 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE EXHIBIT C Compensation Schedule Reimbursement to IPA Providers I. Medicare and Commercial Enrollees 1. Except for the Covered Services set forth below, MHI shall directly reimburse IPA Providers who provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. 2. MHI shall reimburse IPA Providers for the following Covered Services as follows: a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} b. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program, except for physical therapy services which will be reimbursed as noted in \"a.\" above. C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - " a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsies', 'Colonoscopy, removal of tumor']} \n", - "\n", - "Start of Page No. = 5 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE e. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price. II. Where rates in any fee schedule described in this Agreement make reference to Medicare or Medicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and Medicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates, shall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon forty five calendar days following the date on which CMS or SDOH publishes such rates. IPA Network Access and Administrative Fees. For Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month for the following administrative services,: Access to IPA's network of Participating IPA Providers Credentialing of Participating IPA Providers Reporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst Administrative services relating to quality improvement and Enrollee satisfaction Education of Participating IPA Providers regarding Healthfirst's policies and procedures and quality improvement programs. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 6 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Risk Sharing with IPA 1 Definitions. For the purposes of this Agreement and Compensation Schedule, the following terms shall have the indicated meanings: 1.1 \"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees, adjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such adjustment is reasonably applied by Healthfirst. 1.2 \"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable reserve for incurred but not received (\"IBNR\") claims for Risk Services. 1.3 \"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target following Reconciliation. 1.4 \"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target. 1.5 \"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine Surpluses and Deficits. 1.6 \"Risk Enrollee\" shall mean Enrollees in Healthfirst's Medicare Advantage plans who select an IPA Provider as their Primary Care Provider. 1.7 \"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of whether such Services were provided by IPA Providers or other health care providers. 1.8 \"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums 1.9 \"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target following Reconciliation. 1.10 \"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target. 2 Risk Sharing. 2.1 Risk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk Services rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk Services provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall assume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst a portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration of Healthfirst's utilization and case management programs, the sharing of utilization data, and education of IPA Providers regarding utilization trends and the provision of quality and medically appropriate services. 2.2 Limited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services other than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 7 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE be required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or health care provider. In all instances, including any termination or expiration of this Agreement, Healthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk Services, to IPA Providers and other health care providers. 3 Regulatory Approval. The parties understand and acknowledge that this risk sharing arrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency having jurisdiction over this Agreement as required by applicable statutes and regulation. No payments shall be made by either party, prior to receiving such regulatory approval. The failure to obtain regulatory shall not be grounds for either party to terminate this Agreement which shall otherwise remain in effect. 4 Quarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each calendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding quarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the measurement period shall commence on the later of a) January 1, 2010 or b) the date of any required regulatory approval. 5 Risk Sharing; Distribution of Surplus and Payment of Deficits. 5.1 Surpluses. In the event that Healthfirst determines that a Surplus exists following Reconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal to the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between the Risk Target and the Upside Risk Limit. 5.2 Deficits. 5.2.1 In the event that Healthfirst determines that a Deficit exists following Reconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal to the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between the Risk Target and the Downside Risk Limit. 5.2.2 In the event that IPA fails to timely pay any amount due under Section 5.2.1, any such amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due to IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against future amounts due to IPA in the succeeding calendar year quarter. 5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless of the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such termination, expiration or non-renewal. 6 Compliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may use any compensation methodology to reimburse IPA Providers for Health Care Services; provided, however, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or amount, either separately or in combination with any compensation formulas which would place IPA Providers at substantial financial risk for the provision of referral services, as determined by the applicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 8 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE reasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts paid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA's compliance with the federal PIP regulations. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 9 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Second Amendment to the Agreement by and between Healthfirst PHSP, Inc. and Independent Practice Association This Amendment shall be effective January 1, 2015 by and between Chinese American Independent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\"). WITNESSETH: WHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of Covered Services to certain Healthfirst Enrollees; and WHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation which Healthfirst pays to IPA. NOW, THEREFORE, in consideration of the mutual covenants and agreements herein contained, Healthfirst and IPA hereby agree to amend the Agreement as follows: 1. Section 2.1 of the Agreement titled Provision of Health Care Services is deleted in its entirety and replaced Section 2.1. 2.1 Provision of Health Care Services. IPA shall arrange for IPA Providers to provide Health Care Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the applicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer participation with Healthfirst pursuant to this Agreement to any health care provider who is a member of IPA or has a contract with IPA, nor disclose share the terms and conditions of this Agreement included rates of reimbursement, without first obtaining the written consent of Healthfirst. Healthfirst shall have the right to refuse the participation of any particular health care provider under this Agreement. Healthfirst's determination to offer participation to any health care provider pursuant to this Agreement or to refuse to accept any health care provider for participation shall be based on Healthfirst's business needs as determined solely by Healthfirst. Healthfirst's Vice-President, Network Management or Executive Vice-President/Chief Operating Officer shall have sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the name, addresses, telephone numbers and other information regarding IPA Providers reasonably requested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use such information in Healthfirst advertising and provider directories. 2. A new Section 2.8 entitled Credentialing of Affiliated Providers is added to the Agreement as follows: 2.8 Credentialing of Affiliated Providers. 2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the responsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider; provided however, that such delegation shall be limited to i) those categories and types of providers which Provider credentials as part of Provider's regular credentialing process and ii) those providers FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 1 of 10\n", - "\n", - "Start of Page No. = 10 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE who have or seek medical staff privileges with Provider. Provider shall not be required to credential Participating Providers which provider does not regularly credential or which do not have or seek medical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such Participating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider's credentialing process and that such process meets the requirements of this Agreement, including this Section 2.8 as of the Effective Date or at the time of Healthfirst's final approval of the credentialing process, whichever is later. 2.8.2 Provider shall have a formal application and review process for such Providers which includes, but is not limited to, a signed application and attestation. Such review shall be conducted by a Provider credentials committee in consultation with the Healthfirst credentials committee. Provider shall have Credentialing Policies and Procedures that defines the following: 2.8.2.1 scope of Affiliated Providers covered; 2.8.2.2 criteria for primary source verification of information; 2.8.2.3 the process used to make credentialing decisions; 2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements; 2.8.2.5 the right of Affiliated Providers to review the information submitted in support of their credentialing application; 2.8.2.6 the process of notification to a Affiliated Provider of any information obtained during the credentialing process that varies substantially from the information provided by the Affiliated Provider; 2.8.2.7 the Affiliated Provider's right to correct erroneous information; 2.8.2.8 the Provider's medical director's direct responsibility and participation in the credentialing program; and 2.8.2.9 the process used to ensure confidentiality of all information obtained in the credentialing process, except otherwise provided by law. 2.8.3 Credentialing Criteria. The Provider's Credentials Committee shall determine the criteria for selection, which shall be consistent with Healthfirst's credentialing policies and procedures, as may be modified by Healthfirst from time to time. Provider's credentialing and recredentialing criteria and procedure, its procedure for selection and termination of Providers, and its Provider grievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be unreasonably withheld. At a minimum, Provider's credentialing and recredentialing criteria and procedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required under Article 28 of the New York State Public Health law and shall include requiring Affiliated Providers to be licensed and qualified to provide the services specified and to maintain such license in good standing. Provider shall include further credentialing requirements in addition to the TJC FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 2 of 10\n", - "\n", - "Start of Page No. = 11 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE reasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision foregoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and his/her participation in the Healthfirst network. 2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the credentialing and recredentialing criteria and procedures used by Provider are consistent with Healthfirst's and TJC's credentialing and recredentialing criteria and procedures. In the event that Provider's credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst's credentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations with Provider, may adjust the compensation payable to Provider under this Agreement to compensate Healthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers employed by or affiliated with Provider. 2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the delegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an ongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and procedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than annually, rosters of Healthfirst Providers and such written verification or other substantiation as requested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are currently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to Healthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv) and to provide Healthfirst with Affiliated Providers' initial appointment date, reappointment start date and reappointment end date. Provider shall retain the right to terminate immediately its agreements with, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated Provider is no longer a member of the Provider's medical staff, if the license to practice or DEA permit or any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or revoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid programs, or if said Affiliated Provider is not covered by insurance as provided pursuant to this Agreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify Healthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any Affiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA permit or any controlled substances registration suspended or otherwise restricted or revoked or is excluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance as required pursuant to this Agreement below or is convicted of any crime related to the provision of health care services. Notwithstanding any other provision of this Section to the contrary, it is expressly understood and agreed that Provider shall not be responsible for conducting individual site visits at provider sites other than at facilities operated by Provider. 2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions of this Agreement shall not require the release or other disclosure by any person of any records or other documents or information to the extent that such release or other disclosure is prohibited by or otherwise contrary to any applicable law. 3 A new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA Providers for the provision of Health Care Services to Enrollees and IPA for the provision of certain FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 3 of 10\n", - "\n", - "Start of Page No. = 12 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE administrative services, a copy of which is attached to this Amendment, is added to the Agreement. The reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service prior to that date. 4 All other terms and conditions of the Agreement shall remain in full force and effect. {'Healthfirst PHSP, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476. Name: Paul E. Portsmore Print Name', 'Title: SVP -', 'Date: 12/31/2014 I 18:32:28 ET -'], 'Chinese American Independent Practice Association, Inc.': ['By: f Sear', 'Name: PEGGY SHENG Print Name', 'Title: Chief Operating Officer', 'Date: December 26, 2014']} {'Healthfirst PHSP, Inc.': ['By: Name: Print Name', 'Title:', 'Date: -'], 'Chinese American Independent Practice Association, Inc.': ['By: Name: Print Name', 'Title:', 'Date:']} FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 4 of 10\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 13 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE EXHIBIT 4.1 COMPENSATION SCHEDULE Reimbursement to IPA Providers 1. Except for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA Providers who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. 2. Healthfirst shall reimburse IPA Providers for the Health Care Services set forth below as follows: a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} b. Physical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall be reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program, except for physical therapy services which will be reimbursed as noted in \"a.\" above. C. Gastroenterology Services. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 5 of 10\n", - "\n", - "\n", - " a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} \n", - "\n", - "Start of Page No. = 14 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE e. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price. IPA Network Access and Administrative Fees. Healthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative services relating to quality improvement and education of Participating IPA Providers regarding Healthfirst's quality improvement programs. IPA and Healthfirst understand and agree that IPA shall not provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant to this Agreement. In the event that IPA or an affiliated of IPA provides management services, the parties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 6 of 10\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 15 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Additional Compensation to IPA in the form of Surplus Distribution. 1. Limited Risk Transfer/No Transfer of Financial Risk 1.1. IPA shall be entitled to Additional Compensation under this Agreement on an annual basis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus funds exist, as specified below. 1.2. Use of Additional Compensation. IPA represents and warrants that Additional Compensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a manner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On request by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds received from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d). 2. Definitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this Exhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the Agreement to which this Exhibit is attached. 2.1. \"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant to Section 5 of this Exhibit 4.1(b). 2.2. \"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk Enrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan Contract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst. 2.3. \"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to Healthfirst's administrative costs as reasonably determined by Healthfirst. 2.4. \"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of Medical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit. 2.5. \"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the Administrative Deduction and any Quality Incentive Deduction. 2.6. \"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for programs in which IPA and IPA Providers participate to improve the quality of Health Care Services which Risk Enrollees receive. 2.7. \"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits. 2.8. \"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary care provider who is a Participating IPA Provider. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 7 of 10\n", - "\n", - "Start of Page No. = 16 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2.9. \"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following Reconciliation 3. Reconciliation of Medical Revenue to Determine Surpluses and Deficits. During each Reconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine the amount of any Surplus of Deficit: 3.1. the costs of claims paid for Health Care Services rendered to Risk Enrollees by Participating IPA Providers, Participating Providers and all other health care providers. 3.2. a reasonable reserve for the cost of claims for Health Care Services rendered to Risk Enrollees but not yet received; 3.3. other costs associated with the provision of Health Care Services to Risk Enrollees including but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs. 3.4. accruals for any risk adjustment applicable to Adjusted Premiums. 4. Reconciliation Schedule. 4.1. Interim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days (120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus or Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines that a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall make an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the event that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the calendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the Deficit shall be handled as set forth in Section 6 below. 4.2. Annual Reconciliation and Distribution. Within one-hundred and twenty days (120) after the end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for that calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that calendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in Section 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that Calendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit shall be handled as set forth in Section 6 below. 5. Payment of Additional Compensation. In the event that Healthfirst determines that a Surplus has been achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which shall be the lesser of the following: 5.1. Ten percent (10%) of the Surplus for the calendar year, or 5.2. An amount such that the Additional Compensation equals twenty-five percent (25%) of the sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 8 of 10\n", - "\n", - "Start of Page No. = 17 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Risk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and iii) the Additional Compensation. 5.3. Healthfirst shall pay any Additional Compensation, within thirty days of each Reconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation. 6. Treatment of Deficits. 6.1. Deficits Rolled Forward. In the event that a Deficit remains following any Reconciliation or a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be rolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar year. 6.2. Deficits at Termination, Expiration or Non-Renewal. In the event that this Agreement terminates, is not renewed or expires, regardless of the reason, Provider shall not be required to reimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or non-renewal. 7. Compliance with Regulatory Provisions Regarding Risk Transfer 7.1. SDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with the applicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH's Provider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount financial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting Guidelines, IPA shall demonstrate financial viability and establish a financial security deposit as required by SDOH. 7.2. Physician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the applicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the placement of stop loss insurance if applicable. 7.3. Financial Stability Monitoring and Reporting. Notwithstanding any other provision of this Agreement Healthfirst shall regularly monitor Provider's financial condition to ensure that Provider remains financially able to perform its obligations under this Agreement and meet any financial solvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material developments which may impair its ability to perform its obligations under this Agreement. In addition, IPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e), which shall include but not be limited to IPA's quarterly and annual financial statements. IPA shall also maintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received from Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and disbursements to Healthfirst on request. 8. Termination or Amendment of Additional Compensation Healthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to which Additional Compensation is determined and paid to IPA as well as eliminate the payment of Additional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 9 of 10\n", - "\n", - "Start of Page No. = 18 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE the Agreement such amendment or termination shall not change any Additional Compensation calculated or paid prior to the effective date of such amendment or termination. 9. Termination, Non-Renewal or Expiration of the Agreement. For a period of twelve months following any termination, non-renewal or expiration of this Agreement, regardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this Agreement until the Final Reconciliation; provided, however, that payments to IPA of Additional Compensation shall be suspended. These Reconciliations shall be on based Health Care Services rendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement. Healthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty days of the Final Reconciliation. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 10 of 10\n", - "\n", - "Start of Page No. = 19 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Fifth Amendment to the Agreement by and between Managed Health, Inc. and Independent Practice Association This Amendment to the Agreement between Managed Health, Inc. (\"Healthfirst\"), and Chinese American Independent Practice Association, Inc. (\"Provider\") is effective January 1, 2015. WITNESSETH: WHEREAS, Healthfirst and Provider have entered into an Agreement (the \"Agreement\") for the provision of Health Care Services to Enrollees; and WHEREAS, Healthfirst and Provider desire to amend that Agreement to delegate credentialing to Provider : NOW, THEREFORE, in consideration of the mutual covenants and agreements herein contained, Healthfirst and Provider hereby agree to amend the Agreement as follows: 1. A new Section 2.8 entitled Credentialing of Affiliated Providers is added to the Agreement as follows: 2.8 Credentialing of Affiliated Providers. 2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the responsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider; provided however, that such delegation shall be limited to i) those categories and types of providers which Provider credentials as part of Provider's regular credentialing process and ii) those providers who have or seek medical staff privileges with Provider. Provider shall not be required to credential Participating Providers which provider does not regularly credential or which do not have or seek medical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such Participating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider's credentialing process and that such process meets the requirements of this Agreement, including this Section 2.8 as of the Effective or at the time of Healthfirst's final approval of the credentialing process, whichever is later. 2.8.2 Provider shall have a formal application and review process for such Providers which includes, but is not limited to, a signed application and attestation. Such review shall be conducted by a Provider credentials committee in consultation with the Healthfirst credentials committee. Provider shall have Credentialing Policies and Procedures that defines the following: 2.8.2.1 scope of Affiliated Providers covered; 2.8.2.2 criteria for primary source verification of information; 2.8.2.3 the process used to make credentialing decisions; 2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements; FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 20 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2.8.2.5 the right of Affiliated Providers to review the information submitted in support of their credentialing application; 2.8.2.6 the process of notification to a Affiliated Provider of any information obtained during the credentialing process that varies substantially from the information provided by the Affiliated Provider; 2.8.2.7 the Affiliated Provider's right to correct erroneous information; 2.8.2.8 the Provider's medical director's direct responsibility and participation in the credentialing program; and 2.8.2.9 the process used to ensure confidentiality of all information obtained in the credentialing process, except otherwise provided by law. 2.8.3 Credentialing Criteria. The Provider's Credentials Committee shall determine the criteria for selection, which shall be consistent with Healthfirst's credentialing policies and procedures, as may be modified by Healthfirst from time to time. Provider's credentialing and recredentialing criteria and procedure, its procedure for selection and termination of Providers, and its Provider grievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be unreasonably withheld. At a minimum, Provider's credentialing and recredentialing criteria and procedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required under Article 28 of the New York State Public Health law and shall include requiring Affiliated Providers to be licensed and qualified to provide the services specified and to maintain such license in good standing. Provider shall include further credentialing requirements in addition to the TJC reasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision foregoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and his/her participation in the Healthfirst network. 2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the credentialing and recredentialing criteria and procedures used by Provider are consistent with Healthfirst's and TJC's credentialing and recredentialing criteria and procedures. In the event that Provider's credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst's credentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations with Provider, may adjust the compensation payable to Provider under this Agreement to compensate Healthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers employed by or affiliated with Provider. 2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the delegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an ongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and procedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than annually, rosters of Healthfirst Providers and such written verification or other substantiation as requested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are currently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to Healthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv) FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 21 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE and to provide Healthfirst with Affiliated Providers' initial appointment date, reappointment start date and reappointment end date. Provider shall retain the right to terminate immediately its agreements with, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated Provider is no longer a member of the Provider's medical staff, if the license to practice or DEA permit or any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or revoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid programs, or if said Affiliated Provider is not covered by insurance as provided pursuant to this Agreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify Healthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any Affiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA permit or any controlled substances registration suspended or otherwise restricted or revoked or is excluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance as required pursuant to this Agreement below or is convicted of any crime related to the provision of health care services. Notwithstanding any other provision of this Section to the contrary, it is expressly understood and agreed that Provider shall not be responsible for conducting individual site visits at provider sites other than at facilities operated by Provider. 2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions of this Agreement shall not require the release or other disclosure by any person of any records or other documents or information to the extent that such release or other disclosure is prohibited by or otherwise contrary to any applicable law. 2. A new Exhibit C setting forth the reimbursement to be paid by MHI to IPA and IPA Providers for the provision of Covered Services to Enrollees, a copy of which is attached to this Amendment, added to the Agreement. The reimbursement set forth in the Exhibit C attached to this amendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits C and the Letter of Agreement effective July 1, 2006 shall continue to apply to dates of service prior to that date. This Amendment supersedes the July 1, 2006 Letter of Agreement setting forth an administrative fee. 3. All other terms and conditions of the Agreement shall remain in full force and effect. IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above. {'Managed Health, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476 Name: Paul E. Portsmore Print Name', 'Title: SVP', 'Date: 12/31/2014 18:32:28 ET'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By: family', 'Name: PEGGY SHENG Print Name', 'Chief Operating Officer Title:', 'Date: Dec. 26, 2015']} {'Managed Health, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:']} FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - " IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above. {'Managed Health, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476 Name: Paul E. Portsmore Print Name', 'Title: SVP', 'Date: 12/31/2014 18:32:28 ET'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By: family', 'Name: PEGGY SHENG Print Name', 'Chief Operating Officer Title:', 'Date: Dec. 26, 2015']} {'Managed Health, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By:', 'Name: Print Name', 'Title:', 'Date:']} {'Managed Health, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476 Name: Paul E. Portsmore Print Name', 'Title: SVP', 'Date: 12/31/2014 18:32:28 ET'], 'Provider: Chinese American Independent Practice Association, Inc.': ['By: family', 'Name: PEGGY SHENG Print Name', 'Chief Operating Officer Title:', 'Date: Dec. 26, 2015']} \n", - "\n", - "Start of Page No. = 22 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE EXHIBIT C Compensation Schedule Reimbursement to IPA Providers I. Medicare and Commercial Enrollees 1. Except for the Covered Services set forth below, MHI shall directly reimburse IPA Providers who provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. 2. MHI shall reimburse IPA Providers for the following Covered Services as follows: a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} b. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program, except for physical therapy services which will be reimbursed as noted in \"a.\" above. C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - " Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: {'Code Range': ['97001-97006', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsies', 'Colonoscopy, removal of tumor']} \n", - "\n", - "Start of Page No. = 23 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE e. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price. II. Where rates in any fee schedule described in this Agreement make reference to Medicare or Medicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and Medicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates, shall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon forty five calendar days following the date on which CMS or SDOH publishes such rates. IPA Network Access and Administrative Fees. For Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month for the following administrative services,: Access to IPA's network of Participating IPA Providers Credentialing of Participating IPA Providers Reporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst Administrative services relating to quality improvement and Enrollee satisfaction Education of Participating IPA Providers regarding Healthfirst's policies and procedures and quality improvement programs. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 24 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Risk Sharing with IPA 1 Definitions. For the purposes of this Agreement and Compensation Schedule, the following terms shall have the indicated meanings: 1.1 \"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees, adjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such adjustment is reasonably applied by Healthfirst. 1.2 \"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable reserve for incurred but not received (\"IBNR\") claims for Risk Services. 1.3 \"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target following Reconciliation. 1.4 \"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target. 1.5 \"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine Surpluses and Deficits. 1.6 \"Risk Enrollee\" shall mean Enrollees in Healthfirst's Medicare Advantage plans who select an IPA Provider as their Primary Care Provider. 1.7 \"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of whether such Services were provided by IPA Providers or other health care providers. 1.8 \"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums 1.9 \"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target following Reconciliation. 1.10 \"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target. 2 Risk Sharing. 2.1 Risk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk Services rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk Services provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall assume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst a portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration of Healthfirst's utilization and case management programs, the sharing of utilization data, and education of IPA Providers regarding utilization trends and the provision of quality and medically appropriate services. 2.2 Limited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services other than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 25 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE be required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or health care provider. In all instances, including any termination or expiration of this Agreement, Healthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk Services, to IPA Providers and other health care providers. 3 Regulatory Approval. The parties understand and acknowledge that this risk sharing arrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency having jurisdiction over this Agreement as required by applicable statutes and regulation. No payments shall be made by either party, prior to receiving such regulatory approval. The failure to obtain regulatory shall not be grounds for either party to terminate this Agreement which shall otherwise remain in effect. 4 Quarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each calendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding quarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the measurement period shall commence on the later of a) January 1, 2010 or b) the date of any required regulatory approval. 5 Risk Sharing; Distribution of Surplus and Payment of Deficits. 5.1 Surpluses. In the event that Healthfirst determines that a Surplus exists following Reconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal to the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between the Risk Target and the Upside Risk Limit. 5.2 Deficits. 5.2.1 In the event that Healthfirst determines that a Deficit exists following Reconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal to the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between the Risk Target and the Downside Risk Limit. 5.2.2 In the event that IPA fails to timely pay any amount due under Section 5.2.1, any such amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due to IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against future amounts due to IPA in the succeeding calendar year quarter. 5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless of the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such termination, expiration or non-renewal. 6 Compliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may use any compensation methodology to reimburse IPA Providers for Health Care Services; provided, however, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or amount, either separately or in combination with any compensation formulas which would place IPA Providers at substantial financial risk for the provision of referral services, as determined by the applicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 26 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE reasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts paid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA's compliance with the federal PIP regulations. FED. TAX ID: 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015\n", - "\n", - "Start of Page No. = 27 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Second Amendment to the Agreement by and between Healthfirst PHSP, Inc. and Independent Practice Association This Amendment shall be effective January 1, 2015 by and between Chinese American Independent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\"). WITNESSETH: WHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of Covered Services to certain Healthfirst Enrollees; and WHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation which Healthfirst pays to IPA. NOW, THEREFORE, in consideration of the mutual covenants and agreements herein contained, Healthfirst and IPA hereby agree to amend the Agreement as follows: 1. Section 2.1 of the Agreement titled Provision of Health Care Services is deleted in its entirety and replaced Section 2.1. 2.1 Provision of Health Care Services. IPA shall arrange for IPA Providers to provide Health Care Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the applicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer participation with Healthfirst pursuant to this Agreement to any health care provider who is a member of IPA or has a contract with IPA, nor disclose share the terms and conditions of this Agreement included rates of reimbursement, without first obtaining the written consent of Healthfirst. Healthfirst shall have the right to refuse the participation of any particular health care provider under this Agreement. Healthfirst's determination to offer participation to any health care provider pursuant to this Agreement or to refuse to accept any health care provider for participation shall be based on Healthfirst's business needs as determined solely by Healthfirst. Healthfirst's Vice-President, Network Management or Executive Vice-President/Chief Operating Officer shall have sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the name, addresses, telephone numbers and other information regarding IPA Providers reasonably requested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use such information in Healthfirst advertising and provider directories. 2. A new Section 2.8 entitled Credentialing of Affiliated Providers is added to the Agreement as follows: 2.8 Credentialing of Affiliated Providers. 2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the responsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider; provided however, that such delegation shall be limited to i) those categories and types of providers which Provider credentials as part of Provider's regular credentialing process and ii) those providers FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 1 of 10\n", - "\n", - "Start of Page No. = 28 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE who have or seek medical staff privileges with Provider. Provider shall not be required to credential Participating Providers which provider does not regularly credential or which do not have or seek medical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such Participating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider's credentialing process and that such process meets the requirements of this Agreement, including this Section 2.8 as of the Effective Date or at the time of Healthfirst's final approval of the credentialing process, whichever is later. 2.8.2 Provider shall have a formal application and review process for such Providers which includes, but is not limited to, a signed application and attestation. Such review shall be conducted by a Provider credentials committee in consultation with the Healthfirst credentials committee. Provider shall have Credentialing Policies and Procedures that defines the following: 2.8.2.1 scope of Affiliated Providers covered; 2.8.2.2 criteria for primary source verification of information; 2.8.2.3 the process used to make credentialing decisions; 2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements; 2.8.2.5 the right of Affiliated Providers to review the information submitted in support of their credentialing application; 2.8.2.6 the process of notification to a Affiliated Provider of any information obtained during the credentialing process that varies substantially from the information provided by the Affiliated Provider; 2.8.2.7 the Affiliated Provider's right to correct erroneous information; 2.8.2.8 the Provider's medical director's direct responsibility and participation in the credentialing program; and 2.8.2.9 the process used to ensure confidentiality of all information obtained in the credentialing process, except otherwise provided by law. 2.8.3 Credentialing Criteria. The Provider's Credentials Committee shall determine the criteria for selection, which shall be consistent with Healthfirst's credentialing policies and procedures, as may be modified by Healthfirst from time to time. Provider's credentialing and recredentialing criteria and procedure, its procedure for selection and termination of Providers, and its Provider grievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be unreasonably withheld. At a minimum, Provider's credentialing and recredentialing criteria and procedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required under Article 28 of the New York State Public Health law and shall include requiring Affiliated Providers to be licensed and qualified to provide the services specified and to maintain such license in good standing. Provider shall include further credentialing requirements in addition to the TJC FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 2 of 10\n", - "\n", - "Start of Page No. = 29 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE reasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision foregoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and his/her participation in the Healthfirst network. 2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the credentialing and recredentialing criteria and procedures used by Provider are consistent with Healthfirst's and TJC's credentialing and recredentialing criteria and procedures. In the event that Provider's credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst's credentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations with Provider, may adjust the compensation payable to Provider under this Agreement to compensate Healthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers employed by or affiliated with Provider. 2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the delegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an ongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and procedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than annually, rosters of Healthfirst Providers and such written verification or other substantiation as requested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are currently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to Healthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv) and to provide Healthfirst with Affiliated Providers' initial appointment date, reappointment start date and reappointment end date. Provider shall retain the right to terminate immediately its agreements with, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated Provider is no longer a member of the Provider's medical staff, if the license to practice or DEA permit or any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or revoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid programs, or if said Affiliated Provider is not covered by insurance as provided pursuant to this Agreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify Healthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any Affiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA permit or any controlled substances registration suspended or otherwise restricted or revoked or is excluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance as required pursuant to this Agreement below or is convicted of any crime related to the provision of health care services. Notwithstanding any other provision of this Section to the contrary, it is expressly understood and agreed that Provider shall not be responsible for conducting individual site visits at provider sites other than at facilities operated by Provider. 2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions of this Agreement shall not require the release or other disclosure by any person of any records or other documents or information to the extent that such release or other disclosure is prohibited by or otherwise contrary to any applicable law. 3 A new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA Providers for the provision of Health Care Services to Enrollees and IPA for the provision of certain FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 3 of 10\n", - "\n", - "Start of Page No. = 30 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE administrative services, a copy of which is attached to this Amendment, is added to the Agreement. The reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service prior to that date. 4 All other terms and conditions of the Agreement shall remain in full force and effect. {'Healthfirst PHSP, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476. Name: Paul E. Portsmore Print Name', 'Title: SVP -', 'Date: 12/31/2014 I 18:32:28 ET -'], 'Chinese American Independent Practice Association, Inc.': ['By: f Sear', 'Name: PEGGY SHENG Print Name', 'Title: Chief Operating Officer', 'Date: December 26, 2014']} {'Healthfirst PHSP, Inc.': ['By: Name: Print Name', 'Title:', 'Date: -'], 'Chinese American Independent Practice Association, Inc.': ['By: Name: Print Name', 'Title:', 'Date:']} FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 4 of 10\n", - "\n", - "\n", - " 4 All other terms and conditions of the Agreement shall remain in full force and effect. {'Healthfirst PHSP, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476. Name: Paul E. Portsmore Print Name', 'Title: SVP -', 'Date: 12/31/2014 I 18:32:28 ET -'], 'Chinese American Independent Practice Association, Inc.': ['By: f Sear', 'Name: PEGGY SHENG Print Name', 'Title: Chief Operating Officer', 'Date: December 26, 2014']} {'Healthfirst PHSP, Inc.': ['By: Name: Print Name', 'Title:', 'Date: -'], 'Chinese American Independent Practice Association, Inc.': ['By: Name: Print Name', 'Title:', 'Date:']} {'Healthfirst PHSP, Inc.': ['DocuSigned by: By: Paul E. Portsmore', '441CEAD8E502476. Name: Paul E. Portsmore Print Name', 'Title: SVP -', 'Date: 12/31/2014 I 18:32:28 ET -'], 'Chinese American Independent Practice Association, Inc.': ['By: f Sear', 'Name: PEGGY SHENG Print Name', 'Title: Chief Operating Officer', 'Date: December 26, 2014']} \n", - "\n", - "Start of Page No. = 31 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE EXHIBIT 4.1 COMPENSATION SCHEDULE Reimbursement to IPA Providers 1. Except for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA Providers who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. 2. Healthfirst shall reimburse IPA Providers for the Health Care Services set forth below as follows: a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} b. Physical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall be reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program, except for physical therapy services which will be reimbursed as noted in \"a.\" above. C. Gastroenterology Services. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 5 of 10\n", - "\n", - "\n", - " a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service. {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} {'Code Range': ['97001-97004', '97010-97028', '97032-97039', '97110-97546', '97750-97755'], 'Code Range Description': ['Physical Medicine Assessments', 'Physical Therapy Treatment Modalities: Supervised', 'Physical Therapy Treatment Modalities: Constant Attendance', 'Other Therapeutic Techniques With Direct Patient Contact', 'Physical performance test & Assistive technology assessment']} {'CPT Code': ['43235', '43239', '45378', '45380', '45385'], 'CPT Description': ['Upper gastrointestinal endoscopy', 'Upper gastrointestinal endoscopy, with biopsy', 'Colonoscopy, diagnostic', 'Colonoscopy, with biopsy', 'Colonoscopy, removal of tumor']} \n", - "\n", - "Start of Page No. = 32 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE e. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price. IPA Network Access and Administrative Fees. Healthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative services relating to quality improvement and education of Participating IPA Providers regarding Healthfirst's quality improvement programs. IPA and Healthfirst understand and agree that IPA shall not provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant to this Agreement. In the event that IPA or an affiliated of IPA provides management services, the parties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 6 of 10\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 33 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Additional Compensation to IPA in the form of Surplus Distribution. 1. Limited Risk Transfer/No Transfer of Financial Risk 1.1. IPA shall be entitled to Additional Compensation under this Agreement on an annual basis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus funds exist, as specified below. 1.2. Use of Additional Compensation. IPA represents and warrants that Additional Compensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a manner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On request by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds received from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d). 2. Definitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this Exhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the Agreement to which this Exhibit is attached. 2.1. \"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant to Section 5 of this Exhibit 4.1(b). 2.2. \"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk Enrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan Contract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst. 2.3. \"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to Healthfirst's administrative costs as reasonably determined by Healthfirst. 2.4. \"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of Medical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit. 2.5. \"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the Administrative Deduction and any Quality Incentive Deduction. 2.6. \"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for programs in which IPA and IPA Providers participate to improve the quality of Health Care Services which Risk Enrollees receive. 2.7. \"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits. 2.8. \"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary care provider who is a Participating IPA Provider. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 7 of 10\n", - "\n", - "Start of Page No. = 34 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2.9. \"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following Reconciliation 3. Reconciliation of Medical Revenue to Determine Surpluses and Deficits. During each Reconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine the amount of any Surplus of Deficit: 3.1. the costs of claims paid for Health Care Services rendered to Risk Enrollees by Participating IPA Providers, Participating Providers and all other health care providers. 3.2. a reasonable reserve for the cost of claims for Health Care Services rendered to Risk Enrollees but not yet received; 3.3. other costs associated with the provision of Health Care Services to Risk Enrollees including but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs. 3.4. accruals for any risk adjustment applicable to Adjusted Premiums. 4. Reconciliation Schedule. 4.1. Interim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days (120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus or Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines that a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall make an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the event that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the calendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the Deficit shall be handled as set forth in Section 6 below. 4.2. Annual Reconciliation and Distribution. Within one-hundred and twenty days (120) after the end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for that calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that calendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in Section 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that Calendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit shall be handled as set forth in Section 6 below. 5. Payment of Additional Compensation. In the event that Healthfirst determines that a Surplus has been achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which shall be the lesser of the following: 5.1. Ten percent (10%) of the Surplus for the calendar year, or 5.2. An amount such that the Additional Compensation equals twenty-five percent (25%) of the sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 8 of 10\n", - "\n", - "Start of Page No. = 35 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE Risk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and iii) the Additional Compensation. 5.3. Healthfirst shall pay any Additional Compensation, within thirty days of each Reconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation. 6. Treatment of Deficits. 6.1. Deficits Rolled Forward. In the event that a Deficit remains following any Reconciliation or a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be rolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar year. 6.2. Deficits at Termination, Expiration or Non-Renewal. In the event that this Agreement terminates, is not renewed or expires, regardless of the reason, Provider shall not be required to reimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or non-renewal. 7. Compliance with Regulatory Provisions Regarding Risk Transfer 7.1. SDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with the applicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH's Provider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount financial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting Guidelines, IPA shall demonstrate financial viability and establish a financial security deposit as required by SDOH. 7.2. Physician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the applicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the placement of stop loss insurance if applicable. 7.3. Financial Stability Monitoring and Reporting. Notwithstanding any other provision of this Agreement Healthfirst shall regularly monitor Provider's financial condition to ensure that Provider remains financially able to perform its obligations under this Agreement and meet any financial solvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material developments which may impair its ability to perform its obligations under this Agreement. In addition, IPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e), which shall include but not be limited to IPA's quarterly and annual financial statements. IPA shall also maintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received from Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and disbursements to Healthfirst on request. 8. Termination or Amendment of Additional Compensation Healthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to which Additional Compensation is determined and paid to IPA as well as eliminate the payment of Additional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 9 of 10\n", - "\n", - "Start of Page No. = 36 DocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE the Agreement such amendment or termination shall not change any Additional Compensation calculated or paid prior to the effective date of such amendment or termination. 9. Termination, Non-Renewal or Expiration of the Agreement. For a period of twelve months following any termination, non-renewal or expiration of this Agreement, regardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this Agreement until the Final Reconciliation; provided, however, that payments to IPA of Additional Compensation shall be suspended. These Reconciliations shall be on based Health Care Services rendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement. Healthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty days of the Final Reconciliation. FED. TAX ID # 133923495 CONTRACT EFFECTIVE DATE: January 1, 2015 SDOH 4814 10 of 10\n" + "Managed Health, Inc.: By:, Provider: Chinese American Independent Practice Association, Inc.: By:, |\n", + "Managed Health, Inc.: Name: Print Name, Provider: Chinese American Independent Practice Association, Inc.: Name: Print Name, |\n", + "Managed Health, Inc.: Title:, Provider: Chinese American Independent Practice Association, Inc.: Title:, |\n", + "Managed Health, Inc.: Date:, Provider: Chinese American Independent Practice Association, Inc.: Date:, |\n", + "\n", + "Code Range: 97001-97006, Code Range Description: Physical Medicine Assessments, |\n", + "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", + "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", + "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", + "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", + "\n", + "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", + "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", + "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", + "CPT Code: 45380, CPT Description: Colonoscopy, with biopsies, |\n", + "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", + "\n", + "CPT Code: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", + "CPT Code: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", + "CPT Code: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", + "CPT Code: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", + "CPT Code: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", + "CPT Code: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", + "CPT Code: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", + "CPT Code: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", + "CPT Code: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", + "CPT Code: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", + "CPT Code: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", + "CPT Code: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", + "CPT Code: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", + "CPT Code: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", + "CPT Code: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", + "CPT Code: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", + "\n", + "Healthfirst PHSP, Inc.: By: Name: Print Name, Chinese American Independent Practice Association, Inc.: By: Name: Print Name, |\n", + "Healthfirst PHSP, Inc.: Title:, Chinese American Independent Practice Association, Inc.: Title:, |\n", + "Healthfirst PHSP, Inc.: Date: -, Chinese American Independent Practice Association, Inc.: Date:, |\n", + "\n", + "Code Range: 97001-97004, Code Range Description: Physical Medicine Assessments, |\n", + "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", + "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", + "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", + "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", + "\n", + "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", + "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", + "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", + "CPT Code: 45380, CPT Description: Colonoscopy, with biopsy, |\n", + "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", + "\n", + "CPT Code/Modifier: 99205-P1, CPT Description: PCAP Initial Visit, Amount: $196.07, |\n", + "CPT Code/Modifier: 99212-P2, CPT Description: PCAP Prenatal Visit, Amount: $130.00, |\n", + "CPT Code/Modifier: 99215-P3, CPT Description: PCAP Post partum Visit, Amount: $163.00, |\n", + "CPT Code/Modifier: 59409, CPT Description: Vaginal Delivery, Amount: $1500.00, |\n", + "CPT Code/Modifier: 59514, CPT Description: Cesarean Delivery, Amount: $1500.00, |\n", + "CPT Code/Modifier: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", + "CPT Code/Modifier: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code/Modifier: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code/Modifier: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", + "CPT Code/Modifier: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", + "CPT Code/Modifier: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", + "CPT Code/Modifier: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", + "CPT Code/Modifier: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", + "CPT Code/Modifier: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code/Modifier: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code/Modifier: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", + "CPT Code/Modifier: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", + "CPT Code/Modifier: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", + "CPT Code/Modifier: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", + "CPT Code/Modifier: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", + "CPT Code/Modifier: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", + "CPT Code/Modifier: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", + "CPT Code/Modifier: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", + "CPT Code/Modifier: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", + "CPT Code/Modifier: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", + "\n", + "Managed Health, Inc.: DocuSigned by: By: Paul E. Portsmore, Provider: Chinese American Independent Practice Association, Inc.: By: family, |\n", + "Managed Health, Inc.: 441CEAD8E502476 Name: Paul E. Portsmore Print Name, Provider: Chinese American Independent Practice Association, Inc.: Name: PEGGY SHENG Print Name, |\n", + "Managed Health, Inc.: Title: SVP, Provider: Chinese American Independent Practice Association, Inc.: Chief Operating Officer Title:, |\n", + "Managed Health, Inc.: Date: 12/31/2014 18:32:28 ET, Provider: Chinese American Independent Practice Association, Inc.: Date: Dec. 26, 2015, |\n", + "\n", + "Code Range: 97001-97006, Code Range Description: Physical Medicine Assessments, |\n", + "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", + "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", + "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", + "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", + "\n", + "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", + "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", + "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", + "CPT Code: 45380, CPT Description: Colonoscopy, with biopsies, |\n", + "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", + "\n", + "CPT Code: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", + "CPT Code: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", + "CPT Code: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", + "CPT Code: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", + "CPT Code: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", + "CPT Code: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", + "CPT Code: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", + "CPT Code: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", + "CPT Code: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", + "CPT Code: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", + "CPT Code: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", + "CPT Code: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", + "CPT Code: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", + "CPT Code: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", + "CPT Code: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", + "CPT Code: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", + "\n", + "Healthfirst PHSP, Inc.: DocuSigned by: By: Paul E. Portsmore, Chinese American Independent Practice Association, Inc.: By: f Sear, |\n", + "Healthfirst PHSP, Inc.: 441CEAD8E502476. Name: Paul E. Portsmore Print Name, Chinese American Independent Practice Association, Inc.: Name: PEGGY SHENG Print Name, |\n", + "Healthfirst PHSP, Inc.: Title: SVP -, Chinese American Independent Practice Association, Inc.: Title: Chief Operating Officer, |\n", + "Healthfirst PHSP, Inc.: Date: 12/31/2014 I 18:32:28 ET -, Chinese American Independent Practice Association, Inc.: Date: December 26, 2014, |\n", + "\n", + "Code Range: 97001-97004, Code Range Description: Physical Medicine Assessments, |\n", + "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", + "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", + "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", + "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", + "\n", + "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", + "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", + "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", + "CPT Code: 45380, CPT Description: Colonoscopy, with biopsy, |\n", + "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", + "\n", + "CPT Code/Modifier: 99205-P1, CPT Description: PCAP Initial Visit, Amount: $196.07, |\n", + "CPT Code/Modifier: 99212-P2, CPT Description: PCAP Prenatal Visit, Amount: $130.00, |\n", + "CPT Code/Modifier: 99215-P3, CPT Description: PCAP Post partum Visit, Amount: $163.00, |\n", + "CPT Code/Modifier: 59409, CPT Description: Vaginal Delivery, Amount: $1500.00, |\n", + "CPT Code/Modifier: 59514, CPT Description: Cesarean Delivery, Amount: $1500.00, |\n", + "CPT Code/Modifier: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", + "CPT Code/Modifier: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code/Modifier: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code/Modifier: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", + "CPT Code/Modifier: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", + "CPT Code/Modifier: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", + "CPT Code/Modifier: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", + "CPT Code/Modifier: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", + "CPT Code/Modifier: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", + "CPT Code/Modifier: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", + "CPT Code/Modifier: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", + "CPT Code/Modifier: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", + "CPT Code/Modifier: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", + "CPT Code/Modifier: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", + "CPT Code/Modifier: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", + "CPT Code/Modifier: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", + "CPT Code/Modifier: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", + "CPT Code/Modifier: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", + "CPT Code/Modifier: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", + "CPT Code/Modifier: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", + "\n" ] } ], "source": [ - "print(contract_text)" + "contract_text = preprocess.clean_newlines(input_dict['133923495 CAIPA 2015 PHSP.txt'])\n", + "text_dict = preprocess.split_text(contract_text)\n", + "text_dict = align_and_format_tables(text_dict)\n", + "text_dict = preprocess.highlight_rates(text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "answer_strings = bottom_up_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000)\n", + "answer_dicts = dict_operations.primary_string_to_dict(answer_strings)\n", + "answer_dicts = postprocess.primary_filter(answer_dicts)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'SERVICE': 'Initial visit, new patient, infant (under 1 year)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$65.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, early childhood (age 1-4)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, late childhood (age 5-11)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, adolescent (age 12-17)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$65.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, age 18-39',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$74.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, age 40-64',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$80.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Initial visit, new patient, age 65 and older',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$118.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, infant (under 1 year)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, early childhood (age 1-4)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, late childhood (age 5-11)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, adolescent (age 12-17)',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, age 18-39',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$75.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, age 40-64',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$90.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Periodic visit, established patient, age 65 and older',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$99.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Individual, approx 15 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$31.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Individual, approx 30 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$40.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Individual, approx 45 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Individual, approx 60 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$79.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Group, approx 30 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$28.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Preventive medicine counseling, Group, approx 60 minutes',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$57.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': 'N/A',\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Hearing Aids',\n", + " 'REIMBURSEMENT_FLAT_FEE': 'N/A',\n", + " 'REIMBURSEMENT_RATE': '80%',\n", + " 'FULL_METHODOLOGY': \"IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price.\",\n", + " 'page_num': '5'},\n", + " {'SERVICE': 'Network Access and Administrative Fees for Medicare Advantage members',\n", + " 'REIMBURSEMENT_FLAT_FEE': '$5.00',\n", + " 'REIMBURSEMENT_RATE': 'N/A',\n", + " 'FULL_METHODOLOGY': \"Healthfirst shall compensate IPA $5.00 per Member per month for the following administrative services,: Access to IPA's network of Participating IPA Providers Credentialing of Participating IPA Providers Reporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst Administrative services relating to quality improvement and Enrollee satisfaction Education of Participating IPA Providers regarding Healthfirst's policies and procedures and quality improvement programs.\",\n", + " 'page_num': '5'}]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "answer_dicts" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "def BOTTOM_UP_CODES(d, page):\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + "\n", + "The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\n", + "Service: {d['SERVICE']}\n", + "Methodology: {d['FULL_METHODOLOGY']}\n", + "\n", + "Your job is to identify codes associated with this specific term.\n", + "\n", + "Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\n", + "'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility.\n", + "'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\n", + "'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar.\n", + "'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options.\n", + "'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service.\n", + "'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\n", + "'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes.\n", + "'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status.\n", + "\n", + "Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\n", + "\n", + "Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.\n", + "\"\"\"" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Document Index\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 1\\nFifth Amendment to the Agreement by and between 1Managed Health, Inc. and Independent Practice Association 1\\nWITNESSETH: 1\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 2\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 3\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 4\\nEXHIBIT C 4\\nCompensation Schedule 4\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 5\\nd. The following preventive care services will be reimbursed at the rates noted below: 5\\nIPA Network Access and Administrative Fees. 5\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 6\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 7\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 8\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 9\\nWITNESSETH: 9\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 10\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 11\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 12\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 13\\nCOMPENSATION SCHEDULE 13\\nReimbursement to IPA Providers 13\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 14\\nd. The following services will be reimbursed at the rates noted below: 14\\nIPA Network Access and Administrative Fees. 14\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 15\\nAdditional Compensation to IPA in the form of Surplus Distribution. 15\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 16\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 17\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 18\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 19\\nFifth Amendment to the Agreement by and between 19Managed Health, Inc. and Independent Practice Association 19\\nWITNESSETH: 19\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 20\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 21\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 22\\nEXHIBIT C 22\\nCompensation Schedule 22\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 23\\nd. The following preventive care services will be reimbursed at the rates noted below: 23\\nIPA Network Access and Administrative Fees. 23\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 24\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 25\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 26\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 27\\nSecond Amendment to the Agreement 27\\nby and between 27Healthfirst PHSP, Inc. and Independent Practice Association 27\\nWITNESSETH: 27\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 28\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 29\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 30\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 31\\nCOMPENSATION SCHEDULE 31\\nReimbursement to IPA Providers 31\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 32\\nd. The following services will be reimbursed at the rates noted below: 32\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 33\\nAdditional Compensation to IPA in the form of Surplus Distribution. 33\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 34\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 35\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE 36\\n\\n\\n\\nStart of Page No. = 1\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nFifth Amendment to the Agreement by and between\\nManaged Health, Inc. and Independent Practice Association\\nThis Amendment to the Agreement between Managed Health, Inc. (\"Healthfirst\"), and Chinese\\nAmerican Independent Practice Association, Inc. (\"Provider\") is effective January 1, 2015.\\nWITNESSETH:\\nWHEREAS, Healthfirst and Provider have entered into an Agreement (the \"Agreement\") for the\\nprovision of Health Care Services to Enrollees; and\\nWHEREAS, Healthfirst and Provider desire to amend that Agreement to delegate credentialing\\nto Provider :\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and Provider hereby agree to amend the Agreement as follows:\\n1.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective or at the time of Healthfirst\\'s final approval of the credentialing process,\\nwhichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 2\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 3\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n2.\\nA new Exhibit C setting forth the reimbursement to be paid by MHI to IPA and IPA\\nProviders for the provision of Covered Services to Enrollees, a copy of which is attached to this\\nAmendment, added to the Agreement. The reimbursement set forth in the Exhibit C attached to this\\namendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits C and the Letter\\nof Agreement effective July 1, 2006 shall continue to apply to dates of service prior to that date. This\\nAmendment supersedes the July 1, 2006 Letter of Agreement setting forth an administrative fee.\\n3.\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nIN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective\\nDate above.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above.\\n<<<<<<{\\'Managed Health, Inc.\\': [\\'By:\\', \\'Name: Print Name\\', \\'Title:\\', \\'Date:\\'], \\'Provider: Chinese American Independent Practice Association, Inc.\\': [\\'By:\\', \\'Name: Print Name\\', \\'Title:\\', \\'Date:\\']}\\n-------Table End--------\\n\\nStart of Page No. = 4\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT C\\nCompensation Schedule\\nReimbursement to IPA Providers\\nI.\\nMedicare and Commercial Enrollees\\n1.\\nExcept for the Covered Services set forth below, MHI shall directly reimburse IPA Providers\\nwho provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum\\nallowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\")\\nfor payments under the Medicare program.\\n2.\\nMHI shall reimburse IPA Providers for the following Covered Services as follows:\\na. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a\\nmulti-specialty office or facility. If provided by an individually-operated Physical Therapist office\\n(solely physical therapy services), physical therapy services set forth below shall be reimbursed at a\\nrate of $65 per service. These services, which shall be updated from time to time, shall include:\\nb. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a\\nfee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee\\nschedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program, except for physical therapy services which will be reimbursed as\\nnoted in \"a.\" above.\\nC. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office\\n(as mandated by the New York State Department of Health), will be reimbursed on a fee-for-\\nservice basis in an amount equal to 100% of the maximum allowable Region I fee schedule\\nestablished by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the\\nMedicare program.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include:\\n<<<{\\'Code Range\\': [\\'97001-97006\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program.\\n<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsies\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 5\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following preventive care services will be reimbursed at the rates noted below:\\ne.\\nHearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent\\n(80%) of the manufacturer\\'s suggested retail price.\\nII.\\nWhere rates in any fee schedule described in this Agreement make reference to Medicare or\\nMedicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and\\nMedicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates,\\nshall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon\\nforty five calendar days following the date on which CMS or SDOH publishes such rates.\\nIPA Network Access and Administrative Fees.\\nFor Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month\\nfor the following administrative services,:\\nAccess to IPA\\'s network of Participating IPA Providers\\nCredentialing of Participating IPA Providers\\nReporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst\\nAdministrative services relating to quality improvement and Enrollee satisfaction\\nEducation of Participating IPA Providers regarding Healthfirst\\'s policies and procedures and\\nquality improvement programs.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following preventive care services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code\\': [\\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 6\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Sharing with IPA\\n1\\nDefinitions. For the purposes of this Agreement and Compensation Schedule, the following\\nterms shall have the indicated meanings:\\n1.1\\n\"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees,\\nadjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such\\nadjustment is reasonably applied by Healthfirst.\\n1.2\\n\"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable\\nreserve for incurred but not received (\"IBNR\") claims for Risk Services.\\n1.3\\n\"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target\\nfollowing Reconciliation.\\n1.4\\n\"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target.\\n1.5\\n\"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine\\nSurpluses and Deficits.\\n1.6\\n\"Risk Enrollee\" shall mean Enrollees in Healthfirst\\'s Medicare Advantage plans who\\nselect an IPA Provider as their Primary Care Provider.\\n1.7\\n\"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of\\nwhether such Services were provided by IPA Providers or other health care providers.\\n1.8\\n\"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums\\n1.9\\n\"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target\\nfollowing Reconciliation.\\n1.10\\n\"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target.\\n2\\nRisk Sharing.\\n2.1\\nRisk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk\\nServices rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk\\nServices provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall\\nassume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst\\na portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration\\nof Healthfirst\\'s utilization and case management programs, the sharing of utilization data, and education\\nof IPA Providers regarding utilization trends and the provision of quality and medically appropriate\\nservices.\\n2.2\\nLimited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services\\nother than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 7\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nbe required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or\\nhealth care provider. In all instances, including any termination or expiration of this Agreement,\\nHealthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk\\nServices, to IPA Providers and other health care providers.\\n3\\nRegulatory Approval.\\nThe parties understand and acknowledge that this risk sharing\\narrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency\\nhaving jurisdiction over this Agreement as required by applicable statutes and regulation. No payments\\nshall be made by either party, prior to receiving such regulatory approval. The failure to obtain\\nregulatory shall not be grounds for either party to terminate this Agreement which shall otherwise\\nremain in effect.\\n4\\nQuarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each\\ncalendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding\\nquarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the\\nmeasurement period shall commence on the later of a) January 1, 2010 or b) the date of any required\\nregulatory approval.\\n5\\nRisk Sharing; Distribution of Surplus and Payment of Deficits.\\n5.1\\nSurpluses. In the event that Healthfirst determines that a Surplus exists following\\nReconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal\\nto the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Upside Risk Limit.\\n5.2\\nDeficits.\\n5.2.1 In the event that Healthfirst determines that a Deficit exists following\\nReconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal\\nto the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Downside Risk Limit.\\n5.2.2\\nIn the event that IPA fails to timely pay any amount due under Section 5.2.1, any\\nsuch amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due\\nto IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against\\nfuture amounts due to IPA in the succeeding calendar year quarter.\\n5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless\\nof the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such\\ntermination, expiration or non-renewal.\\n6\\nCompliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may\\nuse any compensation methodology to reimburse IPA Providers for Health Care Services; provided,\\nhowever, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or\\namount, either separately or in combination with any compensation formulas which would place IPA\\nProviders at substantial financial risk for the provision of referral services, as determined by the\\napplicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 8\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts\\npaid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA\\'s compliance with the\\nfederal PIP regulations.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 9\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nSecond Amendment to the Agreement\\nby and between\\nHealthfirst PHSP, Inc. and Independent Practice Association\\nThis Amendment shall be effective January 1, 2015 by and between Chinese American\\nIndependent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\").\\nWITNESSETH:\\nWHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of\\nCovered Services to certain Healthfirst Enrollees; and\\nWHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation\\nwhich Healthfirst pays to IPA.\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and IPA hereby agree to amend the Agreement as follows:\\n1.\\nSection 2.1 of the Agreement titled Provision of Health Care Services is deleted in its\\nentirety and replaced Section 2.1.\\n2.1\\nProvision of Health Care Services. IPA shall arrange for IPA Providers to provide Health\\nCare Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the\\napplicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer\\nparticipation with Healthfirst pursuant to this Agreement to any health care provider who is a\\nmember of IPA or has a contract with IPA, nor disclose share the terms and conditions of this\\nAgreement included rates of reimbursement, without first obtaining the written consent of\\nHealthfirst. Healthfirst shall have the right to refuse the participation of any particular health care\\nprovider under this Agreement. Healthfirst\\'s determination to offer participation to any health care\\nprovider pursuant to this Agreement or to refuse to accept any health care provider for participation\\nshall be based on Healthfirst\\'s business needs as determined solely by Healthfirst. Healthfirst\\'s\\nVice-President, Network Management or Executive Vice-President/Chief Operating Officer shall\\nhave sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the\\nname, addresses, telephone numbers and other information regarding IPA Providers reasonably\\nrequested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use\\nsuch information in Healthfirst advertising and provider directories.\\n2.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n1 of 10\\n\\nStart of Page No. = 10\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective Date or at the time of Healthfirst\\'s final approval of the credentialing\\nprocess, whichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n2 of 10\\n\\nStart of Page No. = 11\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted\\nor\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n3\\nA new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA\\nProviders for the provision of Health Care Services to Enrollees and IPA for the provision of certain\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n3 of 10\\n\\nStart of Page No. = 12\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nadministrative services, a copy of which is attached to this Amendment, is added to the Agreement.\\nThe reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of\\nservice on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service\\nprior to that date.\\n4\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n4 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n 4 All other terms and conditions of the Agreement shall remain in full force and effect.\\n<<<<<<{\\'Healthfirst PHSP, Inc.\\': [\\'By: Name: Print Name\\', \\'Title:\\', \\'Date: -\\'], \\'Chinese American Independent Practice Association, Inc.\\': [\\'By: Name: Print Name\\', \\'Title:\\', \\'Date:\\']}\\n-------Table End--------\\n\\nStart of Page No. = 13\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT 4.1\\nCOMPENSATION SCHEDULE\\nReimbursement to IPA Providers\\n1.\\nExcept for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA\\nProviders who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the\\nmaximum allowable Region I fee schedule established by the Center for Medicaid and Medicare\\nServices (\"CMS\") for payments under the Medicare program.\\n2.\\nHealthfirst shall reimburse IPA Providers for the Health Care Services set forth below as\\nfollows:\\na. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of\\n$45 per service if provided in a multi-specialty office or facility. If provided by an individually-\\noperated Physical Therapist office (solely physical therapy services), physical therapy services set\\nforth below shall be reimbursed at a rate of $60 per service.\\nb.\\nPhysical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall\\nbe reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable\\nRegion I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for\\npayments under the Medicare program, except for physical therapy services which will be\\nreimbursed as noted in \"a.\" above.\\nC.\\nGastroenterology Services. The following services, when rendered in an \"Office Based Surgery\"\\n(OBS) accredited office (as mandated by the New York State Department of Health), will be\\nreimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region\\nI fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n5 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'Code Range\\': [\\'97001-97004\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsy\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 14\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%)\\nof the manufacturer\\'s suggested retail price.\\nIPA Network Access and Administrative Fees.\\nHealthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative\\nservices relating to quality improvement and education of Participating IPA Providers regarding\\nHealthfirst\\'s quality improvement programs. IPA and Healthfirst understand and agree that IPA shall\\nnot provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant\\nto this Agreement. In the event that IPA or an affiliated of IPA provides management services, the\\nparties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n6 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code/Modifier\\': [\\'99205-P1\\', \\'99212-P2\\', \\'99215-P3\\', \\'59409\\', \\'59514\\', \\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'PCAP Initial Visit\\', \\'PCAP Prenatal Visit\\', \\'PCAP Post partum Visit\\', \\'Vaginal Delivery\\', \\'Cesarean Delivery\\', \\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$196.07\\', \\'$130.00\\', \\'$163.00\\', \\'$1500.00\\', \\'$1500.00\\', \\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 15\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nAdditional Compensation to IPA in the form of Surplus Distribution.\\n1.\\nLimited Risk Transfer/No Transfer of Financial Risk\\n1.1.\\nIPA shall be entitled to Additional Compensation under this Agreement on an annual\\nbasis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus\\nfunds exist, as specified below.\\n1.2.\\nUse of Additional Compensation. IPA represents and warrants that Additional\\nCompensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a\\nmanner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On\\nrequest by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds\\nreceived from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d).\\n2.\\nDefinitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this\\nExhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the\\nAgreement to which this Exhibit is attached.\\n2.1.\\n\"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant\\nto Section 5 of this Exhibit 4.1(b).\\n2.2.\\n\"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk\\nEnrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan\\nContract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst.\\n2.3.\\n\"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to\\nHealthfirst\\'s administrative costs as reasonably determined by Healthfirst.\\n2.4.\\n\"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of\\nMedical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or\\nDeficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit.\\n2.5.\\n\"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the\\nAdministrative Deduction and any Quality Incentive Deduction.\\n2.6.\\n\"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for\\nprograms in which IPA and IPA Providers participate to improve the quality of Health Care Services\\nwhich Risk Enrollees receive.\\n2.7.\\n\"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical\\nRevenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits.\\n2.8.\\n\"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary\\ncare provider who is a Participating IPA Provider.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n7 of 10\\n\\nStart of Page No. = 16\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.9.\\n\"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following\\nReconciliation\\n3.\\nReconciliation of Medical Revenue to Determine Surpluses and Deficits. During each\\nReconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine\\nthe amount of any Surplus of Deficit:\\n3.1.\\nthe costs of claims paid for Health Care Services rendered to Risk Enrollees by\\nParticipating IPA Providers, Participating Providers and all other health care providers.\\n3.2.\\na\\nreasonable reserve for the cost of claims for Health Care Services rendered to Risk\\nEnrollees but not yet received;\\n3.3.\\nother costs associated with the provision of Health Care Services to Risk Enrollees\\nincluding but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs.\\n3.4.\\naccruals for any risk adjustment applicable to Adjusted Premiums.\\n4.\\nReconciliation Schedule.\\n4.1.\\nInterim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days\\n(120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus\\nor Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines\\nthat a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall\\nmake an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the\\nevent that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the\\ncalendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the\\nDeficit shall be handled as set forth in Section 6 below.\\n4.2.\\nAnnual Reconciliation and Distribution. Within one-hundred and twenty days (120) after\\nthe end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for\\nthat calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that\\ncalendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in\\nSection 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that\\nCalendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit\\nshall be handled as set forth in Section 6 below.\\n5.\\nPayment of Additional Compensation. In the event that Healthfirst determines that a Surplus has\\nbeen achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which\\nshall be the lesser of the following:\\n5.1.\\nTen percent (10%) of the Surplus for the calendar year, or\\n5.2.\\nAn amount such that the Additional Compensation equals twenty-five percent (25%) of\\nthe sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n8 of 10\\n\\nStart of Page No. = 17\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and\\niii) the Additional Compensation.\\n5.3.\\nHealthfirst shall pay any Additional Compensation, within thirty days of each\\nReconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation.\\n6.\\nTreatment of Deficits.\\n6.1.\\nDeficits Rolled Forward. In the event that a Deficit remains following any Reconciliation\\nor a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be\\nrolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar\\nyear.\\n6.2.\\nDeficits at Termination, Expiration or Non-Renewal. In the event that this Agreement\\nterminates, is not renewed or expires, regardless of the reason, Provider shall not be required to\\nreimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or\\nnon-renewal.\\n7.\\nCompliance with Regulatory Provisions Regarding Risk Transfer\\n7.1.\\nSDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with\\nthe\\napplicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH\\'s\\nProvider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount\\nfinancial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting\\nGuidelines, IPA shall demonstrate financial viability and establish a financial security deposit as\\nrequired by SDOH.\\n7.2.\\nPhysician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the\\napplicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the\\nplacement of stop loss insurance if applicable.\\n7.3.\\nFinancial Stability Monitoring and Reporting. Notwithstanding any other provision of\\nthis Agreement Healthfirst shall regularly monitor Provider\\'s financial condition to ensure that Provider\\nremains financially able to perform its obligations under this Agreement and meet any financial\\nsolvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material\\ndevelopments which may impair its ability to perform its obligations under this Agreement. In addition,\\nIPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e),\\nwhich shall include but not be limited to IPA\\'s quarterly and annual financial statements. IPA shall also\\nmaintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received\\nfrom Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and\\ndisbursements to Healthfirst on request.\\n8.\\nTermination or Amendment of Additional Compensation\\nHealthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to\\nwhich Additional Compensation is determined and paid to IPA as well as eliminate the payment of\\nAdditional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n9 of 10\\n\\nStart of Page No. = 18\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nthe Agreement such amendment or termination shall not change any Additional Compensation\\ncalculated or paid prior to the effective date of such amendment or termination.\\n9.\\nTermination, Non-Renewal or Expiration of the Agreement.\\nFor a period of twelve months following any termination, non-renewal or expiration of this Agreement,\\nregardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this\\nAgreement until the Final Reconciliation; provided, however, that payments to IPA of Additional\\nCompensation shall be suspended. These Reconciliations shall be on based Health Care Services\\nrendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement.\\nHealthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty\\ndays of the Final Reconciliation.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n10 of 10\\n\\nStart of Page No. = 19\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nFifth Amendment to the Agreement by and between\\nManaged Health, Inc. and Independent Practice Association\\nThis Amendment to the Agreement between Managed Health, Inc. (\"Healthfirst\"), and Chinese\\nAmerican Independent Practice Association, Inc. (\"Provider\") is effective January 1, 2015.\\nWITNESSETH:\\nWHEREAS, Healthfirst and Provider have entered into an Agreement (the \"Agreement\") for the\\nprovision of Health Care Services to Enrollees; and\\nWHEREAS, Healthfirst and Provider desire to amend that Agreement to delegate credentialing\\nto Provider :\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and Provider hereby agree to amend the Agreement as follows:\\n1.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective or at the time of Healthfirst\\'s final approval of the credentialing process,\\nwhichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 20\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 21\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted or\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n2.\\nA new Exhibit C setting forth the reimbursement to be paid by MHI to IPA and IPA\\nProviders for the provision of Covered Services to Enrollees, a copy of which is attached to this\\nAmendment, added to the Agreement. The reimbursement set forth in the Exhibit C attached to this\\namendment shall apply to dates of service on and after January 1, 2015. Prior Exhibits C and the Letter\\nof Agreement effective July 1, 2006 shall continue to apply to dates of service prior to that date. This\\nAmendment supersedes the July 1, 2006 Letter of Agreement setting forth an administrative fee.\\n3.\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nIN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective\\nDate above.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n IN WITNESS WHEREOF, the parties hereto have executed this Agreement as of the Effective Date above.\\n<<<<<<{\\'Managed Health, Inc.\\': [\\'DocuSigned by: By: Paul E. Portsmore\\', \\'441CEAD8E502476 Name: Paul E. Portsmore Print Name\\', \\'Title: SVP\\', \\'Date: 12/31/2014 18:32:28 ET\\'], \\'Provider: Chinese American Independent Practice Association, Inc.\\': [\\'By: family\\', \\'Name: PEGGY SHENG Print Name\\', \\'Chief Operating Officer Title:\\', \\'Date: Dec. 26, 2015\\']}\\n-------Table End--------\\n\\nStart of Page No. = 22\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT C\\nCompensation Schedule\\nReimbursement to IPA Providers\\nI.\\nMedicare and Commercial Enrollees\\n1.\\nExcept for the Covered Services set forth below, MHI shall directly reimburse IPA Providers\\nwho provide Covered Services on a fee-for-service basis in an amount equal to 100% of the maximum\\nallowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\")\\nfor payments under the Medicare program.\\n2.\\nMHI shall reimburse IPA Providers for the following Covered Services as follows:\\na. Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a\\nmulti-specialty office or facility. If provided by an individually-operated Physical Therapist office\\n(solely physical therapy services), physical therapy services set forth below shall be reimbursed at a\\nrate of $65 per service. These services, which shall be updated from time to time, shall include:\\nb. Services provided by Physical Medicine and Rehabilitation specialists shall be reimbursed on a\\nfee-for-service basis in an amount equal to 70% of the maximum allowable Region I fee\\nschedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program, except for physical therapy services which will be reimbursed as\\nnoted in \"a.\" above.\\nC. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office\\n(as mandated by the New York State Department of Health), will be reimbursed on a fee-for-\\nservice basis in an amount equal to 100% of the maximum allowable Region I fee schedule\\nestablished by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the\\nMedicare program.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include:\\n<<<{\\'Code Range\\': [\\'97001-97006\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>\\n Physical Therapy services shall be reimbursed at a flat rate of $55 per date of service if provided in a multi-specialty office or facility. If provided by an individually-operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $65 per service. These services, which shall be updated from time to time, shall include: C. The following services, when rendered in an \"Office Based Surgery\" (OBS) accredited office (as mandated by the New York State Department of Health), will be reimbursed on a fee-for- service basis in an amount equal to 100% of the maximum allowable Region I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments under the Medicare program.\\n<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsies\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 23\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following preventive care services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent\\n(80%) of the manufacturer\\'s suggested retail price.\\nII.\\nWhere rates in any fee schedule described in this Agreement make reference to Medicare or\\nMedicaid rates, any Medicare or Medicaid rates enacted by the federal Centers for Medicare and\\nMedicaid Services (\"CMS\") or the New York State Department of Health (\"SDOH\"), including updates,\\nshall apply to dates of service occurring on the later of (a) the effective date of such rates or (b) upon\\nforty five calendar days following the date on which CMS or SDOH publishes such rates.\\nIPA Network Access and Administrative Fees.\\nFor Medicare Advantage members only, Healthfirst shall compensate IPA $5.00 per Member per month\\nfor the following administrative services,:\\nAccess to IPA\\'s network of Participating IPA Providers\\nCredentialing of Participating IPA Providers\\nReporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst\\nAdministrative services relating to quality improvement and Enrollee satisfaction\\nEducation of Participating IPA Providers regarding Healthfirst\\'s policies and procedures and\\nquality improvement programs.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following preventive care services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code\\': [\\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 24\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Sharing with IPA\\n1\\nDefinitions. For the purposes of this Agreement and Compensation Schedule, the following\\nterms shall have the indicated meanings:\\n1.1\\n\"Adjusted Premiums\" shall mean premiums Healthfirst receives for Risk Enrollees,\\nadjusted by any risk score or premium adjustment published by CMS, SDOH or other payor as such\\nadjustment is reasonably applied by Healthfirst.\\n1.2\\n\"Cost of Risk Services\" shall mean the the cost of claims paid for Risk plus a reasonable\\nreserve for incurred but not received (\"IBNR\") claims for Risk Services.\\n1.3\\n\"Deficit\" shall be the amount that the Cost of Risk Services exceeds the Risk Target\\nfollowing Reconciliation.\\n1.4\\n\"Upside Risk Limit\" shall be equal to seventy eight percent (78__%) of Risk Target.\\n1.5\\n\"Reconciliation\" shall mean the quarterly reconciliation by Healthfirst to determine\\nSurpluses and Deficits.\\n1.6\\n\"Risk Enrollee\" shall mean Enrollees in Healthfirst\\'s Medicare Advantage plans who\\nselect an IPA Provider as their Primary Care Provider.\\n1.7\\n\"Risk Services\" shall mean all Health Care Services provided to Enrollees regardless of\\nwhether such Services were provided by IPA Providers or other health care providers.\\n1.8\\n\"Risk Target\" shall be equal to eighty three percent (83%) of Adjusted Premiums\\n1.9\\n\"Surplus\" shall be the amount that Cost of Risk Services is less than the Risk Target\\nfollowing Reconciliation.\\n1.10\\n\"Downside Risk Limit\" shall mean one hundred threepercent (103__%) of Risk Target.\\n2\\nRisk Sharing.\\n2.1\\nRisk Sharing and Cooperation. Healthfirst and IPA shall share the financial risk for Risk\\nServices rendered to Risk Enrollees as set forth below. IPA shall assume upside risk for the cost of Risk\\nServices provided to Risk Enrollees to the extent that IPA may receive portion of any Surplus. IPA shall\\nassume downside risk for the cost of Risk Services to the extent that IPA is required to repay Healthfirst\\na portion of any Deficit. Healthfirst and IPA shall cooperate in the implementation and administration\\nof Healthfirst\\'s utilization and case management programs, the sharing of utilization data, and education\\nof IPA Providers regarding utilization trends and the provision of quality and medically appropriate\\nservices.\\n2.2\\nLimited Risk Transfer. IPA shall not assume risk for the cost of Health Care Services\\nother than through the receipt of Surpluses and payment of Deficits. Under no circumstances shall IPA\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 25\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nbe required to pay claims for Health Care Services, including Risk Services, to any IPA Provider or\\nhealth care provider. In all instances, including any termination or expiration of this Agreement,\\nHealthfirst shall remain solely responsible for paying claims for Health Care Services, including Risk\\nServices, to IPA Providers and other health care providers.\\n3\\nRegulatory Approval.\\nThe parties understand and acknowledge that this risk sharing\\narrangement may be subject to the prior review and approval of CMS, SDOH or other regulatory agency\\nhaving jurisdiction over this Agreement as required by applicable statutes and regulation. No payments\\nshall be made by either party, prior to receiving such regulatory approval. The failure to obtain\\nregulatory shall not be grounds for either party to terminate this Agreement which shall otherwise\\nremain in effect.\\n4\\nQuarterly Reconciliation. Within one-hundred and twenty days (120) after the end of each\\ncalendar year quarter, Healthfirst shall determine the amount of the Surplus or Deficit, for all preceding\\nquarters during the applicable calendar year. For purposes of calculating Surplus and Deficits, the\\nmeasurement period shall commence on the later of a) January 1, 2010 or b) the date of any required\\nregulatory approval.\\n5\\nRisk Sharing; Distribution of Surplus and Payment of Deficits.\\n5.1\\nSurpluses. In the event that Healthfirst determines that a Surplus exists following\\nReconciliation Healthfirst shall, within thirty days of the Reconciliation, make a payment to IPA equal\\nto the lesser of i) fifty percent (50%) of the Surplus or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Upside Risk Limit.\\n5.2\\nDeficits.\\n5.2.1 In the event that Healthfirst determines that a Deficit exists following\\nReconciliation, IPA shall, within thirty days of the Reconciliation, make a payment to Healthfirst equal\\nto the lesser of i) fifty percent (50%) of the Deficit or ii) fifty percent (50%) of the difference between\\nthe Risk Target and the Downside Risk Limit.\\n5.2.2\\nIn the event that IPA fails to timely pay any amount due under Section 5.2.1, any\\nsuch amounts shall be offset against up to one hundred percent (100%) percent of surplus amounts due\\nto IPA by Managed Health, Inc. Any amounts remaining following such offset will be offset against\\nfuture amounts due to IPA in the succeeding calendar year quarter.\\n5.2.3 In the event that this Agreement terminates, is not renewed or expires, regardless\\nof the reason, IPA shall reimburse Healthfirst for any Deficit amounts due Healthfirst at the time of such\\ntermination, expiration or non-renewal.\\n6\\nCompliance with Physician Incentive Plan (\"PIP\") Regulations. IPA, in its sole discretion, may\\nuse any compensation methodology to reimburse IPA Providers for Health Care Services; provided,\\nhowever, that no payments shall be made by IPA to IPA Providers who are physicians in a manner or\\namount, either separately or in combination with any compensation formulas which would place IPA\\nProviders at substantial financial risk for the provision of referral services, as determined by the\\napplicable federal PIP regulations contained in 42 CFR 417.479. IPA shall provide information\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 26\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably requested by Healthfirst establishing i) the manner and extent to which any Surplus amounts\\npaid to IPA pursuant to this Agreement are paid to IPA Providers and ii) IPA\\'s compliance with the\\nfederal PIP regulations.\\nFED. TAX ID: 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\n\\nStart of Page No. = 27\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nSecond Amendment to the Agreement\\nby and between\\nHealthfirst PHSP, Inc. and Independent Practice Association\\nThis Amendment shall be effective January 1, 2015 by and between Chinese American\\nIndependent Practice Association, Inc. (\"IPA\") and Healthfirst PHSP, Inc. (\"Healthfirst\").\\nWITNESSETH:\\nWHEREAS, Healthfirst and IPA entered into an Agreement on July 1, 2010, for the provision of\\nCovered Services to certain Healthfirst Enrollees; and\\nWHEREAS, Healthfirst and IPA desire to amend that Agreement to change the compensation\\nwhich Healthfirst pays to IPA.\\nNOW, THEREFORE, in consideration of the mutual covenants and agreements herein\\ncontained, Healthfirst and IPA hereby agree to amend the Agreement as follows:\\n1.\\nSection 2.1 of the Agreement titled Provision of Health Care Services is deleted in its\\nentirety and replaced Section 2.1.\\n2.1\\nProvision of Health Care Services. IPA shall arrange for IPA Providers to provide Health\\nCare Services to Enrollees pursuant to the terms of this Agreement, the Provider Manual, and the\\napplicable Plan Contract(s) for those Plan(s) identified by IPA in Exhibit 2.1. IPA shall not offer\\nparticipation with Healthfirst pursuant to this Agreement to any health care provider who is a\\nmember of IPA or has a contract with IPA, nor disclose share the terms and conditions of this\\nAgreement included rates of reimbursement, without first obtaining the written consent of\\nHealthfirst. Healthfirst shall have the right to refuse the participation of any particular health care\\nprovider under this Agreement. Healthfirst\\'s determination to offer participation to any health care\\nprovider pursuant to this Agreement or to refuse to accept any health care provider for participation\\nshall be based on Healthfirst\\'s business needs as determined solely by Healthfirst. Healthfirst\\'s\\nVice-President, Network Management or Executive Vice-President/Chief Operating Officer shall\\nhave sole authority issue consent under this Section 2.1. IPA shall provide Healthfirst with the\\nname, addresses, telephone numbers and other information regarding IPA Providers reasonably\\nrequested by Healthfirst. IPA shall permit, and require IPA Providers to permit, Healthfirst to use\\nsuch information in Healthfirst advertising and provider directories.\\n2.\\nA new Section 2.8 entitled Credentialing of Affiliated Providers is added to the\\nAgreement as follows:\\n2.8\\nCredentialing of Affiliated Providers.\\n2.8.1 Delegation. Healthfirst delegates to Provider and Provider agrees to accept the\\nresponsibility for credentialing potential Affiliated Providers employed by or affiliated with Provider;\\nprovided however, that such delegation shall be limited to i) those categories and types of providers\\nwhich Provider credentials as part of Provider\\'s regular credentialing process and ii) those providers\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n1 of 10\\n\\nStart of Page No. = 28\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nwho have or seek medical staff privileges with Provider. Provider shall not be required to credential\\nParticipating Providers which provider does not regularly credential or which do not have or seek\\nmedical staff privileges with Provider. Healthfirst shall retain sole responsibility for credentialing such\\nParticipating Providers. Healthfirst acknowledges and agrees that Healthfirst has reviewed Provider\\'s\\ncredentialing process and that such process meets the requirements of this Agreement, including this\\nSection 2.8 as of the Effective Date or at the time of Healthfirst\\'s final approval of the credentialing\\nprocess, whichever is later.\\n2.8.2 Provider shall have a formal application and review process for such Providers\\nwhich includes, but is not limited to, a signed application and attestation. Such review shall be\\nconducted by a Provider credentials committee in consultation with the Healthfirst credentials\\ncommittee. Provider shall have Credentialing Policies and Procedures that defines the following:\\n2.8.2.1 scope of Affiliated Providers covered;\\n2.8.2.2 criteria for primary source verification of information;\\n2.8.2.3 the process used to make credentialing decisions;\\n2.8.2.4 the extent of any delegated credentialing/recredentialing arrangements;\\n2.8.2.5 the right of Affiliated Providers to review the information submitted in\\nsupport of their credentialing application;\\n2.8.2.6 the process of notification to a Affiliated Provider of any information\\nobtained during the credentialing process that varies substantially from the information provided by the\\nAffiliated Provider;\\n2.8.2.7 the Affiliated Provider\\'s right to correct erroneous information;\\n2.8.2.8 the Provider\\'s medical director\\'s direct responsibility and participation in\\nthe credentialing program; and\\n2.8.2.9 the process used to ensure confidentiality of all information obtained in\\nthe credentialing process, except otherwise provided by law.\\n2.8.3 Credentialing Criteria. The Provider\\'s Credentials Committee shall determine the\\ncriteria for selection, which shall be consistent with Healthfirst\\'s credentialing policies and procedures,\\nas may be modified by Healthfirst from time to time. Provider\\'s credentialing and recredentialing\\ncriteria and procedure, its procedure for selection and termination of Providers, and its Provider\\ngrievance mechanism shall be subject to review and approval by Healthfirst, which approval shall not be\\nunreasonably withheld. At a minimum, Provider\\'s credentialing and recredentialing criteria and\\nprocedure shall be consistent with The Joint Commission (\"TJC\") and applicable standards required\\nunder Article 28 of the New York State Public Health law and shall include requiring Affiliated\\nProviders to be licensed and qualified to provide the services specified and to maintain such license in\\ngood standing. Provider shall include further credentialing requirements in addition to the TJC\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n2 of 10\\n\\nStart of Page No. = 29\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nreasonably required by Healthfirst and acceptable to Provider. Notwithstanding any provision\\nforegoing, Healthfirst shall retain final authority concerning the credentialing of Affiliated Providers and\\nhis/her participation in the Healthfirst network.\\n2.8.4 Cooperation. Provider shall cooperate with Healthfirst to determine if the\\ncredentialing and recredentialing criteria and procedures used by Provider are consistent with\\nHealthfirst\\'s and TJC\\'s credentialing and recredentialing criteria and procedures. In the event that\\nProvider\\'s credentialing and recredentialing criteria and procedures are inconsistent with Healthfirst\\'s\\ncredentialing and recredentialing criteria and procedures then Healthfirst, after good faith negotiations\\nwith Provider, may adjust the compensation payable to Provider under this Agreement to compensate\\nHealthfirst for the reasonable cost of credentialing and recredentialing potential Affiliated Providers\\nemployed by or affiliated with Provider.\\n2.8.5 Oversight by Healthfirst. To allow Healthfirst to oversee and monitor the\\ndelegation of credentialing to Provider, Provider, upon the reasonable request of Healthfirst and on an\\nongoing basis, shall (i) permit Healthfirst to conduct on-site audits of Provider credentialing policies and\\nprocedures and credentialing files, upon reasonable notice; (ii) to provide to Healthfirst no less than\\nannually, rosters of Healthfirst Providers and such written verification or other substantiation as\\nrequested by Healthfirst that Affiliated Providers employed by or on the medical staff of Provider are\\ncurrently and fully credentialed in accordance with TJC and Healthfirst standards; (iii) to provide to\\nHealthfirst, upon request, copies of credentialing files excluding peer review documentation; and, (iv)\\nand to provide Healthfirst with Affiliated Providers\\' initial appointment date, reappointment start date\\nand reappointment end date. Provider shall retain the right to terminate immediately its agreements\\nwith, or the employment of, any Affiliated Provider, subject to applicable appeals, if the Affiliated\\nProvider is no longer a member of the Provider\\'s medical staff, if the license to practice or DEA permit\\nor any controlled substances registration of said Affiliated Provider is suspended, otherwise restricted\\nor\\nrevoked, or if said Affiliated Provider is excluded or terminated from either the Medicare or Medicaid\\nprograms, or if said Affiliated Provider is not covered by insurance as provided pursuant to this\\nAgreement or is convicted of any crime (either a misdemeanor or a felony). Provider shall notify\\nHealthfirst, immediately, but in any event within forty-eight (48) hours of receipt of notice, if any\\nAffiliated Provider employed by or affiliated with Provider has his or her license to practice or DEA\\npermit or any controlled substances registration suspended or otherwise restricted or revoked or is\\nexcluded or terminated from either the Medicare or Medicaid programs, or is not covered by insurance\\nas required pursuant to this Agreement below or is convicted of any crime related to the provision of\\nhealth care services. Notwithstanding any other provision of this Section to the contrary, it is expressly\\nunderstood and agreed that Provider shall not be responsible for conducting individual site visits at\\nprovider sites other than at facilities operated by Provider.\\n2.8.6 Confidentiality. The foregoing provisions of this Section and the other provisions\\nof this Agreement shall not require the release or other disclosure by any person of any records or other\\ndocuments or information to the extent that such release or other disclosure is prohibited by or otherwise\\ncontrary to any applicable law.\\n3\\nA new Exhibit 4.1 setting forth the reimbursement to be paid by Healthfirst to IPA\\nProviders for the provision of Health Care Services to Enrollees and IPA for the provision of certain\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n3 of 10\\n\\nStart of Page No. = 30\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nadministrative services, a copy of which is attached to this Amendment, is added to the Agreement.\\nThe reimbursement set forth in the Exhibit 4.1 attached to this Amendment shall apply to dates of\\nservice on and after January 1, 2015. Prior Exhibits 4.1 shall continue to apply to dates of service\\nprior to that date.\\n4\\nAll other terms and conditions of the Agreement shall remain in full force and effect.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n4 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n 4 All other terms and conditions of the Agreement shall remain in full force and effect.\\n<<<<<<{\\'Healthfirst PHSP, Inc.\\': [\\'DocuSigned by: By: Paul E. Portsmore\\', \\'441CEAD8E502476. Name: Paul E. Portsmore Print Name\\', \\'Title: SVP -\\', \\'Date: 12/31/2014 I 18:32:28 ET -\\'], \\'Chinese American Independent Practice Association, Inc.\\': [\\'By: f Sear\\', \\'Name: PEGGY SHENG Print Name\\', \\'Title: Chief Operating Officer\\', \\'Date: December 26, 2014\\']}\\n-------Table End--------\\n\\nStart of Page No. = 31\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nEXHIBIT 4.1\\nCOMPENSATION SCHEDULE\\nReimbursement to IPA Providers\\n1.\\nExcept for the Health Care Services set forth below, Healthfirst shall directly reimburse IPA\\nProviders who provide Health Care Services on a fee-for-service basis in an amount equal to 90% of the\\nmaximum allowable Region I fee schedule established by the Center for Medicaid and Medicare\\nServices (\"CMS\") for payments under the Medicare program.\\n2.\\nHealthfirst shall reimburse IPA Providers for the Health Care Services set forth below as\\nfollows:\\na. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of\\n$45 per service if provided in a multi-specialty office or facility. If provided by an individually-\\noperated Physical Therapist office (solely physical therapy services), physical therapy services set\\nforth below shall be reimbursed at a rate of $60 per service.\\nb.\\nPhysical Medicine and Rehabilitation Services. Physical medicine and rehabilitation services shall\\nbe reimbursed on a fee-for-service basis in an amount equal to 70% of the maximum allowable\\nRegion I fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for\\npayments under the Medicare program, except for physical therapy services which will be\\nreimbursed as noted in \"a.\" above.\\nC.\\nGastroenterology Services. The following services, when rendered in an \"Office Based Surgery\"\\n(OBS) accredited office (as mandated by the New York State Department of Health), will be\\nreimbursed on a fee-for-service basis in an amount equal to 100% of the maximum allowable Region\\nI fee schedule established by the Center for Medicaid and Medicare Services (\"CMS\") for payments\\nunder the Medicare program.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n5 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'Code Range\\': [\\'97001-97004\\', \\'97010-97028\\', \\'97032-97039\\', \\'97110-97546\\', \\'97750-97755\\'], \\'Code Range Description\\': [\\'Physical Medicine Assessments\\', \\'Physical Therapy Treatment Modalities: Supervised\\', \\'Physical Therapy Treatment Modalities: Constant Attendance\\', \\'Other Therapeutic Techniques With Direct Patient Contact\\', \\'Physical performance test & Assistive technology assessment\\']}\\n-------Table End--------\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n a. Physical Therapy Services. Physical therapy services set forth below shall be reimbursed at a rate of $45 per service if provided in a multi-specialty office or facility. If provided by an individually- operated Physical Therapist office (solely physical therapy services), physical therapy services set forth below shall be reimbursed at a rate of $60 per service.\\n<<<<<<<<<<<<{\\'CPT Code\\': [\\'43235\\', \\'43239\\', \\'45378\\', \\'45380\\', \\'45385\\'], \\'CPT Description\\': [\\'Upper gastrointestinal endoscopy\\', \\'Upper gastrointestinal endoscopy, with biopsy\\', \\'Colonoscopy, diagnostic\\', \\'Colonoscopy, with biopsy\\', \\'Colonoscopy, removal of tumor\\']}\\n-------Table End--------\\n\\nStart of Page No. = 32\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nd. The following services will be reimbursed at the rates noted below:\\ne. Hearing Aids. IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%)\\nof the manufacturer\\'s suggested retail price.\\nIPA Network Access and Administrative Fees.\\nHealthfirst shall pay IPA an administrative fee of $2.50 per Member per month for administrative\\nservices relating to quality improvement and education of Participating IPA Providers regarding\\nHealthfirst\\'s quality improvement programs. IPA and Healthfirst understand and agree that IPA shall\\nnot provide any management services, as that term is defined in 11 NYCRR 98, to Healthfirst pursuant\\nto this Agreement. In the event that IPA or an affiliated of IPA provides management services, the\\nparties shall execute a separate management agreement approved by SDOH pursuant to 11 NYCRR 98.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n6 of 10\\n\\n\\n-------Table Start--------THIS IS PRETABLE TEXT>>>THIS IS PRETABLE TEXT>>>\\n d. The following services will be reimbursed at the rates noted below:\\n<<<<<<{\\'CPT Code/Modifier\\': [\\'99205-P1\\', \\'99212-P2\\', \\'99215-P3\\', \\'59409\\', \\'59514\\', \\'99381\\', \\'99382\\', \\'99383\\', \\'99384\\', \\'99385\\', \\'99386\\', \\'99387\\', \\'99391\\', \\'99392\\', \\'99393\\', \\'99394\\', \\'99395\\', \\'99396\\', \\'99397\\', \\'99401\\', \\'99402\\', \\'99403\\', \\'99404\\', \\'99411\\', \\'99412\\'], \\'CPT Description\\': [\\'PCAP Initial Visit\\', \\'PCAP Prenatal Visit\\', \\'PCAP Post partum Visit\\', \\'Vaginal Delivery\\', \\'Cesarean Delivery\\', \\'Initial visit, new patient, infant (under 1 year)\\', \\'Initial visit, new patient, early childhood (age 1-4)\\', \\'Initial visit, new patient, late childhood (age 5-11)\\', \\'Initial visit, new patient, adolescent (age 12-17)\\', \\'Initial visit, new patient, age 18-39\\', \\'Initial visit, new patient, age 40-64\\', \\'Initial visit, new patient, age 65 and older\\', \\'Periodic visit, established patient, infant (under 1 year)\\', \\'Periodic visit, established patient, early childhood (age 1-4)\\', \\'Periodic visit, established patient, late childhood (age 5-11)\\', \\'Periodic visit, established patient, adolescent (age 12-17)\\', \\'Periodic visit, established patient, age 18-39\\', \\'Periodic visit, established patient, age 40-64\\', \\'Periodic visit, established patient, age 65 and older\\', \\'Preventive medicine counseling, Individual, approx 15 minutes\\', \\'Preventive medicine counseling, Individual, approx 30 minutes\\', \\'Preventive medicine counseling, Individual, approx 45 minutes\\', \\'Preventive medicine counseling, Individual, approx 60 minutes\\', \\'Preventive medicine counseling, Group, approx 30 minutes\\', \\'Preventive medicine counseling, Group, approx 60 minutes\\'], \\'Amount\\': [\\'$196.07\\', \\'$130.00\\', \\'$163.00\\', \\'$1500.00\\', \\'$1500.00\\', \\'$65.00\\', \\'$61.00\\', \\'$60.00\\', \\'$65.00\\', \\'$74.00\\', \\'$80.00\\', \\'$118.00\\', \\'$61.00\\', \\'$61.00\\', \\'$60.00\\', \\'$60.00\\', \\'$75.00\\', \\'$90.00\\', \\'$99.00\\', \\'$31.00\\', \\'$40.00\\', \\'$60.00\\', \\'$79.00\\', \\'$28.00\\', \\'$57.00\\']}\\n-------Table End--------\\n\\nStart of Page No. = 33\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nAdditional Compensation to IPA in the form of Surplus Distribution.\\n1.\\nLimited Risk Transfer/No Transfer of Financial Risk\\n1.1.\\nIPA shall be entitled to Additional Compensation under this Agreement on an annual\\nbasis in the form of a Surplus (as defined below) distribution, to the extent that any available Surplus\\nfunds exist, as specified below.\\n1.2.\\nUse of Additional Compensation. IPA represents and warrants that Additional\\nCompensation paid to IPA, if any, shall be distributed to Participating IPA Providers or used by IPA in a\\nmanner designed to improve the quality of care received by Enrollees and Enrollee satisfaction. On\\nrequest by Healthfirst, IPA shall provide Healthfirst with financial records accounting in detail all funds\\nreceived from Healthfirst and the disbursement of all such funds as set forth in 10 NYCRR 98-1.18(d).\\n2.\\nDefinitions. Capitalized terms used in this Exhibit 4.1(b) shall have the meaning given in this\\nExhibit or, if any term is not defined in this Exhibit 4.1(b), it shall have the meaning given in the\\nAgreement to which this Exhibit is attached.\\n2.1.\\n\"Additional Compensation\" shall mean the portion of any Surplus paid to IPA pursuant\\nto Section 5 of this Exhibit 4.1(b).\\n2.2.\\n\"Adjusted Premiums\" shall mean the premiums received by Healthfirst for Risk\\nEnrollees, as adjusted by any risk score or other adjustment to premiums pursuant to the applicable Plan\\nContract and applicable to Risk Enrollees as reasonably determined and applied by Healthfirst.\\n2.3.\\n\"Administrative Deduction\" shall mean the deduction from Adjusted Premiums equal to\\nHealthfirst\\'s administrative costs as reasonably determined by Healthfirst.\\n2.4.\\n\"Final Reconciliation\" shall mean the final the accounting, as set forth in Section 3, of\\nMedical Revenue, claims paid for Health Care Services and other expenses to determine Surpluses or\\nDeficits conducted twelve months following the expiration, non-renewal or termination of this Exhibit.\\n2.5.\\n\"Medical Revenue\" shall mean the balance of Adjusted Premiums after subtracting the\\nAdministrative Deduction and any Quality Incentive Deduction.\\n2.6.\\n\"Quality Incentive Deduction\" shall mean an amount taken from Adjusted Premiums for\\nprograms in which IPA and IPA Providers participate to improve the quality of Health Care Services\\nwhich Risk Enrollees receive.\\n2.7.\\n\"Reconciliation\" shall mean the accounting, as set forth in Section 3, of Medical\\nRevenue, claims paid for Health Care Services and other expenses to determine Surpluses or Deficits.\\n2.8.\\n\"Risk Enrollee\" shall mean an Enrollee who has selected, or been assigned to, a primary\\ncare provider who is a Participating IPA Provider.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n7 of 10\\n\\nStart of Page No. = 34\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\n2.9.\\n\"Surplus\" or \"Deficit\" shall mean the balance of Medical Revenue following\\nReconciliation\\n3.\\nReconciliation of Medical Revenue to Determine Surpluses and Deficits. During each\\nReconciliation, Healthfirst shall make the following deductions from Medical Revenue and determine\\nthe amount of any Surplus of Deficit:\\n3.1.\\nthe costs of claims paid for Health Care Services rendered to Risk Enrollees by\\nParticipating IPA Providers, Participating Providers and all other health care providers.\\n3.2.\\na\\nreasonable reserve for the cost of claims for Health Care Services rendered to Risk\\nEnrollees but not yet received;\\n3.3.\\nother costs associated with the provision of Health Care Services to Risk Enrollees\\nincluding but not limited to the net cost of reinsurance and any internal aggregate risk sharing programs.\\n3.4.\\naccruals for any risk adjustment applicable to Adjusted Premiums.\\n4.\\nReconciliation Schedule.\\n4.1.\\nInterim Quarterly Reconciliation and Distribution. Within one-hundred and twenty days\\n(120) after the end of each calendar year quarter, Healthfirst shall determine the amount of any Surplus\\nor Deficit for all preceding quarters during that calendar year. In the event that Healthfirst determines\\nthat a Surplus has been achieved for the preceding quarters during the calendar year, Healthfirst shall\\nmake an interim payment to IPA of Additional Compensation as set forth in Section 5 below. In the\\nevent that Healthfirst determines that a Deficit has been achieved for the preceding quarters druing the\\ncalendar year, Healthfirst not make an interim payment to IPA of Additional Compensation and the\\nDeficit shall be handled as set forth in Section 6 below.\\n4.2.\\nAnnual Reconciliation and Distribution. Within one-hundred and twenty days (120) after\\nthe end of each calendar year, Healthfirst shall determine the amount of any Surplus for all quarters for\\nthat calendar year. In the event that Healthfirst determines that a Surplus has been achieved for that\\ncalendar year, Healthfirst shall make a final payment to IPA of Additional Compensation as set forth in\\nSection 5 below. In the event that Healthfirst determines that a Deficit has been achieved for that\\nCalendar Year, Healthfirst not make a final payment to IPA of Additional Compensation and the Deficit\\nshall be handled as set forth in Section 6 below.\\n5.\\nPayment of Additional Compensation. In the event that Healthfirst determines that a Surplus has\\nbeen achieved as set forth Section 3 above, Healthfirst shall pay to IPA Additional Compensation which\\nshall be the lesser of the following:\\n5.1.\\nTen percent (10%) of the Surplus for the calendar year, or\\n5.2.\\nAn amount such that the Additional Compensation equals twenty-five percent (25%) of\\nthe sum of i) the total compensation to Participating IPA Providers for Health Care Services rendered to\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n8 of 10\\n\\nStart of Page No. = 35\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nRisk Enrollees pursuant to Exhibit 4.1(a); any compensation paid to IPA pursuant to Exhibit 4.1(a) and\\niii) the Additional Compensation.\\n5.3.\\nHealthfirst shall pay any Additional Compensation, within thirty days of each\\nReconciliation. Healthfirst shall, in no event, advance to IPA any Additional Compensation.\\n6.\\nTreatment of Deficits.\\n6.1.\\nDeficits Rolled Forward. In the event that a Deficit remains following any Reconciliation\\nor a Final Reconciliation, the Deficit equal to the percentage set forth in Section 5.1 above shall be\\nrolled forward to reduce any Surplus that may accrue under this Agreement in the succeeding calendar\\nyear.\\n6.2.\\nDeficits at Termination, Expiration or Non-Renewal. In the event that this Agreement\\nterminates, is not renewed or expires, regardless of the reason, Provider shall not be required to\\nreimburse Healthfirst for any Deficits which may exist at the time of such termination, expiration or\\nnon-renewal.\\n7.\\nCompliance with Regulatory Provisions Regarding Risk Transfer\\n7.1.\\nSDOH Provider Contract Guidelines. Healthfirst and IPA shall comply with\\nthe\\napplicable requirements of SDOH regarding transfer of financial risk to IPAs as set forth in SDOH\\'s\\nProvider Contracting Guidelines, as amended by SDOH from time to time. In the event that the amount\\nfinancial risk transferred to IPA constitutes a Level 4 risk transfer pursuant to the Providing Contracting\\nGuidelines, IPA shall demonstrate financial viability and establish a financial security deposit as\\nrequired by SDOH.\\n7.2.\\nPhysician Incentive Plan (\"PIP\") Regulations. Healthfirst and IPA shall comply with the\\napplicable requirements of the federal PIP regulations contained in 42 CFR 422.208, including the\\nplacement of stop loss insurance if applicable.\\n7.3.\\nFinancial Stability Monitoring and Reporting. Notwithstanding any other provision of\\nthis Agreement Healthfirst shall regularly monitor Provider\\'s financial condition to ensure that Provider\\nremains financially able to perform its obligations under this Agreement and meet any financial\\nsolvency requirements of SDOH or DFS. IPA shall promptly advise Healthfirst of any material\\ndevelopments which may impair its ability to perform its obligations under this Agreement. In addition,\\nIPA shall provide Healthfirst with the reports and information as required by 10 NYCRR 98-1.18(e),\\nwhich shall include but not be limited to IPA\\'s quarterly and annual financial statements. IPA shall also\\nmaintain, as required by 10 NYCRR 98-1.18(d), financial records accounting in detail all funds received\\nfrom Healthfirst and the disbursement of all such funds. IPA shall provide such accounting of funds and\\ndisbursements to Healthfirst on request.\\n8.\\nTermination or Amendment of Additional Compensation\\nHealthfirst may on, thirty (30) days prior notice to IPA, amend the terms and conditions pursuant to\\nwhich Additional Compensation is determined and paid to IPA as well as eliminate the payment of\\nAdditional Compensation to IPA; provided however, that unless IPA or IPA Providers have breached\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n9 of 10\\n\\nStart of Page No. = 36\\nDocuSign Envelope ID: 9C620FF8-D29C-41D1-AC34-A57E55353ABE\\nthe Agreement such amendment or termination shall not change any Additional Compensation\\ncalculated or paid prior to the effective date of such amendment or termination.\\n9.\\nTermination, Non-Renewal or Expiration of the Agreement.\\nFor a period of twelve months following any termination, non-renewal or expiration of this Agreement,\\nregardless of the cause Healthfirst will conduct Reconciliations of Medical Revenue pursuant to this\\nAgreement until the Final Reconciliation; provided, however, that payments to IPA of Additional\\nCompensation shall be suspended. These Reconciliations shall be on based Health Care Services\\nrendered to Risk Enrollees prior to the termination, non-renewal or expiration of this Agreement.\\nHealthfirst shall pay to any Additional Compensation existing at the Final Reconciliation within thirty\\ndays of the Final Reconciliation.\\nFED. TAX ID # 133923495\\nCONTRACT EFFECTIVE DATE: January 1, 2015\\nSDOH 4814\\n10 of 10'" - ] - }, - "execution_count": 39, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "contract_text" + "prompt = BOTTOM_UP_CODES(answer_dicts[0], text_dict['5'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "code_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)" ] }, { From e46038e8f1cc7bfcc23bf4af7353d857db3b27e9 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 8 May 2024 12:01:34 -0700 Subject: [PATCH 16/78] LOB and Program cleaning --- src/config.py | 2 +- src/postprocess.py | 54 +++++ src/prompts.py | 13 +- src/test.ipynb | 497 --------------------------------------------- src/test.py | 67 ++++++ 5 files changed, 127 insertions(+), 506 deletions(-) delete mode 100644 src/test.ipynb create mode 100644 src/test.py diff --git a/src/config.py b/src/config.py index 75bba8d..1c67ad3 100644 --- a/src/config.py +++ b/src/config.py @@ -31,7 +31,7 @@ AWS_SECRET_ACCESS_KEY="U3zdt9cAgCFVdy48uwktZSx2owC4QXo4/mtWCwdQ" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIv//////////wEaCXVzLWVhc3QtMiJHMEUCIQCxyHC2X3koEVwj1v5GQCySf/crrB3a1muqvXsMEUOvjwIgbX9sZ25cw8Hc0zklioXs/5rq9hfXna4AeAynR3O0+eUqjAMI5P//////////ARAAGgw2NjAxMzEwNjg3ODIiDDZ45+avGABDdEaUiirgAkJzeXOOSyKxFQv6Mq3sCSFlAGkjzUuMAziu1RZdWcF99u0rQthauWdxJ6rU4kwfd/M2LoWRQgu/8VxsktWxcivbleRrxwGAtcayMOBDWzeCdCDnvWZgKxZULiefEeJK3BDwUsWRN541FgZ/xqsAoASQlMO98rpNZ6W8xNq961xSpVUItMRPjOuHAot9YhPi1xLl+haeIdpsxGhx4ztQuJ+/ag5KAVrKpEiSsrQqf2L/q4FHB/3/YYzzAgXqP7Wy4qoPsWxfARvCw1FAoOqqPGjcmFwD3kov8q0FyYG4cc7/0UySSHxl7qUHFAr8vPuF7i+BdAFxeZEpAFd4s00gIm0V4ne8ta0wELiIfbkULdKZu7UezROqhm4+q9nQByrog4d1+YRE7B6x3zA9koFogcHQizZtZyoBDYK6Q3p+eab+SjIW6NndyYhCT1diebVLfnpth0MDMnIHibMHetm+XGwwiMnrsQY6pgGKQdkRrc2lhFuoRtw4uh4GTTz9PTQap3Ts2UNe7sT/49TpWxtqSB5WUyFGn2GAqnh1sOBZON3jo1e8qjibVxfACfv38PUBlktj6bVI0izhKfcEdoSyYWW1Cs5CTt1wlF3sOu5ztkTB+Gq8+H20gYAH+BgctDy43H6OwCGbBcveRxOEVH9jn+9GC6I99GNmjQ0w3Is9jT7zaJf52lE9nAA4n/p64Rro" # File Paths -LOCAL_PATH = '../docs/test/' # Replace with local +LOCAL_PATH = 'docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/postprocess.py b/src/postprocess.py index e890b87..ea5e3be 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,4 +1,7 @@ import config +import difflib + +import prompts # Filter applied between primary and secondary def primary_filter(answer_dict): @@ -11,11 +14,57 @@ def primary_filter(answer_dict): filtered_dicts.append(d) return filtered_dicts +def get_closest_match(val, val_list, similarity_threshold=0.6): + matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) + if matches: + index = val_list.index(matches[0]) + return val_list[index] + else: + return None + +def get_mappings(original_vals, actual_vals, similarity_threshold): + field_mapping = {} + non_field_mapping = {} + for original_val in original_vals: + val = original_val.upper() + if ',' in val: + val_list = [v.strip() for v in val.split(',')] + else: + val_list = [val] + + map_to = [] + non_field = [] + for sub_val in val_list: + if sub_val in actual_vals: + map_to.append(sub_val) + else: + closest_match = get_closest_match(sub_val, actual_vals, similarity_threshold) + if closest_match: + map_to.append(closest_match) + else: + non_field.append(sub_val) + field_mapping[original_val] = ', '.join(map_to) + if non_field: + non_field_mapping[original_val] = ', '.join(non_field) + return field_mapping, non_field_mapping + +def lob_product_program_clean(df): + original_lobs = df['CONTRACT_LOB'].astype(str).unique() + original_programs = df['CONTRACT_PROGRAM'].astype(str).unique() + # Get Mappings + lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) + + program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) + + # INCOMPLETE + return df def bottom_up_postprocess(df): # Remove Unnamed columns df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] + df = lob_product_program_clean(df) + # Reimbursement Flat Fee df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True) @@ -24,3 +73,8 @@ def bottom_up_postprocess(df): return df + + + + + diff --git a/src/prompts.py b/src/prompts.py index 8cd8061..24cc012 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -32,12 +32,13 @@ Your job is to identify what LOBs, Products, Plans, and Provider Types the reimb Return a dictionary object with the following attributes: 'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all. -'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, or similar. This will likely only be listed once per page, if at all. +'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, DSNP, Dual, or similar. This will likely only be listed once per page, if at all. 'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the same as LOB. It is NOT the name of a hospital. This will likely only be listed once per page, if at all. 'PROV_TYPE' : For what provider type or place of service is this fee schedule for? It could be Hospital, Professional, Ancillary, or similar. This will likely only be listed once per page, if at all. If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages. If there are multiple correct answers for the second page, return them both in a comma-separated list. However, in most instances there will be only one. +Sometimes the information is presented as 'SELECTED' OR 'NOT_SELECTED' - when this is the case, 'SELECTED' indicates the correct value(s). Return N/A for any answers not found. @@ -138,13 +139,9 @@ Only return the dictionary, with no other commentary or explanation. Ensure you """ - - - - - - - +VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] +VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] +VALID_PROV_TYPES = ['INPATIENT', 'OUTPATIENT', 'SKILLED NURSING FACILITY', 'AMBULATORY SURGERY CENTER', 'HOME HEALTH', 'RURAL HEALTH CLINIC'] diff --git a/src/test.ipynb b/src/test.ipynb deleted file mode 100644 index 56219a7..0000000 --- a/src/test.ipynb +++ /dev/null @@ -1,497 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import utils\n", - "import preprocess\n", - "import re\n", - "import pandas as pd\n", - "import bottom_up_funcs\n", - "import dict_operations\n", - "import postprocess\n", - "import prompts\n", - "import claude_funcs\n", - "\n", - "pd.set_option('display.max_rows', None)\n", - "pd.set_option('display.max_columns', None)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "input_dict = utils.read_input()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['133923495 CAIPA 2015 PHSP.txt'])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "input_dict.keys()" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "def format_table(table):\n", - " table = table.replace('>>>', '').replace('<<<', '')\n", - " table_dict = eval(table)\n", - " \n", - " keys = list(table_dict.keys())\n", - " # If values are lists\n", - " if isinstance(table_dict[keys[0]], list):\n", - " final_string = \"\"\n", - " for i in range(len(table_dict[keys[0]])):\n", - " for k in keys:\n", - " key_string = k + ': ' + table_dict[k][i] + ', '\n", - " final_string += key_string\n", - " final_string += '|\\n'\n", - " else:\n", - " # If values are not lists\n", - " final_string = \"\"\n", - " for k in keys:\n", - " key_string = k + ': ' + table_dict[k] + ', '\n", - " final_string += key_string\n", - " final_string += '|\\n'\n", - " return final_string\n", - "\n", - "def align_and_format_tables(text_dict):\n", - " pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)\n", - " for page, text in text_dict.items():\n", - " tables = pattern.findall(text)\n", - " for table in tables:\n", - " pre_table = table.split('{')[0].strip() # Get the pretable text for alignment\n", - " text = text.replace(table, '') # Remove full table\n", - " table_only = table.replace(pre_table, '')\n", - " try:\n", - " formatted_table = format_table(table_only)\n", - " except:\n", - " formatted_table = table\n", - " text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table\n", - " text = text.replace('-------Table Start--------', '')\n", - " text = text.replace('-------Table End--------', '') \n", - " text_dict[page] = text\n", - " return text_dict" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Managed Health, Inc.: By:, Provider: Chinese American Independent Practice Association, Inc.: By:, |\n", - "Managed Health, Inc.: Name: Print Name, Provider: Chinese American Independent Practice Association, Inc.: Name: Print Name, |\n", - "Managed Health, Inc.: Title:, Provider: Chinese American Independent Practice Association, Inc.: Title:, |\n", - "Managed Health, Inc.: Date:, Provider: Chinese American Independent Practice Association, Inc.: Date:, |\n", - "\n", - "Code Range: 97001-97006, Code Range Description: Physical Medicine Assessments, |\n", - "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", - "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", - "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", - "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", - "\n", - "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", - "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", - "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", - "CPT Code: 45380, CPT Description: Colonoscopy, with biopsies, |\n", - "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", - "\n", - "CPT Code: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", - "CPT Code: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", - "CPT Code: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", - "CPT Code: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", - "CPT Code: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", - "CPT Code: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", - "CPT Code: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", - "CPT Code: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", - "CPT Code: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", - "CPT Code: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", - "CPT Code: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", - "CPT Code: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", - "CPT Code: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", - "CPT Code: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", - "CPT Code: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", - "CPT Code: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", - "\n", - "Healthfirst PHSP, Inc.: By: Name: Print Name, Chinese American Independent Practice Association, Inc.: By: Name: Print Name, |\n", - "Healthfirst PHSP, Inc.: Title:, Chinese American Independent Practice Association, Inc.: Title:, |\n", - "Healthfirst PHSP, Inc.: Date: -, Chinese American Independent Practice Association, Inc.: Date:, |\n", - "\n", - "Code Range: 97001-97004, Code Range Description: Physical Medicine Assessments, |\n", - "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", - "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", - "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", - "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", - "\n", - "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", - "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", - "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", - "CPT Code: 45380, CPT Description: Colonoscopy, with biopsy, |\n", - "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", - "\n", - "CPT Code/Modifier: 99205-P1, CPT Description: PCAP Initial Visit, Amount: $196.07, |\n", - "CPT Code/Modifier: 99212-P2, CPT Description: PCAP Prenatal Visit, Amount: $130.00, |\n", - "CPT Code/Modifier: 99215-P3, CPT Description: PCAP Post partum Visit, Amount: $163.00, |\n", - "CPT Code/Modifier: 59409, CPT Description: Vaginal Delivery, Amount: $1500.00, |\n", - "CPT Code/Modifier: 59514, CPT Description: Cesarean Delivery, Amount: $1500.00, |\n", - "CPT Code/Modifier: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", - "CPT Code/Modifier: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code/Modifier: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code/Modifier: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", - "CPT Code/Modifier: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", - "CPT Code/Modifier: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", - "CPT Code/Modifier: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", - "CPT Code/Modifier: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", - "CPT Code/Modifier: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code/Modifier: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code/Modifier: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", - "CPT Code/Modifier: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", - "CPT Code/Modifier: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", - "CPT Code/Modifier: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", - "CPT Code/Modifier: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", - "CPT Code/Modifier: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", - "CPT Code/Modifier: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", - "CPT Code/Modifier: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", - "CPT Code/Modifier: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", - "CPT Code/Modifier: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", - "\n", - "Managed Health, Inc.: DocuSigned by: By: Paul E. Portsmore, Provider: Chinese American Independent Practice Association, Inc.: By: family, |\n", - "Managed Health, Inc.: 441CEAD8E502476 Name: Paul E. Portsmore Print Name, Provider: Chinese American Independent Practice Association, Inc.: Name: PEGGY SHENG Print Name, |\n", - "Managed Health, Inc.: Title: SVP, Provider: Chinese American Independent Practice Association, Inc.: Chief Operating Officer Title:, |\n", - "Managed Health, Inc.: Date: 12/31/2014 18:32:28 ET, Provider: Chinese American Independent Practice Association, Inc.: Date: Dec. 26, 2015, |\n", - "\n", - "Code Range: 97001-97006, Code Range Description: Physical Medicine Assessments, |\n", - "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", - "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", - "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", - "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", - "\n", - "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", - "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", - "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", - "CPT Code: 45380, CPT Description: Colonoscopy, with biopsies, |\n", - "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", - "\n", - "CPT Code: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", - "CPT Code: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", - "CPT Code: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", - "CPT Code: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", - "CPT Code: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", - "CPT Code: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", - "CPT Code: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", - "CPT Code: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", - "CPT Code: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", - "CPT Code: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", - "CPT Code: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", - "CPT Code: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", - "CPT Code: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", - "CPT Code: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", - "CPT Code: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", - "CPT Code: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", - "\n", - "Healthfirst PHSP, Inc.: DocuSigned by: By: Paul E. Portsmore, Chinese American Independent Practice Association, Inc.: By: f Sear, |\n", - "Healthfirst PHSP, Inc.: 441CEAD8E502476. Name: Paul E. Portsmore Print Name, Chinese American Independent Practice Association, Inc.: Name: PEGGY SHENG Print Name, |\n", - "Healthfirst PHSP, Inc.: Title: SVP -, Chinese American Independent Practice Association, Inc.: Title: Chief Operating Officer, |\n", - "Healthfirst PHSP, Inc.: Date: 12/31/2014 I 18:32:28 ET -, Chinese American Independent Practice Association, Inc.: Date: December 26, 2014, |\n", - "\n", - "Code Range: 97001-97004, Code Range Description: Physical Medicine Assessments, |\n", - "Code Range: 97010-97028, Code Range Description: Physical Therapy Treatment Modalities: Supervised, |\n", - "Code Range: 97032-97039, Code Range Description: Physical Therapy Treatment Modalities: Constant Attendance, |\n", - "Code Range: 97110-97546, Code Range Description: Other Therapeutic Techniques With Direct Patient Contact, |\n", - "Code Range: 97750-97755, Code Range Description: Physical performance test & Assistive technology assessment, |\n", - "\n", - "CPT Code: 43235, CPT Description: Upper gastrointestinal endoscopy, |\n", - "CPT Code: 43239, CPT Description: Upper gastrointestinal endoscopy, with biopsy, |\n", - "CPT Code: 45378, CPT Description: Colonoscopy, diagnostic, |\n", - "CPT Code: 45380, CPT Description: Colonoscopy, with biopsy, |\n", - "CPT Code: 45385, CPT Description: Colonoscopy, removal of tumor, |\n", - "\n", - "CPT Code/Modifier: 99205-P1, CPT Description: PCAP Initial Visit, Amount: $196.07, |\n", - "CPT Code/Modifier: 99212-P2, CPT Description: PCAP Prenatal Visit, Amount: $130.00, |\n", - "CPT Code/Modifier: 99215-P3, CPT Description: PCAP Post partum Visit, Amount: $163.00, |\n", - "CPT Code/Modifier: 59409, CPT Description: Vaginal Delivery, Amount: $1500.00, |\n", - "CPT Code/Modifier: 59514, CPT Description: Cesarean Delivery, Amount: $1500.00, |\n", - "CPT Code/Modifier: 99381, CPT Description: Initial visit, new patient, infant (under 1 year), Amount: $65.00, |\n", - "CPT Code/Modifier: 99382, CPT Description: Initial visit, new patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code/Modifier: 99383, CPT Description: Initial visit, new patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code/Modifier: 99384, CPT Description: Initial visit, new patient, adolescent (age 12-17), Amount: $65.00, |\n", - "CPT Code/Modifier: 99385, CPT Description: Initial visit, new patient, age 18-39, Amount: $74.00, |\n", - "CPT Code/Modifier: 99386, CPT Description: Initial visit, new patient, age 40-64, Amount: $80.00, |\n", - "CPT Code/Modifier: 99387, CPT Description: Initial visit, new patient, age 65 and older, Amount: $118.00, |\n", - "CPT Code/Modifier: 99391, CPT Description: Periodic visit, established patient, infant (under 1 year), Amount: $61.00, |\n", - "CPT Code/Modifier: 99392, CPT Description: Periodic visit, established patient, early childhood (age 1-4), Amount: $61.00, |\n", - "CPT Code/Modifier: 99393, CPT Description: Periodic visit, established patient, late childhood (age 5-11), Amount: $60.00, |\n", - "CPT Code/Modifier: 99394, CPT Description: Periodic visit, established patient, adolescent (age 12-17), Amount: $60.00, |\n", - "CPT Code/Modifier: 99395, CPT Description: Periodic visit, established patient, age 18-39, Amount: $75.00, |\n", - "CPT Code/Modifier: 99396, CPT Description: Periodic visit, established patient, age 40-64, Amount: $90.00, |\n", - "CPT Code/Modifier: 99397, CPT Description: Periodic visit, established patient, age 65 and older, Amount: $99.00, |\n", - "CPT Code/Modifier: 99401, CPT Description: Preventive medicine counseling, Individual, approx 15 minutes, Amount: $31.00, |\n", - "CPT Code/Modifier: 99402, CPT Description: Preventive medicine counseling, Individual, approx 30 minutes, Amount: $40.00, |\n", - "CPT Code/Modifier: 99403, CPT Description: Preventive medicine counseling, Individual, approx 45 minutes, Amount: $60.00, |\n", - "CPT Code/Modifier: 99404, CPT Description: Preventive medicine counseling, Individual, approx 60 minutes, Amount: $79.00, |\n", - "CPT Code/Modifier: 99411, CPT Description: Preventive medicine counseling, Group, approx 30 minutes, Amount: $28.00, |\n", - "CPT Code/Modifier: 99412, CPT Description: Preventive medicine counseling, Group, approx 60 minutes, Amount: $57.00, |\n", - "\n" - ] - } - ], - "source": [ - "contract_text = preprocess.clean_newlines(input_dict['133923495 CAIPA 2015 PHSP.txt'])\n", - "text_dict = preprocess.split_text(contract_text)\n", - "text_dict = align_and_format_tables(text_dict)\n", - "text_dict = preprocess.highlight_rates(text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "answer_strings = bottom_up_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000)\n", - "answer_dicts = dict_operations.primary_string_to_dict(answer_strings)\n", - "answer_dicts = postprocess.primary_filter(answer_dicts)" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'SERVICE': 'Initial visit, new patient, infant (under 1 year)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$65.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, early childhood (age 1-4)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, late childhood (age 5-11)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, adolescent (age 12-17)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$65.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, age 18-39',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$74.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, age 40-64',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$80.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Initial visit, new patient, age 65 and older',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$118.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, infant (under 1 year)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, early childhood (age 1-4)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$61.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, late childhood (age 5-11)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, adolescent (age 12-17)',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, age 18-39',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$75.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, age 40-64',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$90.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Periodic visit, established patient, age 65 and older',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$99.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Individual, approx 15 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$31.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Individual, approx 30 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$40.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Individual, approx 45 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$60.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Individual, approx 60 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$79.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Group, approx 30 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$28.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Preventive medicine counseling, Group, approx 60 minutes',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$57.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': 'N/A',\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Hearing Aids',\n", - " 'REIMBURSEMENT_FLAT_FEE': 'N/A',\n", - " 'REIMBURSEMENT_RATE': '80%',\n", - " 'FULL_METHODOLOGY': \"IPA Providers shall be reimbursed for the cost of hearing aids at eighty percent (80%) of the manufacturer's suggested retail price.\",\n", - " 'page_num': '5'},\n", - " {'SERVICE': 'Network Access and Administrative Fees for Medicare Advantage members',\n", - " 'REIMBURSEMENT_FLAT_FEE': '$5.00',\n", - " 'REIMBURSEMENT_RATE': 'N/A',\n", - " 'FULL_METHODOLOGY': \"Healthfirst shall compensate IPA $5.00 per Member per month for the following administrative services,: Access to IPA's network of Participating IPA Providers Credentialing of Participating IPA Providers Reporting on Health Care Services provided to Enrollees as reasonably requested by Healthfirst Administrative services relating to quality improvement and Enrollee satisfaction Education of Participating IPA Providers regarding Healthfirst's policies and procedures and quality improvement programs.\",\n", - " 'page_num': '5'}]" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "answer_dicts" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [], - "source": [ - "def BOTTOM_UP_CODES(d, page):\n", - " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", - "\n", - "The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below:\n", - "Service: {d['SERVICE']}\n", - "Methodology: {d['FULL_METHODOLOGY']}\n", - "\n", - "Your job is to identify codes associated with this specific term.\n", - "\n", - "Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided):\n", - "'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility.\n", - "'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes.\n", - "'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar.\n", - "'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options.\n", - "'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service.\n", - "'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures.\n", - "'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes.\n", - "'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status.\n", - "\n", - "Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range.\n", - "\n", - "Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting.\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "prompt = BOTTOM_UP_CODES(answer_dicts[0], text_dict['5'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "code_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/test.py b/src/test.py new file mode 100644 index 0000000..bb900a4 --- /dev/null +++ b/src/test.py @@ -0,0 +1,67 @@ + +import pandas as pd +import numpy as np +import difflib + +import prompts + + +df = pd.read_csv('output/consolidated_results_20240503_v1.csv') + +def get_closest_match(val, val_list, similarity_threshold=0.6): + matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) + if matches: + index = val_list.index(matches[0]) + return val_list[index] + else: + return None + +def get_mappings(original_vals, actual_vals, similarity_threshold): + field_mapping = {} + non_field_mapping = {} + for original_val in original_vals: + val = original_val.upper() + if ',' in val: + val_list = [v.strip() for v in val.split(',')] + else: + val_list = [val] + + map_to = [] + non_field = [] + for sub_val in val_list: + if sub_val in actual_vals: + map_to.append(sub_val) + else: + closest_match = get_closest_match(sub_val, actual_vals, similarity_threshold) + if closest_match: + map_to.append(closest_match) + else: + non_field.append(sub_val) + field_mapping[original_val] = ', '.join(map_to) + if non_field: + non_field_mapping[original_val] = ', '.join(non_field) + return field_mapping, non_field_mapping + +def lob_product_program_clean(df): + original_lobs = df['CONTRACT_LOB'].astype(str).unique() + original_programs = df['CONTRACT_PROGRAM'].astype(str).unique() + # Get Mappings + lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) + + program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) + + + + + print(program_mapping) + print(non_program_mapping) + #print(program_mapping) + + + + +#print(get_closest_match('COMMERCIAL PRODUCTS', prompts.VALID_LOBS)) + +lob_product_program_clean(df) +#print(df['CONTRACT_LOB'].value_counts(dropna=False)) +#print(df['CONTRACT_PROGRAM'].value_counts(dropna=False)) From b130eece8a30881753f5e9c6f8175605152e5341 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 9 May 2024 08:27:15 -0700 Subject: [PATCH 17/78] Add cleaning steps --- src/postprocess.py | 53 ++++++++++++++++++++--- src/prompts.py | 58 ++++++++++++++++++++++++- src/test.py | 104 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 189 insertions(+), 26 deletions(-) diff --git a/src/postprocess.py b/src/postprocess.py index ea5e3be..7d5dac6 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -14,6 +14,13 @@ def primary_filter(answer_dict): filtered_dicts.append(d) return filtered_dicts +import pandas as pd +import difflib + +VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] +VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] +#VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] + def get_closest_match(val, val_list, similarity_threshold=0.6): matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) if matches: @@ -30,8 +37,7 @@ def get_mappings(original_vals, actual_vals, similarity_threshold): if ',' in val: val_list = [v.strip() for v in val.split(',')] else: - val_list = [val] - + val_list = [val] map_to = [] non_field = [] for sub_val in val_list: @@ -49,16 +55,49 @@ def get_mappings(original_vals, actual_vals, similarity_threshold): return field_mapping, non_field_mapping def lob_product_program_clean(df): - original_lobs = df['CONTRACT_LOB'].astype(str).unique() - original_programs = df['CONTRACT_PROGRAM'].astype(str).unique() + # Ensure string type and handle NaN + df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str) + df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str) + df['PRODUCT'] = df['PRODUCT'].fillna('').astype(str) + + original_lobs = df['CONTRACT_LOB'].unique() + original_programs = df['CONTRACT_PROGRAM'].unique() + # Get Mappings - lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) + lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5) + program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6) - program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) + # Apply mappings + df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB']) + df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM']) - # INCOMPLETE + # Initialize new column + df['potential_products_detected'] = '' + + for index, row in df.iterrows(): + lob_entries = row['CONTRACT_LOB'].split(', ') + program_entries = row['CONTRACT_PROGRAM'].split(', ') + product_entries = row['PRODUCT'].split(', ') + + # Filtering valid entries + valid_lob_entries = [entry for entry in lob_entries if entry in VALID_LOBS] + valid_program_entries = [entry for entry in program_entries if entry in VALID_PROGRAMS] + + # Update the DataFrame with valid entries + df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries) + df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries) + + # Detect and handle products + detected_products = set(valid_lob_entries).intersection(product_entries) | set(valid_program_entries).intersection(product_entries) + df.at[index, 'potential_products_detected'] = ', '.join(detected_products) + + # Remove detected products from LOB and Program + df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products]) + df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products]) + return df + def bottom_up_postprocess(df): # Remove Unnamed columns df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] diff --git a/src/prompts.py b/src/prompts.py index 24cc012..6ebbca4 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -141,7 +141,63 @@ Only return the dictionary, with no other commentary or explanation. Ensure you VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] -VALID_PROV_TYPES = ['INPATIENT', 'OUTPATIENT', 'SKILLED NURSING FACILITY', 'AMBULATORY SURGERY CENTER', 'HOME HEALTH', 'RURAL HEALTH CLINIC'] +VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] + +VALID_PROV_TYPES = ['INPATIENT', 'OUTPATIENT', 'Hospital', 'PROFESSIONAL', 'IPA PROVIDERS', 'SKILLED NURSING FACILITY', 'AMBULATORY SURGERY CENTER', 'HOME HEALTH', 'RURAL HEALTH CLINIC'] + + + +CARVEOUT_LIST = ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery', +'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN', +'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events', +'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures', +'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics', +'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs', +'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries', +'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician', +'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path', +'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures', +'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit', +'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine', +'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis', +'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker', +'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation', +'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental', +'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists', +'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT', +'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation', +'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care', +'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies', +'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate', +'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby', +'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse', +'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute', +'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty', +'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section', +'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology', +'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band', +'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy', +'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology'] + + +# SERVICE values that are never carveouts +NON_CARVEOUT_LIST = ['Health Care Services', 'Covered Services', 'Outpatient Services', 'Inpatient Services'] + +# IPA Providers --> Professional + +""" +Service/Prov Type: +Step 1: Clean Service column +Step 2: Clean Prov Type column +Step 3: Handle Service/Prov Type interactions (Inpatient v Outpatient etc) + +LOB/Program/Product/Network +Step 1: Clean individual columns +Step 2: Handle column interactions + +Carveout Breakout + +""" diff --git a/src/test.py b/src/test.py index bb900a4..28b413c 100644 --- a/src/test.py +++ b/src/test.py @@ -6,7 +6,48 @@ import difflib import prompts -df = pd.read_csv('output/consolidated_results_20240503_v1.csv') +#df = pd.read_csv('output/consolidated_results_20240503_v1.csv') + +def create_in_out_column(df): + df['IN_OUT'] = '' + for idx, row in df.iterrows(): + service = str(row['SERVICE']).upper() + prov_type = str(row['PROV_TYPE']).upper() + if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient' + new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '') + df.loc[idx, 'SERVICE'] = new_service + elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Inpatient' + new_service = service.replace('INPATIENT', '').replace('IP ', '') + df.loc[idx, 'SERVICE'] = new_service + elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Outpatient' + new_service = service.replace('OUTPATIENT', '').replace('OP ', '') + df.loc[idx, 'SERVICE'] = new_service + else: + pass + return df + +#df_new = create_in_out_column(df) +#df_new.to_csv('output/post_test.csv') + +def primary_filter(answer_dict): + filtered_dicts = [] + for d in answer_dict: + if 'SERVICE' in d.keys(): + if 'INSURANCE' not in d['SERVICE'].upper(): + filtered_dicts.append(d) + else: + filtered_dicts.append(d) + return filtered_dicts + +import pandas as pd +import difflib + +VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] +VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] +VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] def get_closest_match(val, val_list, similarity_threshold=0.6): matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) @@ -14,7 +55,8 @@ def get_closest_match(val, val_list, similarity_threshold=0.6): index = val_list.index(matches[0]) return val_list[index] else: - return None + return None # Claude prompt here prompt = f"{value} What value in this list is the value closest to? {list}. If none are close return None." + def get_mappings(original_vals, actual_vals, similarity_threshold): field_mapping = {} @@ -24,8 +66,7 @@ def get_mappings(original_vals, actual_vals, similarity_threshold): if ',' in val: val_list = [v.strip() for v in val.split(',')] else: - val_list = [val] - + val_list = [val] map_to = [] non_field = [] for sub_val in val_list: @@ -43,25 +84,52 @@ def get_mappings(original_vals, actual_vals, similarity_threshold): return field_mapping, non_field_mapping def lob_product_program_clean(df): - original_lobs = df['CONTRACT_LOB'].astype(str).unique() - original_programs = df['CONTRACT_PROGRAM'].astype(str).unique() + # Ensure string type and handle NaN + df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str) + df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str) + df['PRODUCT'] = df['PRODUCT'].fillna('').astype(str) + + original_lobs = df['CONTRACT_LOB'].unique() + original_programs = df['CONTRACT_PROGRAM'].unique() + # Get Mappings - lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) + lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5) + program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6) - program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) + # Apply mappings + df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB']) + df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM']) - + # Initialize new column + df['potential_products_detected'] = '' + for index, row in df.iterrows(): + lob_entries = [v.strip() for v in row['CONTRACT_LOB'].split(',')] + program_entries = [v.strip() for v in row['CONTRACT_PROGRAM'].split(',')] + product_entries = [v.strip() for v in row['PRODUCT'].split(',')] - print(program_mapping) - print(non_program_mapping) - #print(program_mapping) - + # Filtering valid entries + valid_lob_entries = [entry for entry in lob_entries if entry.upper() in VALID_LOBS] + valid_program_entries = [entry for entry in program_entries if entry.upper() in VALID_PROGRAMS] + + # Update the DataFrame with valid entries + df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries) + df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries) + + # Detect and handle products + detected_products = set(valid_lob_entries) | set(valid_program_entries) + df.at[index, 'potential_products_detected'] = ', '.join(detected_products) + + # Remove detected products from LOB and Program + df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products]) + df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products]) - + # If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected + # If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected + # If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same. -#print(get_closest_match('COMMERCIAL PRODUCTS', prompts.VALID_LOBS)) + return df + +#processed_df = lob_product_program_clean(df) +#processed_df.to_csv('output/test.csv') -lob_product_program_clean(df) -#print(df['CONTRACT_LOB'].value_counts(dropna=False)) -#print(df['CONTRACT_PROGRAM'].value_counts(dropna=False)) From 7d6a8cf2d8e6291decc8f8e567a40f57161b61ed Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 9 May 2024 10:47:10 -0700 Subject: [PATCH 18/78] Improved Lesser of prompt --- src/bottom_up_funcs.py | 2 +- src/config.py | 16 ++++++++-------- src/prompts.py | 16 ++++++++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 461c9f8..0bac920 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -149,7 +149,7 @@ def bottom_up(file_object): #### POSTPROCESS #### #answer_dicts = append_secondary(answer_dicts) - answer_df = postprocess.bottom_up_postprocess(answer_df) + #answer_df = postprocess.bottom_up_postprocess(answer_df) # Write output if config.WRITE_OUTPUT: diff --git a/src/config.py b/src/config.py index 1c67ad3..b3461f3 100644 --- a/src/config.py +++ b/src/config.py @@ -18,17 +18,17 @@ MAX_WORKERS = 20 # Prompt Debugging RUN_PRIMARY = True # Always True -RUN_LOB = True +RUN_LOB = False RUN_LESSER = True -RUN_METHODOLOGY = True -RUN_FS = True -RUN_EXCEPTION = True -RUN_CODES = True +RUN_METHODOLOGY = False +RUN_FS = False +RUN_EXCEPTION = False +RUN_CODES = False # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAOY7QOHS" -AWS_SECRET_ACCESS_KEY="U3zdt9cAgCFVdy48uwktZSx2owC4QXo4/mtWCwdQ" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIv//////////wEaCXVzLWVhc3QtMiJHMEUCIQCxyHC2X3koEVwj1v5GQCySf/crrB3a1muqvXsMEUOvjwIgbX9sZ25cw8Hc0zklioXs/5rq9hfXna4AeAynR3O0+eUqjAMI5P//////////ARAAGgw2NjAxMzEwNjg3ODIiDDZ45+avGABDdEaUiirgAkJzeXOOSyKxFQv6Mq3sCSFlAGkjzUuMAziu1RZdWcF99u0rQthauWdxJ6rU4kwfd/M2LoWRQgu/8VxsktWxcivbleRrxwGAtcayMOBDWzeCdCDnvWZgKxZULiefEeJK3BDwUsWRN541FgZ/xqsAoASQlMO98rpNZ6W8xNq961xSpVUItMRPjOuHAot9YhPi1xLl+haeIdpsxGhx4ztQuJ+/ag5KAVrKpEiSsrQqf2L/q4FHB/3/YYzzAgXqP7Wy4qoPsWxfARvCw1FAoOqqPGjcmFwD3kov8q0FyYG4cc7/0UySSHxl7qUHFAr8vPuF7i+BdAFxeZEpAFd4s00gIm0V4ne8ta0wELiIfbkULdKZu7UezROqhm4+q9nQByrog4d1+YRE7B6x3zA9koFogcHQizZtZyoBDYK6Q3p+eab+SjIW6NndyYhCT1diebVLfnpth0MDMnIHibMHetm+XGwwiMnrsQY6pgGKQdkRrc2lhFuoRtw4uh4GTTz9PTQap3Ts2UNe7sT/49TpWxtqSB5WUyFGn2GAqnh1sOBZON3jo1e8qjibVxfACfv38PUBlktj6bVI0izhKfcEdoSyYWW1Cs5CTt1wlF3sOu5ztkTB+Gq8+H20gYAH+BgctDy43H6OwCGbBcveRxOEVH9jn+9GC6I99GNmjQ0w3Is9jT7zaJf52lE9nAA4n/p64Rro" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXC6PRLROX" +AWS_SECRET_ACCESS_KEY="3UNtSv9VyM2WFccTZWBeBCdxZE4nK00rY/cTSIug" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELD//////////wEaCXVzLWVhc3QtMiJHMEUCIQCBH8QRxxRn9Cscy5DbAiF520ouH2jDR3DPOYVTBKlfhAIgIk2i9y4UT25ndFseQNGfPorENxsdG4FKcVNRBiEsLx8qgwMIGRAAGgw2NjAxMzEwNjg3ODIiDAWrX1j/1i3BywL6iyrgAmC0BF21WmupY/8rcaQ0B7/cm0Wg4yphqP81R6j/VYZKvIp81UdMzbMaTOsfzpRcjgF89sstUGTCIA93u2liI3gkfcEXQVA8/XxyF96gKok3OP9jBHgIbBJQDXXpVSr7TUUZUpFLbYFDISrVMJDmte9UJIh/Dwj1P4r8OSiVwXWrfvgPienLBu2xmrRyNuhijV260EoJMOGLTkWWZpaYsWua50R5PXvGKIdJizMsr4u+VA8r+XAyqp5c0FIFdcPmbiFGg1W48hjxIZ86Ec2SHIowEqkI62kcXJ7JSNnOR0s/53ok07UZQ2Khr8RYEa9k5SR2H6GESlag/sTRIRngyo/qCThJ8eKW6VQKttC1UrFURNIu4rkajuV7vOY2U5P0tqsLvUuOVspIt5ViielkvfihQ6USCV7w0+E0FJIdztviIm1VTGX5kF+O4ctt1xwblBaqc5A+rTwQoZac80jGrS8wp+vzsQY6pgHQv5Ojg/pVpKLhQ/TQiGIMWw4n88Uo9N1BucRRciTjhXmTdLN5ZyKopiAIDlnaiKqhYDRbvhBbYc63jU3Y8RzwUHjC8RY6+hAqB/LuK6LweOE3pwanfK8uDp3tXBkiyfyz/++xUtcrz2ujf8DDRsyynxkNPI6G0R8s8RwLyjnAhvKn3nxxpJbqTSUTdFLHzy/WxNHxAOpQT+MocnFVIdjJTaIHi8lW" # File Paths LOCAL_PATH = 'docs/test/' # Replace with local diff --git a/src/prompts.py b/src/prompts.py index 6ebbca4..e8b526f 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -20,6 +20,8 @@ Here are the attributes to be included in each dictionary, and instructions on h 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. +Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. + Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ @@ -51,18 +53,20 @@ def BOTTOM_UP_LESSER(d, page): The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: Service: {d['SERVICE']} -Methodology: {d['FULL_METHODOLOGY']} +Original_Methodology: {d['FULL_METHODOLOGY']} Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above. Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply. Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): -'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. -'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Methodology. Write 'N' if not. -'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence - only include the first half of the lesser/greater of statement. For example, if the reimbursement is the lesser of X or Y, return X. -'REIMBURSEMENT_RATE' : If the REIMBURSEMENT_METHODOLOGY found above is a percent of something, write just the numeric percent value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. -'REIMBURSEMENT_FLAT_FEE' : If the REIMBURSEMENT_METHODOLOGY found above is a dollar value, write just the numeric value here. If not, write N/A. Note that this is a separate value from that found in the Methodology above. +'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. +'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. +'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence - only include the half of the lesser/greater of statement that is NOT Original_Methodology. For example, if the reimbursement is the lesser of X or Original_Methodology, return only X. +'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. If not, write N/A. +'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. If not, write N/A. + +For REIMBURSEMENT_RATE and REIMBURSEMENT_FLAT_FEE, do NOT include the Rate or Flat Fee from Original_Methodology. Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ From f4976a98c0372366fb8efb31b832c14ee25448da Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 9 May 2024 15:18:41 -0700 Subject: [PATCH 19/78] Modify prov type and lob logic --- src/config.py | 15 ++--- src/postprocess.py | 99 ++++++---------------------- src/prompts.py | 6 +- src/test.py | 156 +++++++++++++++++++++++---------------------- 4 files changed, 111 insertions(+), 165 deletions(-) diff --git a/src/config.py b/src/config.py index b3461f3..e0e9ec6 100644 --- a/src/config.py +++ b/src/config.py @@ -18,20 +18,17 @@ MAX_WORKERS = 20 # Prompt Debugging RUN_PRIMARY = True # Always True -RUN_LOB = False +RUN_LOB = True RUN_LESSER = True -RUN_METHODOLOGY = False -RUN_FS = False -RUN_EXCEPTION = False -RUN_CODES = False +RUN_METHODOLOGY = True +RUN_FS = True +RUN_EXCEPTION = True +RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXC6PRLROX" -AWS_SECRET_ACCESS_KEY="3UNtSv9VyM2WFccTZWBeBCdxZE4nK00rY/cTSIug" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELD//////////wEaCXVzLWVhc3QtMiJHMEUCIQCBH8QRxxRn9Cscy5DbAiF520ouH2jDR3DPOYVTBKlfhAIgIk2i9y4UT25ndFseQNGfPorENxsdG4FKcVNRBiEsLx8qgwMIGRAAGgw2NjAxMzEwNjg3ODIiDAWrX1j/1i3BywL6iyrgAmC0BF21WmupY/8rcaQ0B7/cm0Wg4yphqP81R6j/VYZKvIp81UdMzbMaTOsfzpRcjgF89sstUGTCIA93u2liI3gkfcEXQVA8/XxyF96gKok3OP9jBHgIbBJQDXXpVSr7TUUZUpFLbYFDISrVMJDmte9UJIh/Dwj1P4r8OSiVwXWrfvgPienLBu2xmrRyNuhijV260EoJMOGLTkWWZpaYsWua50R5PXvGKIdJizMsr4u+VA8r+XAyqp5c0FIFdcPmbiFGg1W48hjxIZ86Ec2SHIowEqkI62kcXJ7JSNnOR0s/53ok07UZQ2Khr8RYEa9k5SR2H6GESlag/sTRIRngyo/qCThJ8eKW6VQKttC1UrFURNIu4rkajuV7vOY2U5P0tqsLvUuOVspIt5ViielkvfihQ6USCV7w0+E0FJIdztviIm1VTGX5kF+O4ctt1xwblBaqc5A+rTwQoZac80jGrS8wp+vzsQY6pgHQv5Ojg/pVpKLhQ/TQiGIMWw4n88Uo9N1BucRRciTjhXmTdLN5ZyKopiAIDlnaiKqhYDRbvhBbYc63jU3Y8RzwUHjC8RY6+hAqB/LuK6LweOE3pwanfK8uDp3tXBkiyfyz/++xUtcrz2ujf8DDRsyynxkNPI6G0R8s8RwLyjnAhvKn3nxxpJbqTSUTdFLHzy/WxNHxAOpQT+MocnFVIdjJTaIHi8lW" # File Paths -LOCAL_PATH = 'docs/test/' # Replace with local +LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/postprocess.py b/src/postprocess.py index 7d5dac6..f59e202 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -14,87 +14,26 @@ def primary_filter(answer_dict): filtered_dicts.append(d) return filtered_dicts -import pandas as pd -import difflib -VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] -VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] -#VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] - -def get_closest_match(val, val_list, similarity_threshold=0.6): - matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) - if matches: - index = val_list.index(matches[0]) - return val_list[index] - else: - return None - -def get_mappings(original_vals, actual_vals, similarity_threshold): - field_mapping = {} - non_field_mapping = {} - for original_val in original_vals: - val = original_val.upper() - if ',' in val: - val_list = [v.strip() for v in val.split(',')] +def create_in_out_column(df): + df['IN_OUT'] = '' + for idx, row in df.iterrows(): + service = str(row['SERVICE']).upper() + prov_type = str(row['PROV_TYPE']).upper() + if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient' + new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '') + df.loc[idx, 'SERVICE'] = new_service + elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Inpatient' + new_service = service.replace('INPATIENT', '').replace('IP ', '') + df.loc[idx, 'SERVICE'] = new_service + elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type): + df.loc[idx, 'IN_OUT'] = 'Outpatient' + new_service = service.replace('OUTPATIENT', '').replace('OP ', '') + df.loc[idx, 'SERVICE'] = new_service else: - val_list = [val] - map_to = [] - non_field = [] - for sub_val in val_list: - if sub_val in actual_vals: - map_to.append(sub_val) - else: - closest_match = get_closest_match(sub_val, actual_vals, similarity_threshold) - if closest_match: - map_to.append(closest_match) - else: - non_field.append(sub_val) - field_mapping[original_val] = ', '.join(map_to) - if non_field: - non_field_mapping[original_val] = ', '.join(non_field) - return field_mapping, non_field_mapping - -def lob_product_program_clean(df): - # Ensure string type and handle NaN - df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str) - df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str) - df['PRODUCT'] = df['PRODUCT'].fillna('').astype(str) - - original_lobs = df['CONTRACT_LOB'].unique() - original_programs = df['CONTRACT_PROGRAM'].unique() - - # Get Mappings - lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5) - program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6) - - # Apply mappings - df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB']) - df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM']) - - # Initialize new column - df['potential_products_detected'] = '' - - for index, row in df.iterrows(): - lob_entries = row['CONTRACT_LOB'].split(', ') - program_entries = row['CONTRACT_PROGRAM'].split(', ') - product_entries = row['PRODUCT'].split(', ') - - # Filtering valid entries - valid_lob_entries = [entry for entry in lob_entries if entry in VALID_LOBS] - valid_program_entries = [entry for entry in program_entries if entry in VALID_PROGRAMS] - - # Update the DataFrame with valid entries - df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries) - df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries) - - # Detect and handle products - detected_products = set(valid_lob_entries).intersection(product_entries) | set(valid_program_entries).intersection(product_entries) - df.at[index, 'potential_products_detected'] = ', '.join(detected_products) - - # Remove detected products from LOB and Program - df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products]) - df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products]) - + pass return df @@ -102,7 +41,7 @@ def bottom_up_postprocess(df): # Remove Unnamed columns df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] - df = lob_product_program_clean(df) + #df = lob_product_program_clean(df) # Reimbursement Flat Fee df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True) diff --git a/src/prompts.py b/src/prompts.py index e8b526f..8a1d884 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -36,7 +36,11 @@ Return a dictionary object with the following attributes: 'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all. 'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, DSNP, Dual, or similar. This will likely only be listed once per page, if at all. 'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the same as LOB. It is NOT the name of a hospital. This will likely only be listed once per page, if at all. -'PROV_TYPE' : For what provider type or place of service is this fee schedule for? It could be Hospital, Professional, Ancillary, or similar. This will likely only be listed once per page, if at all. +'PROV_TYPE' : For what provider type or place of service is this fee schedule for? This will likely only be listed once per page, if at all. + +For CONTRACT_LOB, choose ONLY from the following: 'Medicaid', 'Medicare Advantage', 'Medicare Supplement', 'Marketplace', 'Commercial', 'Group' +Some examples of PROV_TYPE are: 'Primary Care Provider', 'Specialist', 'Ancillary', 'Ambulatory Surgery Center', 'Hospital','Rehab Facility', 'Skilled Nursing Facility', 'Ambulance', 'Rural Health Clinic', 'Home Health' +This list is non-exhaustive. If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages. If there are multiple correct answers for the second page, return them both in a comma-separated list. However, in most instances there will be only one. diff --git a/src/test.py b/src/test.py index 28b413c..750196a 100644 --- a/src/test.py +++ b/src/test.py @@ -4,50 +4,11 @@ import numpy as np import difflib import prompts +import claude_funcs -#df = pd.read_csv('output/consolidated_results_20240503_v1.csv') +df = pd.read_csv('output/consolidated_results_20240503_v1.csv') -def create_in_out_column(df): - df['IN_OUT'] = '' - for idx, row in df.iterrows(): - service = str(row['SERVICE']).upper() - prov_type = str(row['PROV_TYPE']).upper() - if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient' - new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '') - df.loc[idx, 'SERVICE'] = new_service - elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Inpatient' - new_service = service.replace('INPATIENT', '').replace('IP ', '') - df.loc[idx, 'SERVICE'] = new_service - elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Outpatient' - new_service = service.replace('OUTPATIENT', '').replace('OP ', '') - df.loc[idx, 'SERVICE'] = new_service - else: - pass - return df - -#df_new = create_in_out_column(df) -#df_new.to_csv('output/post_test.csv') - -def primary_filter(answer_dict): - filtered_dicts = [] - for d in answer_dict: - if 'SERVICE' in d.keys(): - if 'INSURANCE' not in d['SERVICE'].upper(): - filtered_dicts.append(d) - else: - filtered_dicts.append(d) - return filtered_dicts - -import pandas as pd -import difflib - -VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] -VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] -VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] def get_closest_match(val, val_list, similarity_threshold=0.6): matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) @@ -55,8 +16,9 @@ def get_closest_match(val, val_list, similarity_threshold=0.6): index = val_list.index(matches[0]) return val_list[index] else: - return None # Claude prompt here prompt = f"{value} What value in this list is the value closest to? {list}. If none are close return None." - + prompt = f"{val.upper()} is closest to which medical program mentioned in the list {val_list}. If none are close return None. If you are not sure return None. Answer in one word." + #return claude_funcs.invoke_claude_3(prompt, max_tokens=100) + return None def get_mappings(original_vals, actual_vals, similarity_threshold): field_mapping = {} @@ -84,6 +46,10 @@ def get_mappings(original_vals, actual_vals, similarity_threshold): return field_mapping, non_field_mapping def lob_product_program_clean(df): + # 1. If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected + # 2. If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected + # 3. If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same. + # Ensure string type and handle NaN df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str) df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str) @@ -93,43 +59,83 @@ def lob_product_program_clean(df): original_programs = df['CONTRACT_PROGRAM'].unique() # Get Mappings - lob_mapping, non_lob_mapping = get_mappings(original_lobs, VALID_LOBS, 0.5) - program_mapping, non_program_mapping = get_mappings(original_programs, VALID_PROGRAMS, 0.6) + lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) + program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) - # Apply mappings - df['CONTRACT_LOB'] = df['CONTRACT_LOB'].map(lob_mapping).fillna(df['CONTRACT_LOB']) - df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].map(program_mapping).fillna(df['CONTRACT_PROGRAM']) + # Get all lob mappings + all_lob_mappings = {} + for original_lob in lob_mapping: + full_lob_map = {} + full_lob_map['new_lob'] = lob_mapping[original_lob] + if original_lob in non_lob_mapping.keys(): + all_non_lobs = non_lob_mapping[original_lob] + for non_lob in all_non_lobs.split(','): + non_lob = non_lob.strip() + closest_program = get_closest_match(non_lob, prompts.VALID_PROGRAMS, 0.7) + if closest_program: + full_lob_map['new_program'] = closest_program + else: + full_lob_map['potential_product'] = non_lob + all_lob_mappings[original_lob] = full_lob_map - # Initialize new column - df['potential_products_detected'] = '' + # Get all program mappings + all_program_mappings = {} + for original_program in program_mapping: + full_program_map = {} + full_program_map['new_program'] = program_mapping[original_program] + if original_program in non_program_mapping.keys(): + all_non_programs = non_program_mapping[original_program] + for non_program in all_non_programs.split(','): + non_program = non_program.strip() + closest_lob = get_closest_match(non_program, prompts.VALID_LOBS, 0.7) + if closest_lob: + full_program_map['new_lob'] = closest_lob + else: + full_program_map['potential_product'] = non_program + all_program_mappings[original_program] = full_program_map - for index, row in df.iterrows(): - lob_entries = [v.strip() for v in row['CONTRACT_LOB'].split(',')] - program_entries = [v.strip() for v in row['CONTRACT_PROGRAM'].split(',')] - product_entries = [v.strip() for v in row['PRODUCT'].split(',')] + df['Potential_Product'] = "" + for idx, row in df.iterrows(): + original_lob = row['CONTRACT_LOB'] + original_program = row['CONTRACT_PROGRAM'] + original_product = row['PRODUCT'] - # Filtering valid entries - valid_lob_entries = [entry for entry in lob_entries if entry.upper() in VALID_LOBS] - valid_program_entries = [entry for entry in program_entries if entry.upper() in VALID_PROGRAMS] + # Handle LOB + lob_dict = all_lob_mappings[original_lob] + new_lob = lob_dict['new_lob'] + if 'new_program' in lob_dict.keys(): + new_program = lob_dict['new_program'] + else: + new_program = "" + if 'potential_product' in lob_dict.keys(): + potential_product = lob_dict['potential_product'] + else: + potential_product = "" + df.loc[idx, 'CONTRACT_LOB'] = new_lob + if original_program != new_program: + df.loc[idx, 'CONTRACT_PROGRAM'] = original_program + ' ' + new_program + if original_product != potential_product: + df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product - # Update the DataFrame with valid entries - df.at[index, 'CONTRACT_LOB'] = ', '.join(valid_lob_entries) - df.at[index, 'CONTRACT_PROGRAM'] = ', '.join(valid_program_entries) - - # Detect and handle products - detected_products = set(valid_lob_entries) | set(valid_program_entries) - df.at[index, 'potential_products_detected'] = ', '.join(detected_products) - - # Remove detected products from LOB and Program - df.at[index, 'CONTRACT_LOB'] = ', '.join([lob for lob in valid_lob_entries if lob not in detected_products]) - df.at[index, 'CONTRACT_PROGRAM'] = ', '.join([prog for prog in valid_program_entries if prog not in detected_products]) - - # If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected - # If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected - # If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same. - + # Handle Program + program_dict = all_program_mappings[original_program] + new_program = program_dict['new_program'] + if 'new_lob' in program_dict.keys(): + new_lob = program_dict['new_lob'] + else: + new_lob = "" + if 'potential_product' in program_dict.keys(): + potential_product = program_dict['potential_product'] + else: + potential_product = "" + df.loc[idx, 'CONTRACT_PROGRAM'] = new_program + if original_lob != new_lob: + df.loc[idx, 'CONTRACT_LOB'] = original_lob + ' ' + new_lob + if original_product != potential_product: + df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product + return df -#processed_df = lob_product_program_clean(df) -#processed_df.to_csv('output/test.csv') +processed_df = lob_product_program_clean(df) +processed_df.to_csv('output/test.csv') From 2ea1bd6bb9ad7e8715a5340934c4ee9e06149552 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 10 May 2024 07:11:07 -0700 Subject: [PATCH 20/78] postprocess additions, minor prompt mods --- src/config.py | 3 + src/dict_operations.py | 2 + src/postprocess.py | 22 ++++++ src/prompts.py | 4 +- src/test.py | 160 ++++++++--------------------------------- 5 files changed, 59 insertions(+), 132 deletions(-) diff --git a/src/config.py b/src/config.py index e0e9ec6..3b97b4a 100644 --- a/src/config.py +++ b/src/config.py @@ -26,6 +26,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys +AWS_ACCESS_KEY_ID="update here" +AWS_SECRET_ACCESS_KEY="update here" +AWS_SESSION_TOKEN="update here" # File Paths LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local diff --git a/src/dict_operations.py b/src/dict_operations.py index 6eb5b88..7b927e3 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -3,6 +3,7 @@ import json from collections import defaultdict def secondary_string_to_dict(dict_string): + print(dict_string) dict_string = dict_string.replace('<<<', '') dict_string = dict_string.replace('>>>', '') dict_string = dict_string.replace('\n', '') # Remove new line @@ -18,6 +19,7 @@ def secondary_string_to_dict(dict_string): return result_dict def primary_string_to_dict(string_dict): + print(string_dict) data = [] pattern = r'\{.*?\}' for page_num in string_dict.keys(): diff --git a/src/postprocess.py b/src/postprocess.py index f59e202..728b473 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -37,6 +37,28 @@ def create_in_out_column(df): return df +def flag_123(df): + # Group by 'SERVICE', 'CONTRACT_LOB', and 'Filename' + grouped = df.groupby(['SERVICE', 'CONTRACT_LOB', 'Filename']) + + # Initialize flag column + df['Flag_123'] = 'Primary' + + # Iterate over each group + for _, group in grouped: + # Assign flag based on the position in the group + group_size = len(group) + for i in range(group_size): + if i == 0: + df.loc[group.index[i], 'Flag_123'] = 'Primary' + elif i == 1: + df.loc[group.index[i], 'Flag_123'] = 'Secondary' + elif i == 2: + df.loc[group.index[i], 'Flag_123'] = 'Tertiary' + else: + df.loc[group.index[i], 'Flag_123'] = '' + return df + def bottom_up_postprocess(df): # Remove Unnamed columns df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] diff --git a/src/prompts.py b/src/prompts.py index 8a1d884..577ee77 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -20,7 +20,7 @@ Here are the attributes to be included in each dictionary, and instructions on h 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. -Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. +Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ @@ -43,7 +43,7 @@ Some examples of PROV_TYPE are: 'Primary Care Provider', 'Specialist', 'Ancillar This list is non-exhaustive. If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages. -If there are multiple correct answers for the second page, return them both in a comma-separated list. However, in most instances there will be only one. +If there are multiple correct answers for the second page, return them both in a comma-separated list. This is extremely rare - there will almost always be one. Sometimes the information is presented as 'SELECTED' OR 'NOT_SELECTED' - when this is the case, 'SELECTED' indicates the correct value(s). Return N/A for any answers not found. diff --git a/src/test.py b/src/test.py index 750196a..c4930dd 100644 --- a/src/test.py +++ b/src/test.py @@ -1,141 +1,41 @@ - import pandas as pd -import numpy as np -import difflib -import prompts -import claude_funcs +import utils +import postprocess -df = pd.read_csv('output/consolidated_results_20240503_v1.csv') +df = pd.read_csv('output/consolidated_results_20240509_v1.csv') +df = postprocess.bottom_up_postprocess(df) +print(df.columns) -def get_closest_match(val, val_list, similarity_threshold=0.6): - matches = difflib.get_close_matches(val.upper(), val_list, n=1, cutoff=similarity_threshold) - if matches: - index = val_list.index(matches[0]) - return val_list[index] - else: - prompt = f"{val.upper()} is closest to which medical program mentioned in the list {val_list}. If none are close return None. If you are not sure return None. Answer in one word." - #return claude_funcs.invoke_claude_3(prompt, max_tokens=100) - return None - -def get_mappings(original_vals, actual_vals, similarity_threshold): - field_mapping = {} - non_field_mapping = {} - for original_val in original_vals: - val = original_val.upper() - if ',' in val: - val_list = [v.strip() for v in val.split(',')] - else: - val_list = [val] - map_to = [] - non_field = [] - for sub_val in val_list: - if sub_val in actual_vals: - map_to.append(sub_val) - else: - closest_match = get_closest_match(sub_val, actual_vals, similarity_threshold) - if closest_match: - map_to.append(closest_match) - else: - non_field.append(sub_val) - field_mapping[original_val] = ', '.join(map_to) - if non_field: - non_field_mapping[original_val] = ', '.join(non_field) - return field_mapping, non_field_mapping - -def lob_product_program_clean(df): - # 1. If something in LOB is invalid, check if it is a Program. If yes, move to Program (append if needed), if no, move to potential_products_detected - # 2. If something in Program is invalid, check if it is an LOB. If yes, move to LOB (append if needed). If no, move to potential_products_detected - # 3. If something in Product is a valid LOB, copy to LOB (append if needed). If something in Product is a valid Program, copy to Program (append if needed). Keep product the same. - - # Ensure string type and handle NaN - df['CONTRACT_LOB'] = df['CONTRACT_LOB'].fillna('').astype(str) - df['CONTRACT_PROGRAM'] = df['CONTRACT_PROGRAM'].fillna('').astype(str) - df['PRODUCT'] = df['PRODUCT'].fillna('').astype(str) - - original_lobs = df['CONTRACT_LOB'].unique() - original_programs = df['CONTRACT_PROGRAM'].unique() +def flag_123(df): + # Group by 'SERVICE', 'CONTRACT_LOB', and 'Filename' + grouped = df.groupby(['SERVICE', 'CONTRACT_LOB', 'Filename']) - # Get Mappings - lob_mapping, non_lob_mapping = get_mappings(original_lobs, prompts.VALID_LOBS, 0.5) - program_mapping, non_program_mapping = get_mappings(original_programs, prompts.VALID_PROGRAMS, 0.6) - - # Get all lob mappings - all_lob_mappings = {} - for original_lob in lob_mapping: - full_lob_map = {} - full_lob_map['new_lob'] = lob_mapping[original_lob] - if original_lob in non_lob_mapping.keys(): - all_non_lobs = non_lob_mapping[original_lob] - for non_lob in all_non_lobs.split(','): - non_lob = non_lob.strip() - closest_program = get_closest_match(non_lob, prompts.VALID_PROGRAMS, 0.7) - if closest_program: - full_lob_map['new_program'] = closest_program - else: - full_lob_map['potential_product'] = non_lob - all_lob_mappings[original_lob] = full_lob_map - - # Get all program mappings - all_program_mappings = {} - for original_program in program_mapping: - full_program_map = {} - full_program_map['new_program'] = program_mapping[original_program] - if original_program in non_program_mapping.keys(): - all_non_programs = non_program_mapping[original_program] - for non_program in all_non_programs.split(','): - non_program = non_program.strip() - closest_lob = get_closest_match(non_program, prompts.VALID_LOBS, 0.7) - if closest_lob: - full_program_map['new_lob'] = closest_lob - else: - full_program_map['potential_product'] = non_program - all_program_mappings[original_program] = full_program_map - - df['Potential_Product'] = "" - for idx, row in df.iterrows(): - original_lob = row['CONTRACT_LOB'] - original_program = row['CONTRACT_PROGRAM'] - original_product = row['PRODUCT'] - - # Handle LOB - lob_dict = all_lob_mappings[original_lob] - new_lob = lob_dict['new_lob'] - if 'new_program' in lob_dict.keys(): - new_program = lob_dict['new_program'] - else: - new_program = "" - if 'potential_product' in lob_dict.keys(): - potential_product = lob_dict['potential_product'] - else: - potential_product = "" - df.loc[idx, 'CONTRACT_LOB'] = new_lob - if original_program != new_program: - df.loc[idx, 'CONTRACT_PROGRAM'] = original_program + ' ' + new_program - if original_product != potential_product: - df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product - - # Handle Program - program_dict = all_program_mappings[original_program] - new_program = program_dict['new_program'] - if 'new_lob' in program_dict.keys(): - new_lob = program_dict['new_lob'] - else: - new_lob = "" - if 'potential_product' in program_dict.keys(): - potential_product = program_dict['potential_product'] - else: - potential_product = "" - df.loc[idx, 'CONTRACT_PROGRAM'] = new_program - if original_lob != new_lob: - df.loc[idx, 'CONTRACT_LOB'] = original_lob + ' ' + new_lob - if original_product != potential_product: - df.loc[idx, 'Potential_Product'] = df.loc[idx, 'Potential_Product'] + ', ' + potential_product + # Initialize flag column + df['Flag_123'] = 'Primary' + + # Iterate over each group + for _, group in grouped: + # Assign flag based on the position in the group + group_size = len(group) + for i in range(group_size): + if i == 0: + df.loc[group.index[i], 'Flag_123'] = 'Primary' + elif i == 1: + df.loc[group.index[i], 'Flag_123'] = 'Secondary' + elif i == 2: + df.loc[group.index[i], 'Flag_123'] = 'Tertiary' + else: + df.loc[group.index[i], 'Flag_123'] = '' return df -processed_df = lob_product_program_clean(df) -processed_df.to_csv('output/test.csv') +# Flag primary, secondary, tertiary +df = flag_123(df) + +print(df.columns) +print(df['Flag_123'].value_counts(dropna=False)) +df.to_csv('output/consolidated_results_20240509_v2.csv') \ No newline at end of file From 21672ade913da733bb48aec96f9bacce5cdbd0c9 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 14 May 2024 11:12:20 -0700 Subject: [PATCH 21/78] Initial TD BU consolidation commit --- src/claude_funcs.py | 1 + src/config.py | 7 +- src/dict_operations.py | 1 + src/{bottom_up_main.py => main.py} | 7 +- src/postprocess.py | 80 --------------------- src/preprocess.py | 4 +- src/{bottom_up_funcs.py => prompt_funcs.py} | 3 +- src/prompts.py | 64 ----------------- src/test.py | 41 ----------- src/utils.py | 1 + 10 files changed, 15 insertions(+), 194 deletions(-) rename src/{bottom_up_main.py => main.py} (93%) delete mode 100644 src/postprocess.py rename src/{bottom_up_funcs.py => prompt_funcs.py} (98%) delete mode 100644 src/test.py diff --git a/src/claude_funcs.py b/src/claude_funcs.py index db7ad06..b4a656e 100644 --- a/src/claude_funcs.py +++ b/src/claude_funcs.py @@ -1,3 +1,4 @@ + import json import anthropic diff --git a/src/config.py b/src/config.py index 3b97b4a..a59d6ab 100644 --- a/src/config.py +++ b/src/config.py @@ -1,4 +1,5 @@ + from datetime import datetime import boto3 @@ -26,9 +27,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="update here" -AWS_SECRET_ACCESS_KEY="update here" -AWS_SESSION_TOKEN="update here" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXK6KGAGQU" +AWS_SECRET_ACCESS_KEY="v3q+R5QQUcV/4/KiRQeacVbzhLqUq7odldmMVGof" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjECoaCXVzLWVhc3QtMiJGMEQCIBkx50iMSg0w6ghdK7+3IvA1L8wL6NuCyyR3CzTGDwBCAiAN7/AkQCVKF48z5LZyD1qIRtyj+jA3hnUds98Od7BxLSqMAwiT//////////8BEAAaDDY2MDEzMTA2ODc4MiIMTBr6D5ElCcEBiYgcKuACjM5ZgbJ6uOtsZDPfCxysiCAxOaFccomp45MSUGxeH72a0hVYO2MbqPJnimNDCF5IsIRAX/uJX/MTQJZykZyP8YRkGtjFoQx7BzQYX9rNxmGKXCc2yYC9w66POLqcHhcMfPBlFIn8Tux4ocXSs5t6aVOoqAhftv9GUiVYDKDpNGjWZPjczWj4sDVWYXkBvPn3LjAC69tQM2bYLAzwYZWu92r1doRbN+oZewjPuOzShsMHEidTkkjIPUvHa5mOqdIQmTYZi8QFHWHOZ7ui8XaHlxV5EozIlUYAYkYYaBR53XuduGTn2AQOmXM6pFdkCK6r6KwuLH0+g2NsXex5Cx37OAr3e/yDnYcASOiLf1ySJ1E7/QICZMtFDR46BILAYA8oFaNPJUHh2c7HndZFMTkveprB5BCMf+SNV+WhgSZu87rv4zNo+CbLxMEU1a2kPKZtROTUvV6qVqhzDKdGpdKs0zDvzY6yBjqnATvEOOkXIa5Z+QKAsDff7kpAqpQrNReV88D0E8pOTGNIp/EgS2jwQwf1vWtMGlEbB2HHZ4jyydL6B1OM+5W8pPWuJKrjaOcoDWyUld5aZysdEOxCmXGVW8NLC5HSieTrAxkdDTQ8DpuefHXN4QeeGiwzrOJzTubH4CLuJOCRcog6dtMouJOO2ccVNTNU0s2Y1eZUdsg28ZTDCa1G506M/Fu7gaGEuw9K" # File Paths LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local diff --git a/src/dict_operations.py b/src/dict_operations.py index 7b927e3..93ff4b2 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -1,3 +1,4 @@ + import re import json from collections import defaultdict diff --git a/src/bottom_up_main.py b/src/main.py similarity index 93% rename from src/bottom_up_main.py rename to src/main.py index 636c4b1..8d7b6d3 100644 --- a/src/bottom_up_main.py +++ b/src/main.py @@ -1,3 +1,4 @@ + # Imports import os import concurrent.futures @@ -5,7 +6,7 @@ import traceback import utils import config -import bottom_up_funcs +import prompt_funcs import claude_funcs def main(): @@ -27,12 +28,12 @@ def main(): # # Run bottom up primary with multithreading # with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: - # executor.map(bottom_up_funcs.bottom_up, input_dict.items()) + # executor.map(prompt_funcs.bottom_up, input_dict.items()) def process_item(item): key, value = item try: - bottom_up_funcs.bottom_up(item) + prompt_funcs.bottom_up(item) except Exception as e: # Print the error message and traceback print(f"Error processing item {key}: {e}") diff --git a/src/postprocess.py b/src/postprocess.py deleted file mode 100644 index 728b473..0000000 --- a/src/postprocess.py +++ /dev/null @@ -1,80 +0,0 @@ -import config -import difflib - -import prompts - -# Filter applied between primary and secondary -def primary_filter(answer_dict): - filtered_dicts = [] - for d in answer_dict: - if 'SERVICE' in d.keys(): - if 'INSURANCE' not in d['SERVICE'].upper(): - filtered_dicts.append(d) - else: - filtered_dicts.append(d) - return filtered_dicts - - -def create_in_out_column(df): - df['IN_OUT'] = '' - for idx, row in df.iterrows(): - service = str(row['SERVICE']).upper() - prov_type = str(row['PROV_TYPE']).upper() - if ('INPATIENT' in service and 'OUTPATIENT' in service) or ('INPATIENT' in prov_type and 'OUTPATIENT' in prov_type) or ('IP ' in service and 'OP ' in service) or ('IP ' in prov_type and 'OP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Inpatient and Outpatient' - new_service = service.replace('INPATIENT', '').replace('OUTPATIENT', '').replace('IP ', '').replace('OP ', '') - df.loc[idx, 'SERVICE'] = new_service - elif ('INPATIENT' in service) or ('INPATIENT' in prov_type) or ('IP ' in service) or ('IP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Inpatient' - new_service = service.replace('INPATIENT', '').replace('IP ', '') - df.loc[idx, 'SERVICE'] = new_service - elif ('OUTPATIENT' in service) or ('OUTPATIENT' in prov_type) or ('OP ' in service) or ('OP ' in prov_type): - df.loc[idx, 'IN_OUT'] = 'Outpatient' - new_service = service.replace('OUTPATIENT', '').replace('OP ', '') - df.loc[idx, 'SERVICE'] = new_service - else: - pass - return df - - -def flag_123(df): - # Group by 'SERVICE', 'CONTRACT_LOB', and 'Filename' - grouped = df.groupby(['SERVICE', 'CONTRACT_LOB', 'Filename']) - - # Initialize flag column - df['Flag_123'] = 'Primary' - - # Iterate over each group - for _, group in grouped: - # Assign flag based on the position in the group - group_size = len(group) - for i in range(group_size): - if i == 0: - df.loc[group.index[i], 'Flag_123'] = 'Primary' - elif i == 1: - df.loc[group.index[i], 'Flag_123'] = 'Secondary' - elif i == 2: - df.loc[group.index[i], 'Flag_123'] = 'Tertiary' - else: - df.loc[group.index[i], 'Flag_123'] = '' - return df - -def bottom_up_postprocess(df): - # Remove Unnamed columns - df = df[[col for col in df.columns if 'UNNAMED' not in col.upper()]] - - #df = lob_product_program_clean(df) - - # Reimbursement Flat Fee - df.loc[:, 'REIMBURSEMENT_FLAT_FEE'] = df['REIMBURSEMENT_FLAT_FEE'].astype(str).str.replace(r'[^\d.]', '', regex=True) - - # Reimbursement Rate - df.loc[:, 'REIMBURSEMENT_RATE'] = df['REIMBURSEMENT_RATE'].astype(str).str.replace(r'%|<<<|>>>|[^\d.]', '', regex=True) - - return df - - - - - - diff --git a/src/preprocess.py b/src/preprocess.py index e76d580..b65e356 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -1,3 +1,4 @@ + from itertools import groupby, count import re @@ -47,4 +48,5 @@ def clean_billed_charges(text): return match.group() else: return '100% of billed charges' - return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) \ No newline at end of file + return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) + diff --git a/src/bottom_up_funcs.py b/src/prompt_funcs.py similarity index 98% rename from src/bottom_up_funcs.py rename to src/prompt_funcs.py index 0bac920..9b3b6dd 100644 --- a/src/bottom_up_funcs.py +++ b/src/prompt_funcs.py @@ -1,4 +1,5 @@ + import os import pandas as pd @@ -7,7 +8,6 @@ import prompts import claude_funcs import preprocess import dict_operations -import postprocess import table_funcs @@ -127,7 +127,6 @@ def bottom_up(file_object): # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) answer_dicts = dict_operations.primary_string_to_dict(answer_strings) - answer_dicts = postprocess.primary_filter(answer_dicts) # Bottom Up LOB - CONTRACT_LOB, CONTRACT_PROGRAM, CONTRACT_PRODUCT, PROV_TYPE if config.RUN_LOB: diff --git a/src/prompts.py b/src/prompts.py index 577ee77..599563f 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -145,67 +145,3 @@ Return a value for each type of code - return N/A for any values not found. Mult Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ - - -VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] -VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] -VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS', 'PPO-POS'] - -VALID_PROV_TYPES = ['INPATIENT', 'OUTPATIENT', 'Hospital', 'PROFESSIONAL', 'IPA PROVIDERS', 'SKILLED NURSING FACILITY', 'AMBULATORY SURGERY CENTER', 'HOME HEALTH', 'RURAL HEALTH CLINIC'] - - - -CARVEOUT_LIST = ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery', -'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN', -'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events', -'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures', -'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics', -'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs', -'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries', -'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician', -'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path', -'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures', -'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit', -'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine', -'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis', -'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker', -'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation', -'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental', -'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists', -'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT', -'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation', -'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care', -'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies', -'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate', -'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby', -'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse', -'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute', -'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty', -'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section', -'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology', -'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band', -'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy', -'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology'] - - -# SERVICE values that are never carveouts -NON_CARVEOUT_LIST = ['Health Care Services', 'Covered Services', 'Outpatient Services', 'Inpatient Services'] - -# IPA Providers --> Professional - -""" -Service/Prov Type: -Step 1: Clean Service column -Step 2: Clean Prov Type column -Step 3: Handle Service/Prov Type interactions (Inpatient v Outpatient etc) - -LOB/Program/Product/Network -Step 1: Clean individual columns -Step 2: Handle column interactions - -Carveout Breakout - -""" - - - diff --git a/src/test.py b/src/test.py deleted file mode 100644 index c4930dd..0000000 --- a/src/test.py +++ /dev/null @@ -1,41 +0,0 @@ -import pandas as pd - -import utils -import postprocess - - -df = pd.read_csv('output/consolidated_results_20240509_v1.csv') - -df = postprocess.bottom_up_postprocess(df) -print(df.columns) - -def flag_123(df): - # Group by 'SERVICE', 'CONTRACT_LOB', and 'Filename' - grouped = df.groupby(['SERVICE', 'CONTRACT_LOB', 'Filename']) - - # Initialize flag column - df['Flag_123'] = 'Primary' - - # Iterate over each group - for _, group in grouped: - # Assign flag based on the position in the group - group_size = len(group) - for i in range(group_size): - if i == 0: - df.loc[group.index[i], 'Flag_123'] = 'Primary' - elif i == 1: - df.loc[group.index[i], 'Flag_123'] = 'Secondary' - elif i == 2: - df.loc[group.index[i], 'Flag_123'] = 'Tertiary' - else: - df.loc[group.index[i], 'Flag_123'] = '' - - return df - - -# Flag primary, secondary, tertiary -df = flag_123(df) - -print(df.columns) -print(df['Flag_123'].value_counts(dropna=False)) -df.to_csv('output/consolidated_results_20240509_v2.csv') \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index 42388ae..c9fa129 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,3 +1,4 @@ + import os import pandas as pd import shutil From be1567d6e44effb7daa65a88e1488e5ca143a8a9 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 14 May 2024 11:18:02 -0700 Subject: [PATCH 22/78] Add top down functions --- src/prompt_funcs.py | 62 ++++++++++++++++++++++++ src/prompts.py | 115 ++++++++++++++++++++++++++++++++++++++++++++ src/utils.py | 17 +++++++ 3 files changed, 194 insertions(+) diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 9b3b6dd..2ff25d2 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -3,12 +3,74 @@ import os import pandas as pd +from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate +from llama_index.core import SummaryIndex, Document +from llama_index.llms.bedrock import Bedrock + import config import prompts import claude_funcs import preprocess import dict_operations import table_funcs +import utils + + +def run_on_chunks_llama(input_filepath, num_chunks): + filename = os.path.basename(input_filepath) + pages = utils.preprocess_text_file(input_filepath) + + # Setup LLM with llamaindex + llm = Bedrock( + model="anthropic.claude-3-haiku-20240307-v1:0", + aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), + aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'), + aws_session_token=os.getenv('AWS_SESSION_TOKEN'), + region_name="us-east-1" + ) + + total_pages = len(pages) + #pages_per_chunk = (total_pages + num_chunks - 1) // num_chunks + + #chunks = [pages[i:i + pages_per_chunk] for i in range(0, len(pages), pages_per_chunk)] + pages_per_chunk = 1 # Each page is its own chunk + chunks = [pages[i:i + pages_per_chunk] for i in range(0, len(pages), pages_per_chunk)] + + all_results = [] + page_number = 1 + + # Define prompts for each attribute + prompt_functions = { + 'CONTRACT_LOB': prompts.prompt_contract_lob(), + 'PRODUCT': prompts.prompt_product(), + 'CONTRACT_NETWORK': prompts.prompt_network_name(), + 'CONTRACT_SERVICE_AREA': prompts.prompt_service_area(), + 'LOB_PRICING_TERMS_EFFECTIVE_DT': prompts.prompt_effective_date(), + 'LOB_PRICING_TERMS_TERMINATION_DT': prompts.prompt_termination_date(), + 'CONTRACT_MARKETPLACE_METAL_LEVEL' : prompts.prompt_metal_level() + } + + for chunk_index, chunk in enumerate(chunks): + chunk_text = " ".join(chunk) + end_page = page_number + len(chunk) - 1 + index = SummaryIndex([]) + doc = Document(text=chunk_text, id_=f"Chunk_{chunk_index + 1}") + index.insert(doc) + query_engine = index.as_query_engine(llm=llm) + + chunk_results = {'Filename': filename, 'Page': f"Chunk {chunk_index + 1} (Pages {page_number}-{end_page})"} + + for attribute, prompt_function in prompt_functions.items(): + specific_prompt = chunk_text + "\n\n" + prompt_function() # Get dynamic prompt + answer = query_engine.query(specific_prompt) + response = answer.response if hasattr(answer, 'response') else answer.get('response', 'N/A') + chunk_results[attribute] = response + + all_results.append(chunk_results) + page_number += len(chunk) + + final_df = pd.DataFrame(all_results) + return final_df def run_bottom_up_lesser(d, text_dict, tokens=8000): diff --git a/src/prompts.py b/src/prompts.py index 599563f..6311398 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -145,3 +145,118 @@ Return a value for each type of code - return N/A for any values not found. Mult Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ + + +def prompt_contract_lob(): + return """Extract all instances of the Line of Business (LOB) mentioned in the text. + Identify each distinct LOB and return them as separate objects. Possible values include: + - Commercial + - Commercial/Group + - MARKETPLACE + - MEDICAID + - MEDICARE + - Marketplace/Exchange + - Medicaid/Medicare + - Medicare Advantage + - Medicare Supplemental + - Veterans Affairs (VA) + - Civilian Health and Medical Program of the Department of Veterans Affairs (CHAMPVA) + - Tricare (health care program for uniformed service members, retirees, and their families around the world) + If no LOB is found, return 'N/A'. + + IT IS NOT GOING TO BE THINGS LIKE HMO, PPO, CHIP, STAR, ETC + It will not be anything related to a 'Plan', so do not return anything like that. + Return the results. If there are multiple identified then return seperated by commas. Only return the extracted information and no other text or sentences or explanations. Only the final values. """ + + +def prompt_product(): + return """ + Identify and list all instances of the Plan or Product names mentioned in the text. In the context of health insurance, 'Plan' or 'Product' names are typically brand names of health plans provided by insurers. These names often reflect the type of coverage offered, its scope, or the insurer's branding. Examples of the types of coverage include 'Medical', 'Prescription Drug', 'Mental/Behavioral Health (MH/BH)', 'Substance Abuse (SA)', 'Hearing', 'Dental', and 'Vision'. + + To recognize these names, look for capitalized terms or phrases often preceded by words like 'plan', 'coverage', 'program', or directly followed by 'network'. These names can be unique to the insurer or widely recognized in the industry. For instance, names like 'HealthPlus Medical Plan', 'VisionCare Network', or 'DentalSelect Coverage' are typical examples. + + Extract and list all identified plan or product names. If multiple names are found, return them separated by commas. If no specific plan name is found, return 'N/A'. Only return the extracted information and no other text or sentences or explanations. Only the final values. + """ + +def prompt_network_name(): + #MODIFY TO DICTS FOR MAPPING + return """ + Extract all network names mentioned in the text. A 'network name' in the context of health insurance refers to the type of managed care organization involved in the delivery of healthcare services. These names often indicate the structure or model of care delivery, which may define the relationships between insurers, healthcare providers, and insured individuals. + + Key identifiers of network types include terms that suggest managed care arrangements such as 'Health Maintenance Organization (HMO)', 'Preferred Provider Organization (PPO)', 'Exclusive Provider Organization (EPO)', 'Point of Service (POS)', and 'Indemnity or Fee for Service (FFS)'. Other descriptors might include 'Open Access', 'Exclusive Provider', 'Integrated Delivery Network', and 'Tiered Network'. + + Methodically scan the text for any terms that align with these descriptions. Return all found network names separated by commas if multiple. If no network names are found in the text, return 'N/A'. Only return the extracted information and no other text or sentences or explanations. Only the final values. + """ + + +def prompt_service_area(): + return """ + Extract the type of service area mentioned. A 'Service Area' refers to the specific geographic or demographic region type where the health plan provides coverage or services. This could include specific states, counties, cities, or distinctions between urban and rural areas. + Select from the following options: Urban, Rural, State, County, City, ZIP, Address + + Extract and return all found attributes separated by commas in a single string. If no specific service areas are mentioned in the text, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values. + """ + +def prompt_effective_date(): + return """Identify all instances of Effective Dates mentioned within the context of specific compensation exhibits or contracts. + The date format should be MM/DD/YYYY. + Return all dates seperated by commas. If no effective dates are found, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values.' + DO NOT REPEAT THE SAME DATE MULTIPLE TIMES. + """ + +def prompt_termination_date(): + return """ + Extract all termination dates mentioned in the text, specifically those related to the end of a compensation period or agreement. Termination dates specify when a contract or policy provision ceases to be effective. + + To identify termination dates, look for dates associated with terms like 'end', 'expiration', 'termination', or similar phrases that indicate the conclusion of an agreement. These dates are typically formatted in MM/DD/YYYY. + + Carefully scan the text for these indications and extract any dates that conform to this format. List each date and return them separated by commas if multiple are found. If no termination dates are found, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values. + DO NOT REPEAT THE SAME DATE MULTIPLE TIMES. + """ + +def prompt_metal_level(): + return """If the Line of Business is identified as Marketplace, extract all instances of metal levels such as Platinum, Gold, Silver, or Bronze. + List each metal level. If no metal levels are found or if the LOB is not Marketplace, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values seperated by commas.""" + +def prompt_claim_discount_ind(lob): + return f"For this line of business: {lob}, return with a 'Y' or 'NO' only whether the claim for this LOB is discounted. Only return as 'Y' or 'NO'." + +def prompt_claim_discount_percent_rate(lob): + return f"For this LOB only: {lob}, extract the claim discount percent rate." + +def prompt_claim_discount_start_date(lob): + return f"For this LOB only: {lob}, extract the claim discount start date." + +def prompt_claim_discount_termination_date(lob): + return f"For this LOB only: {lob}, extract the claim discount termination date." + +def prompt_premium_ind(lob): + return f"""For this Line of Business{lob}, indicate with a 'Y' or 'N' only whether the premium for this LOB is subject to any adjustments. Return only 'Y' or 'N'.""" + +def prompt_premium_percent(lob): + return f"""For this Line of Business{lob}, extract the percentage rate of premium adjustment if applicable. Provide the percentage as a numerical value only.""" + +def prompt_premium_start_date(lob): + return f"""For this Line of Business{lob}, identify the start date of the premium adjustments. Return the date in MM/DD/YYYY format only.""" + +def prompt_premium_termination_date(): + return """For this Line of Business, determine the termination date of the premium adjustments. Provide the date in MM/DD/YYYY format only.""" + +def prompt_sequestration_ind(): + return """For this Line of Business, indicate with a 'Y' or 'N' whether there is any sequestration applied. Return only 'Y' or 'N'.""" + +def prompt_sequestration_rate(): + return """For this Line of Business, extract the rate of sequestration as a percentage. Return the rate as a numerical value only.""" + +def prompt_sequestration_start_date(): + return """For this Line of Business, determine the start date for sequestration. Return the date in MM/DD/YYYY format only.""" + +def prompt_sequestration_termination_date(): + return """For this Line of Business, identify the termination date for sequestration. Provide the date in MM/DD/YYYY format only.""" + +def prompt_penalties_ind(): + return """For this Line of Business, indicate with a 'Y' or 'N' only whether there are any penalties applied. Return only 'Y' or 'N'.""" + +def prompt_penalties_rate(): + return """For this Line of Business, extract the rate of any penalties applied as a percentage. Provide the rate as a numerical value only.""" + diff --git a/src/utils.py b/src/utils.py index c9fa129..129c6bd 100644 --- a/src/utils.py +++ b/src/utils.py @@ -75,4 +75,21 @@ def consolidate_individual(input_folder='results', output_folder='output'): consolidated_df.to_csv(os.path.join(output_folder, filename)) +def preprocess_text_file(file_path): + # Attempt to open the file with UTF-8 encoding first + try: + with open(file_path, 'r', encoding='utf-8') as file: + text = file.read() + except UnicodeDecodeError: + # If UTF-8 fails, try reading the file with ANSI (cp1252) encoding + try: + with open(file_path, 'r', encoding='cp1252') as file: + text = file.read() + except UnicodeDecodeError: + # If ANSI also fails, log an error message or handle it accordingly + print(f"Failed to decode {file_path} with UTF-8 and cp1252 encodings.") + return [] + # Split the text into pages based on a specific marker + pages = re.split(r'Start of Page No\. = \d+', text) + return pages From 93ac5f46d9f62dd27f336809e714624f288a50df Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 17 May 2024 16:03:38 -0700 Subject: [PATCH 23/78] In progress commit --- src/config.py | 21 ++- src/dict_operations.py | 5 +- src/main.py | 3 +- src/prompt_funcs.py | 133 +++++++------------ src/prompts.py | 87 ++++++------- src/test.py | 5 + src/test_notebook.ipynb | 280 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 393 insertions(+), 141 deletions(-) create mode 100644 src/test.py create mode 100644 src/test_notebook.ipynb diff --git a/src/config.py b/src/config.py index a59d6ab..a7f4ee1 100644 --- a/src/config.py +++ b/src/config.py @@ -2,6 +2,7 @@ from datetime import datetime import boto3 +from llama_index.llms.bedrock import Bedrock # General Settings TEST = False # True to run test prompt - just for testing model connection @@ -27,12 +28,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXK6KGAGQU" -AWS_SECRET_ACCESS_KEY="v3q+R5QQUcV/4/KiRQeacVbzhLqUq7odldmMVGof" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjECoaCXVzLWVhc3QtMiJGMEQCIBkx50iMSg0w6ghdK7+3IvA1L8wL6NuCyyR3CzTGDwBCAiAN7/AkQCVKF48z5LZyD1qIRtyj+jA3hnUds98Od7BxLSqMAwiT//////////8BEAAaDDY2MDEzMTA2ODc4MiIMTBr6D5ElCcEBiYgcKuACjM5ZgbJ6uOtsZDPfCxysiCAxOaFccomp45MSUGxeH72a0hVYO2MbqPJnimNDCF5IsIRAX/uJX/MTQJZykZyP8YRkGtjFoQx7BzQYX9rNxmGKXCc2yYC9w66POLqcHhcMfPBlFIn8Tux4ocXSs5t6aVOoqAhftv9GUiVYDKDpNGjWZPjczWj4sDVWYXkBvPn3LjAC69tQM2bYLAzwYZWu92r1doRbN+oZewjPuOzShsMHEidTkkjIPUvHa5mOqdIQmTYZi8QFHWHOZ7ui8XaHlxV5EozIlUYAYkYYaBR53XuduGTn2AQOmXM6pFdkCK6r6KwuLH0+g2NsXex5Cx37OAr3e/yDnYcASOiLf1ySJ1E7/QICZMtFDR46BILAYA8oFaNPJUHh2c7HndZFMTkveprB5BCMf+SNV+WhgSZu87rv4zNo+CbLxMEU1a2kPKZtROTUvV6qVqhzDKdGpdKs0zDvzY6yBjqnATvEOOkXIa5Z+QKAsDff7kpAqpQrNReV88D0E8pOTGNIp/EgS2jwQwf1vWtMGlEbB2HHZ4jyydL6B1OM+5W8pPWuJKrjaOcoDWyUld5aZysdEOxCmXGVW8NLC5HSieTrAxkdDTQ8DpuefHXN4QeeGiwzrOJzTubH4CLuJOCRcog6dtMouJOO2ccVNTNU0s2Y1eZUdsg28ZTDCa1G506M/Fu7gaGEuw9K" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXEQ2HHKUH" +AWS_SECRET_ACCESS_KEY="H46tkt0rtDzFD9onKyc0Nom1YUqFJT22ETYHu5t3" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHEaCXVzLWVhc3QtMiJHMEUCIH7v2k4Una8/SBCwtkBDcoqMEKdeqcRe4QmIBVyhQQspAiEA/+eZ1Wgd/DpW7nmi/MKDBqPCn3RcorlUTnCN0AKGFocqjAMI2v//////////ARAAGgw2NjAxMzEwNjg3ODIiDFOm/wbdtPj/ODXWqirgAk910BG442ShfneB9ihg3vRax86LbRi1V6WZ5yyo6tMK8MwZiwEtfK/JmB9r1+TxRqex+Ig3+UAxDVdD8xCxQct+5DVMzO6TF5TfN8I7LayxJwuVM5xvy0sQlAD45BLe+lw5HLs1diIDd8biFR4pljtVjaWEEn5zpomSVgQkG/VP+rxMrr1QpXYx//vaoZU7SNLeeXXMvW4CIct1/QGjUH0fkWS1LXbesCrWbP938yoDOsZxniL4K8MyJJ/LRTDacmYXB68iCL3SvfnCh0mLUgo1FlJZEeZH5H5f3T1As4yAssxZ9Zvmn40z08cJr3xuQygvLyURXe+uZiXZYHbEJthnvnIm1i6/CxZMkCteLTVX5o1TIlkTfF1qcXk8WnsjYTpIqGHGVUv84AXXtOOZ5JjPN+JIRPD+Esfrs8WGOI/4MGlnprrDn+hDYhwH6w+EDWC/k+TeKE9bIWr5CoenB38wzpCesgY6pgHDTgNCp5Xm4uHTeMIKpVstw6cfcXmX+Qyt5Z4q/ASj+4la0A+J7Ab31W+uD8UijdGHLJRtiMImuB2QrzxXrz2+kclM8b2X7dcu88Kcj39Ufn1fpQhqGF8o1R/bdsdCtwBXq0d9J2se2cJohRa3IjiPYhmJW+8z3PcbvDGnj4ROCzYVunnMROBdsIydNOfRwhMa5WEBAG0OPuWr4LLsj6IjEo5QSPPp" # File Paths -LOCAL_PATH = 'docs/all_txt_updated/' # Replace with local +LOCAL_PATH = '../docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', @@ -54,3 +55,15 @@ BEDROCK_RUNTIME = boto3.client(service_name="bedrock-runtime", MODEL_ID_CLAUDE3_HAIKU = 'anthropic.claude-3-haiku-20240307-v1:0' MODEL_ID_CLAUDE3_SONNET = 'anthropic.claude-3-sonnet-20240229-v1:0' MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1' + +# LLama Model +LLAMA_MODEL = Bedrock( + model=MODEL_ID_CLAUDE3_HAIKU, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + aws_session_token=AWS_SESSION_TOKEN, + region_name="us-east-1" + ) + + + diff --git a/src/dict_operations.py b/src/dict_operations.py index 93ff4b2..3a41713 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -4,7 +4,6 @@ import json from collections import defaultdict def secondary_string_to_dict(dict_string): - print(dict_string) dict_string = dict_string.replace('<<<', '') dict_string = dict_string.replace('>>>', '') dict_string = dict_string.replace('\n', '') # Remove new line @@ -19,8 +18,7 @@ def secondary_string_to_dict(dict_string): result_dict = json.loads(dict_substring) return result_dict -def primary_string_to_dict(string_dict): - print(string_dict) +def primary_string_to_dict(string_dict, filename): data = [] pattern = r'\{.*?\}' for page_num in string_dict.keys(): @@ -35,6 +33,7 @@ def primary_string_to_dict(string_dict): dict_list = [secondary_string_to_dict(d) for d in dict_list] for dict_ in dict_list: dict_['page_num'] = page_num + dict_['Filename'] = filename data.append(dict_) return data diff --git a/src/main.py b/src/main.py index 8d7b6d3..744f71c 100644 --- a/src/main.py +++ b/src/main.py @@ -33,7 +33,8 @@ def main(): def process_item(item): key, value = item try: - prompt_funcs.bottom_up(item) + prompt_funcs.run_prompts(item) + #prompt_funcs.bottom_up(item) except Exception as e: # Print the error message and traceback print(f"Error processing item {key}: {e}") diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 2ff25d2..45ebddd 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -16,63 +16,6 @@ import table_funcs import utils -def run_on_chunks_llama(input_filepath, num_chunks): - filename = os.path.basename(input_filepath) - pages = utils.preprocess_text_file(input_filepath) - - # Setup LLM with llamaindex - llm = Bedrock( - model="anthropic.claude-3-haiku-20240307-v1:0", - aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), - aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'), - aws_session_token=os.getenv('AWS_SESSION_TOKEN'), - region_name="us-east-1" - ) - - total_pages = len(pages) - #pages_per_chunk = (total_pages + num_chunks - 1) // num_chunks - - #chunks = [pages[i:i + pages_per_chunk] for i in range(0, len(pages), pages_per_chunk)] - pages_per_chunk = 1 # Each page is its own chunk - chunks = [pages[i:i + pages_per_chunk] for i in range(0, len(pages), pages_per_chunk)] - - all_results = [] - page_number = 1 - - # Define prompts for each attribute - prompt_functions = { - 'CONTRACT_LOB': prompts.prompt_contract_lob(), - 'PRODUCT': prompts.prompt_product(), - 'CONTRACT_NETWORK': prompts.prompt_network_name(), - 'CONTRACT_SERVICE_AREA': prompts.prompt_service_area(), - 'LOB_PRICING_TERMS_EFFECTIVE_DT': prompts.prompt_effective_date(), - 'LOB_PRICING_TERMS_TERMINATION_DT': prompts.prompt_termination_date(), - 'CONTRACT_MARKETPLACE_METAL_LEVEL' : prompts.prompt_metal_level() - } - - for chunk_index, chunk in enumerate(chunks): - chunk_text = " ".join(chunk) - end_page = page_number + len(chunk) - 1 - index = SummaryIndex([]) - doc = Document(text=chunk_text, id_=f"Chunk_{chunk_index + 1}") - index.insert(doc) - query_engine = index.as_query_engine(llm=llm) - - chunk_results = {'Filename': filename, 'Page': f"Chunk {chunk_index + 1} (Pages {page_number}-{end_page})"} - - for attribute, prompt_function in prompt_functions.items(): - specific_prompt = chunk_text + "\n\n" + prompt_function() # Get dynamic prompt - answer = query_engine.query(specific_prompt) - response = answer.response if hasattr(answer, 'response') else answer.get('response', 'N/A') - chunk_results[attribute] = response - - all_results.append(chunk_results) - page_number += len(chunk) - - final_df = pd.DataFrame(all_results) - return final_df - - def run_bottom_up_lesser(d, text_dict, tokens=8000): prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) @@ -175,7 +118,33 @@ def consolidate_lob(lob_dicts, results_dicts): return final_dicts -def bottom_up(file_object): +def run_bottom_up(filename, text_dict): + #### PROMPT #### + + # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY + answer_strings = run_bottom_up_primary(text_dict, 8000) + answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) + + # Bottom Up Secondary + results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) + + for d in results_dicts: + d['Filename'] = filename + return results_dicts # List of dictionaries + + +def run_top_down(filename, text_dict): + all_results = [] + for page_num, page_text in text_dict.items(): + if not page_num.isdigit(): + continue + prompt = prompts.TOP_DOWN_PRIMARY(page_text) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries + all_results.append(answer_dicts) + return all_results # List of list of dictionaries + +def run_prompts(file_object): filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") @@ -185,40 +154,26 @@ def bottom_up(file_object): text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) - #### PROMPT #### - # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY - answer_strings = run_bottom_up_primary(text_dict, 8000) - answer_dicts = dict_operations.primary_string_to_dict(answer_strings) + #### TOP DOWN #### + td_results = run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page + print(td_results) + + #### BOTTOM UP #### + bu_results = run_bottom_up(filename, text_dict) # Returns list of dictionaries - # Bottom Up LOB - CONTRACT_LOB, CONTRACT_PROGRAM, CONTRACT_PRODUCT, PROV_TYPE - if config.RUN_LOB: - lob_dicts = run_bottom_up_lob(answer_dicts, text_dict, 4000) - # Bottom Up Secondary - results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) - # Append LOB - if config.RUN_LOB: - final_dicts = consolidate_lob(lob_dicts, results_dicts) - else: - final_dicts = results_dicts - for d in final_dicts: - d['Filename'] = filename + #### OUTPUT #### + # if config.WRITE_OUTPUT: + # if config.OUTPUT_MODE == '_INDIVIDUAL_': + # answer_df.to_csv(f"results/{filename}_results.csv") + # elif config.OUTPUT_MODE == '_CONSOLIDATED_': + # if not os.path.exists('temp'): + # os.makedirs('temp') + # answer_df.to_csv(f"temp/{filename}_results.csv") + # else: + # if config.VERBOSE: print(answer_df) - answer_df = pd.DataFrame(final_dicts) + # Postprocess (added later) - #### POSTPROCESS #### - #answer_dicts = append_secondary(answer_dicts) - #answer_df = postprocess.bottom_up_postprocess(answer_df) - - # Write output - if config.WRITE_OUTPUT: - if config.OUTPUT_MODE == '_INDIVIDUAL_': - answer_df.to_csv(f"results/{filename}_results.csv") - elif config.OUTPUT_MODE == '_CONSOLIDATED_': - if not os.path.exists('temp'): - os.makedirs('temp') - answer_df.to_csv(f"temp/{filename}_results.csv") - else: - if config.VERBOSE: print(answer_df) diff --git a/src/prompts.py b/src/prompts.py index 6311398..4751b48 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -12,8 +12,6 @@ If any of the attributes are not found, return N/A. For all attributes, only wri Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. -Note that some of the answers may be found in tables indicated by -------Table Start-------- and -------Table End--------. Use knowledge of json format to extract relevant table information. - Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. @@ -25,31 +23,6 @@ Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be pop Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ -def BOTTOM_UP_LOB(page): - return f"""### PAGE START ### {page} ### PAGE END - -The above text represents two pages from a contract between a healthcare Payer and Provider. - -Your job is to identify what LOBs, Products, Plans, and Provider Types the reimbursement terms on this page apply to. - -Return a dictionary object with the following attributes: -'CONTRACT_LOB' : What Line of Business is the reimbursement schedule for? It will be Medicaid, Medicare, Marketplace, or similar. It will likely only be listed once per page, if at all. -'CONTRACT_PROGRAM' : What Program is the reimbursement schedule for? This may be something like CHIP, CHIP PERINATE, STAR, STAR PLUS, DSNP, Dual, or similar. This will likely only be listed once per page, if at all. -'PRODUCT' : What Product is the reimbursement schedule for? This the brand name of the health plan. It is NOT the same as LOB. It is NOT the name of a hospital. This will likely only be listed once per page, if at all. -'PROV_TYPE' : For what provider type or place of service is this fee schedule for? This will likely only be listed once per page, if at all. - -For CONTRACT_LOB, choose ONLY from the following: 'Medicaid', 'Medicare Advantage', 'Medicare Supplement', 'Marketplace', 'Commercial', 'Group' -Some examples of PROV_TYPE are: 'Primary Care Provider', 'Specialist', 'Ancillary', 'Ambulatory Surgery Center', 'Hospital','Rehab Facility', 'Skilled Nursing Facility', 'Ambulance', 'Rural Health Clinic', 'Home Health' -This list is non-exhaustive. - -If there are multiple correct answers for any of the above attributes, return only the answer corresponding to the SECOND of the two pages. -If there are multiple correct answers for the second page, return them both in a comma-separated list. This is extremely rare - there will almost always be one. -Sometimes the information is presented as 'SELECTED' OR 'NOT_SELECTED' - when this is the case, 'SELECTED' indicates the correct value(s). - -Return N/A for any answers not found. - -Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. -""" def BOTTOM_UP_LESSER(d, page): return f""" @@ -146,9 +119,27 @@ Return a value for each type of code - return N/A for any values not found. Mult Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ +def TOP_DOWN_PRIMARY(page): + return f"""### PAGE START ### {page} ### PAGE END -def prompt_contract_lob(): - return """Extract all instances of the Line of Business (LOB) mentioned in the text. +The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a list of dictionaries with the values below. + +Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: +'CONTRACT_LOB' : List any Line of Business mentioned on the page. CONTRACT_LOB will ONLY be one of the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs +'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS +'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS +'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. + +If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. + +Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +def prompt_contract_lob(page): + return f"""### PAGE START ### {page} ### PAGE END + + Extract all instances of the Line of Business (LOB) mentioned on the page. Identify each distinct LOB and return them as separate objects. Possible values include: - Commercial - Commercial/Group @@ -169,19 +160,21 @@ def prompt_contract_lob(): Return the results. If there are multiple identified then return seperated by commas. Only return the extracted information and no other text or sentences or explanations. Only the final values. """ -def prompt_product(): - return """ - Identify and list all instances of the Plan or Product names mentioned in the text. In the context of health insurance, 'Plan' or 'Product' names are typically brand names of health plans provided by insurers. These names often reflect the type of coverage offered, its scope, or the insurer's branding. Examples of the types of coverage include 'Medical', 'Prescription Drug', 'Mental/Behavioral Health (MH/BH)', 'Substance Abuse (SA)', 'Hearing', 'Dental', and 'Vision'. +def prompt_product(page): + return f"""### PAGE START ### {page} ### PAGE END + + Identify and list all instances of the Plan or Product names mentioned on the page. In the context of health insurance, 'Plan' or 'Product' names are typically brand names of health plans provided by insurers. These names often reflect the type of coverage offered, its scope, or the insurer's branding. Examples of the types of coverage include 'Medical', 'Prescription Drug', 'Mental/Behavioral Health (MH/BH)', 'Substance Abuse (SA)', 'Hearing', 'Dental', and 'Vision'. To recognize these names, look for capitalized terms or phrases often preceded by words like 'plan', 'coverage', 'program', or directly followed by 'network'. These names can be unique to the insurer or widely recognized in the industry. For instance, names like 'HealthPlus Medical Plan', 'VisionCare Network', or 'DentalSelect Coverage' are typical examples. Extract and list all identified plan or product names. If multiple names are found, return them separated by commas. If no specific plan name is found, return 'N/A'. Only return the extracted information and no other text or sentences or explanations. Only the final values. """ -def prompt_network_name(): +def prompt_network_name(page): #MODIFY TO DICTS FOR MAPPING - return """ - Extract all network names mentioned in the text. A 'network name' in the context of health insurance refers to the type of managed care organization involved in the delivery of healthcare services. These names often indicate the structure or model of care delivery, which may define the relationships between insurers, healthcare providers, and insured individuals. + return f"""### PAGE START ### {page} ### PAGE END + + Extract all network names mentioned on the page. A 'network name' in the context of health insurance refers to the type of managed care organization involved in the delivery of healthcare services. These names often indicate the structure or model of care delivery, which may define the relationships between insurers, healthcare providers, and insured individuals. Key identifiers of network types include terms that suggest managed care arrangements such as 'Health Maintenance Organization (HMO)', 'Preferred Provider Organization (PPO)', 'Exclusive Provider Organization (EPO)', 'Point of Service (POS)', and 'Indemnity or Fee for Service (FFS)'. Other descriptors might include 'Open Access', 'Exclusive Provider', 'Integrated Delivery Network', and 'Tiered Network'. @@ -189,23 +182,27 @@ def prompt_network_name(): """ -def prompt_service_area(): - return """ - Extract the type of service area mentioned. A 'Service Area' refers to the specific geographic or demographic region type where the health plan provides coverage or services. This could include specific states, counties, cities, or distinctions between urban and rural areas. +def prompt_service_area(page): + return f"""### PAGE START ### {page} ### PAGE END + + Extract the type of service area mentioned on the page. A 'Service Area' refers to the specific geographic or demographic region type where the health plan provides coverage or services. This could include specific states, counties, cities, or distinctions between urban and rural areas. Select from the following options: Urban, Rural, State, County, City, ZIP, Address Extract and return all found attributes separated by commas in a single string. If no specific service areas are mentioned in the text, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values. """ -def prompt_effective_date(): - return """Identify all instances of Effective Dates mentioned within the context of specific compensation exhibits or contracts. +def prompt_effective_date(page): + return f"""### PAGE START ### {page} ### PAGE END + + Identify all instances of Effective Dates mentioned within the context of specific compensation exhibits or contracts. The date format should be MM/DD/YYYY. Return all dates seperated by commas. If no effective dates are found, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values.' DO NOT REPEAT THE SAME DATE MULTIPLE TIMES. """ -def prompt_termination_date(): - return """ +def prompt_termination_date(page): + return f"""### PAGE START ### {page} ### PAGE END + Extract all termination dates mentioned in the text, specifically those related to the end of a compensation period or agreement. Termination dates specify when a contract or policy provision ceases to be effective. To identify termination dates, look for dates associated with terms like 'end', 'expiration', 'termination', or similar phrases that indicate the conclusion of an agreement. These dates are typically formatted in MM/DD/YYYY. @@ -214,8 +211,10 @@ def prompt_termination_date(): DO NOT REPEAT THE SAME DATE MULTIPLE TIMES. """ -def prompt_metal_level(): - return """If the Line of Business is identified as Marketplace, extract all instances of metal levels such as Platinum, Gold, Silver, or Bronze. +def prompt_metal_level(page): + return f"""### PAGE START ### {page} ### PAGE END + + If the Line of Business is identified as Marketplace, extract all instances of metal levels such as Platinum, Gold, Silver, or Bronze. List each metal level. If no metal levels are found or if the LOB is not Marketplace, return 'N/A'. only return the extracted information and no other text or sentences or explanations. Only the final values seperated by commas.""" def prompt_claim_discount_ind(lob): diff --git a/src/test.py b/src/test.py new file mode 100644 index 0000000..9d3aa07 --- /dev/null +++ b/src/test.py @@ -0,0 +1,5 @@ + + +#import llama_index +import openai +print(openai.__version__) \ No newline at end of file diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb new file mode 100644 index 0000000..20e9ab1 --- /dev/null +++ b/src/test_notebook.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import pandas as pd\n", + "import re\n", + "import ast\n", + "\n", + "import prompt_funcs\n", + "import config\n", + "import utils\n", + "import preprocess\n", + "import table_funcs\n", + "import claude_funcs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# Upload \n", + "input_dict = utils.read_input()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "filename = '6.2000_Omni Transport Systems_ServicesAgreement MU.txt'\n", + "contract_text = input_dict['6.2000_Omni Transport Systems_ServicesAgreement MU.txt']\n", + "\n", + "contract_text = preprocess.clean_newlines(contract_text)\n", + "text_dict = preprocess.split_text(contract_text)\n", + "text_dict = table_funcs.align_and_format_tables(text_dict)\n", + "text_dict = preprocess.highlight_rates(text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "td_results = prompt_funcs.run_top_down(filename, text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "bu_results = prompt_funcs.run_bottom_up(filename, text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "td_results = [i for d in td_results for i in d] # Turn list of list of dicts into list of dicts" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "def LOB_CHECK(bu_dict, td_dicts, page):\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + " Service: {bu_dict['SERVICE']}\n", + " Methodology: {bu_dict['FULL_METHODOLOGY']}\n", + "\n", + " Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects:\n", + "\n", + " {td_dicts}\n", + "\n", + " Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", + " \"\"\"\n", + "\n", + "def format_td_check(td_dicts):\n", + " final_str = \"\"\n", + " dict_count = 1\n", + " for td_dict in td_dicts:\n", + " final_str += str(dict_count) + '. '\n", + " for k in td_dict.keys():\n", + " if k not in ['Filename', 'page_num']:\n", + " final_str += k + ': ' + td_dict[k] + ', '\n", + " final_str += '\\n'\n", + " dict_count += 1\n", + " return final_str\n", + "\n", + "def td_bu_combine(td, bu):\n", + " combined_list = []\n", + " for bud in bu:\n", + " td_dicts = [d for d in td if d['page_num'] == bud['page_num']]\n", + " # Case 1: Only one option\n", + " if len(td_dicts) == 1:\n", + " td_dict = td_dicts[0]\n", + " bud.update(td_dict)\n", + " elif len(td_dicts) > 1: # Case 2: More than one option\n", + " td_check_formatted = format_td_check(td_dicts)\n", + " prompt = LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']])\n", + " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " print(answer)\n", + " answer_idx = ''.join(re.findall(r'\\d', answer))\n", + " bud.update(td_dicts[int(answer_idx)-1])\n", + " else: # Case 3: No option for page\n", + " continue\n", + " combined_list.append(bud)\n", + " return combined_list\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "2\n", + "2\n" + ] + } + ], + "source": [ + "combined_results = td_bu_combine(td_results, bu_results)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "import re\n", + "import pandas as pd\n", + "import ast\n", + "\n", + "def extract_tables(text):\n", + " pattern = re.compile(r\"-------Table Start--------(.*?)-------Table End--------\", re.DOTALL)\n", + " tables = pattern.findall(text)\n", + " table_list = [table.strip() for table in tables]\n", + " table_list = ['{'+table.split('{')[1] for table in table_list]\n", + " df_list = [pd.DataFrame(ast.literal_eval(table)) for table in table_list]\n", + " for df in df_list:\n", + " print(df.columns)\n", + " return df_list\n", + "\n", + "def extract_all_tables(input_dict):\n", + " tables_dict = {}\n", + " for filename, filetext in input_dict.items():\n", + " tables_dict[filename] = extract_tables(filetext)\n", + " return tables_dict\n", + "\n", + "def format_table(table):\n", + " table = table.replace('>>>', '').replace('<<<', '')\n", + " table_dict = eval(table)\n", + " \n", + " keys = list(table_dict.keys())\n", + " # If values are lists\n", + " if isinstance(table_dict[keys[0]], list):\n", + " final_string = \"\"\n", + " for i in range(len(table_dict[keys[0]])):\n", + " for k in keys:\n", + " key_string = k + ': ' + table_dict[k][i] + ', '\n", + " final_string += key_string\n", + " final_string += '|\\n'\n", + " else:\n", + " # If values are not lists\n", + " final_string = \"\"\n", + " for k in keys:\n", + " key_string = k + ': ' + table_dict[k] + ', '\n", + " final_string += key_string\n", + " final_string += '|\\n'\n", + " return final_string\n", + "\n", + "def align_and_format_tables(text_dict):\n", + " pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)\n", + " for page, text in text_dict.items():\n", + " tables = pattern.findall(text)\n", + " for table in tables:\n", + " pre_table = table.split('{')[0].strip() # Get the pretable text for alignment\n", + " text = text.replace(table, '') # Remove full table\n", + " table_only = table.replace(pre_table, '')\n", + " try:\n", + " formatted_table = format_table(table_only)\n", + " except:\n", + " formatted_table = table\n", + " text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table\n", + " text = text.replace('-------Table Start--------', '')\n", + " text = text.replace('-------Table End--------', '') \n", + " text_dict[page] = text\n", + " return text_dict\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "table_edit = align_and_format_tables(input_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keys(['Baz Allergy Asthma & Sinus_MU.txt'])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "table_edit['Baz Allergy Asthma & Sinus_MU.txt']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 016ba74082e6eaac290e5a1ad109a80e51ecb735 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 21 May 2024 09:57:04 -0400 Subject: [PATCH 24/78] TD/BU Updates --- src/config.py | 6 +- src/prompt_funcs.py | 68 +- src/prompts.py | 30 +- src/test_notebook.ipynb | 2349 ++++++++++++++++++++++++++++++++++++++- src/utils.py | 14 + 5 files changed, 2425 insertions(+), 42 deletions(-) diff --git a/src/config.py b/src/config.py index a7f4ee1..3a42fdf 100644 --- a/src/config.py +++ b/src/config.py @@ -28,9 +28,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXEQ2HHKUH" -AWS_SECRET_ACCESS_KEY="H46tkt0rtDzFD9onKyc0Nom1YUqFJT22ETYHu5t3" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHEaCXVzLWVhc3QtMiJHMEUCIH7v2k4Una8/SBCwtkBDcoqMEKdeqcRe4QmIBVyhQQspAiEA/+eZ1Wgd/DpW7nmi/MKDBqPCn3RcorlUTnCN0AKGFocqjAMI2v//////////ARAAGgw2NjAxMzEwNjg3ODIiDFOm/wbdtPj/ODXWqirgAk910BG442ShfneB9ihg3vRax86LbRi1V6WZ5yyo6tMK8MwZiwEtfK/JmB9r1+TxRqex+Ig3+UAxDVdD8xCxQct+5DVMzO6TF5TfN8I7LayxJwuVM5xvy0sQlAD45BLe+lw5HLs1diIDd8biFR4pljtVjaWEEn5zpomSVgQkG/VP+rxMrr1QpXYx//vaoZU7SNLeeXXMvW4CIct1/QGjUH0fkWS1LXbesCrWbP938yoDOsZxniL4K8MyJJ/LRTDacmYXB68iCL3SvfnCh0mLUgo1FlJZEeZH5H5f3T1As4yAssxZ9Zvmn40z08cJr3xuQygvLyURXe+uZiXZYHbEJthnvnIm1i6/CxZMkCteLTVX5o1TIlkTfF1qcXk8WnsjYTpIqGHGVUv84AXXtOOZ5JjPN+JIRPD+Esfrs8WGOI/4MGlnprrDn+hDYhwH6w+EDWC/k+TeKE9bIWr5CoenB38wzpCesgY6pgHDTgNCp5Xm4uHTeMIKpVstw6cfcXmX+Qyt5Z4q/ASj+4la0A+J7Ab31W+uD8UijdGHLJRtiMImuB2QrzxXrz2+kclM8b2X7dcu88Kcj39Ufn1fpQhqGF8o1R/bdsdCtwBXq0d9J2se2cJohRa3IjiPYhmJW+8z3PcbvDGnj4ROCzYVunnMROBdsIydNOfRwhMa5WEBAG0OPuWr4LLsj6IjEo5QSPPp" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXC5Y4PKXJ" +AWS_SECRET_ACCESS_KEY="f/mOlRpK7+iSlm1P5ZK1LOdS2UjnsbAQ2ysL7Mms" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEM3//////////wEaCXVzLWVhc3QtMiJGMEQCIGqPr40CU3Fz4nvSQJ1jaPgkOGIWWyDXBVLUU+VmsfIjAiAgluQOwSGIYph5hLcKVVsiFQ7OpLOs3lLsumxzeyw6nyqDAwhGEAAaDDY2MDEzMTA2ODc4MiIM5odYPf2eQP8/f9NBKuAC7eMH0qjb6ASoeBhtKNFzMWXpR5snAahfBheyiUo1my8q8VFbZem8m0n5c0vWnKsPhm9+K+qpVqM6ohhjVMIXCta/ZlGNizatHJt7gMpDOGQNyBI4rpXF9/urQMgN8PkjMG2hkD+XefkPYGmvgz9ZVKFs/oAnP/buG21Ju54F/D0qqnxhR/90pVM6Z48sgzkQThHHjB7wNgfy1hIA7CL9yeM0k+sdzlYvriDogDi3lcP3UkXMUJhQl5xQb7t8iLR70L/0H74ynfldpbaBfktWPred4ehoRMv2Zd/Jh6JNBIAeDQyfIKQTYn2FF/RuPBmqa+iLS9lJh/SnhwXNuMYc9JOERwMHaijgqQjenISnysSEr10u1Y+XiUI7waXRO6e8CBS/LygUKZZf8+cSeXIE3VN2XrMW72Ix5sg16QHAMvR3oUDJdYdf8jnWET995kocK6UctJdVKEna/FUnIdNgwzDirrKyBjqnAZjoQpVVuEh8p9tuGrjdWsjzLsXRkNRyqm/6xxIGtqITzvAv+TKQLSgJun/rRTOncaMLDwt0PAKL1Y8cVnOcuSIfVFUtpeZ+p27MgByL/fBQGd0yfZGHy15ZMPw51viYhjQo3MeA4y0YLRW+12v2ilrMxGHYlwTUT19Zj+qQNobTrlcJWQ7737lXmVL2WOZzmtduH7wf/fyAn2ZhZT/lcnSSQsef9ORk" # File Paths LOCAL_PATH = '../docs/test/' # Replace with local diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 45ebddd..bd91010 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -1,6 +1,7 @@ import os +import re import pandas as pd from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate @@ -94,19 +95,6 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts -def run_bottom_up_lob(answer_dicts, text_dict, tokens): - lob_dict = {} - for page_num in set(d['page_num'] for d in answer_dicts): - try: - text_section = text_dict[str(int(page_num)-1)] + text_dict[page_num] - except: - text_section = text_dict[page_num] - prompt = prompts.BOTTOM_UP_LOB(text_section) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - answer_dict = dict_operations.secondary_string_to_dict(answer) - lob_dict[page_num] = answer_dict - return lob_dict - def consolidate_lob(lob_dicts, results_dicts): final_dicts = [] for d in results_dicts: @@ -119,7 +107,6 @@ def consolidate_lob(lob_dicts, results_dicts): def run_bottom_up(filename, text_dict): - #### PROMPT #### # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) @@ -142,7 +129,32 @@ def run_top_down(filename, text_dict): answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries all_results.append(answer_dicts) - return all_results # List of list of dictionaries + all_results_final = [i for d in all_results for i in d] # Consolidate to one list + return all_results_final # List of list of dictionaries + + +def td_bu_combine(td, bu, text_dict): + combined_list = [] + for bud in bu: + td_dicts = [d for d in td if d['page_num'] == bud['page_num']] + # Case 1: Only one option + if len(td_dicts) == 1: + td_dict = td_dicts[0] + bud.update(td_dict) + # Case 2: More than one option + elif len(td_dicts) > 1: + td_check_formatted = utils.format_td_check(td_dicts) + prompt = prompts.LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']]) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + print(answer) + answer_idx = ''.join(re.findall(r'\d', answer)) + bud.update(td_dicts[int(answer_idx)-1]) + # Case 3: No option for page + else: + continue + combined_list.append(bud) + return combined_list + def run_prompts(file_object): filename, contract_text = file_object @@ -156,24 +168,26 @@ def run_prompts(file_object): #### TOP DOWN #### td_results = run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page - print(td_results) #### BOTTOM UP #### bu_results = run_bottom_up(filename, text_dict) # Returns list of dictionaries + #### COMBINE #### + combined_results = td_bu_combine(td_results, bu_results, text_dict) + + answer_df = pd.DataFrame(combined_results) - - #### OUTPUT #### - # if config.WRITE_OUTPUT: - # if config.OUTPUT_MODE == '_INDIVIDUAL_': - # answer_df.to_csv(f"results/{filename}_results.csv") - # elif config.OUTPUT_MODE == '_CONSOLIDATED_': - # if not os.path.exists('temp'): - # os.makedirs('temp') - # answer_df.to_csv(f"temp/{filename}_results.csv") - # else: - # if config.VERBOSE: print(answer_df) + ### OUTPUT #### + if config.WRITE_OUTPUT: + if config.OUTPUT_MODE == '_INDIVIDUAL_': + answer_df.to_csv(f"results/{filename}_results.csv") + elif config.OUTPUT_MODE == '_CONSOLIDATED_': + if not os.path.exists('temp'): + os.makedirs('temp') + answer_df.to_csv(f"temp/{filename}_results.csv") + else: + if config.VERBOSE: print(answer_df) # Postprocess (added later) diff --git a/src/prompts.py b/src/prompts.py index 4751b48..5a89ee3 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -130,12 +130,40 @@ Here are the attributes to be included in each dictionary, and instructions on h 'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. -If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. +If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return them in a comma-separated list. + +For all attributes, only write what is written on the page. Do not make up new phrases or words. Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ +def LOB_CHECK(bu_dict, td_dicts, page): + return f"""### PAGE START ### {page} ### PAGE END + + Service: {bu_dict['SERVICE']} + Methodology: {bu_dict['FULL_METHODOLOGY']} + + Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects: + + {td_dicts} + + Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. + """ + + + + + + + + + + + + + + def prompt_contract_lob(page): return f"""### PAGE START ### {page} ### PAGE END diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb index 20e9ab1..19add88 100644 --- a/src/test_notebook.ipynb +++ b/src/test_notebook.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -33,10 +33,22 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "KeyError", + "evalue": "'6.2000_Omni Transport Systems_ServicesAgreement MU.txt'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[3], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m6.2000_Omni Transport Systems_ServicesAgreement MU.txt\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m----> 2\u001b[0m contract_text \u001b[38;5;241m=\u001b[39m \u001b[43minput_dict\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m6.2000_Omni Transport Systems_ServicesAgreement MU.txt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m 4\u001b[0m contract_text \u001b[38;5;241m=\u001b[39m preprocess\u001b[38;5;241m.\u001b[39mclean_newlines(contract_text)\n\u001b[0;32m 5\u001b[0m text_dict \u001b[38;5;241m=\u001b[39m preprocess\u001b[38;5;241m.\u001b[39msplit_text(contract_text)\n", + "\u001b[1;31mKeyError\u001b[0m: '6.2000_Omni Transport Systems_ServicesAgreement MU.txt'" + ] + } + ], "source": [ - "filename = '6.2000_Omni Transport Systems_ServicesAgreement MU.txt'\n", - "contract_text = input_dict['6.2000_Omni Transport Systems_ServicesAgreement MU.txt']\n", + "filename = 'Baz Allergy Asthma & Sinus_MU.txt'\n", + "contract_text = input_dict['Baz Allergy Asthma & Sinus_MU.txt']\n", "\n", "contract_text = preprocess.clean_newlines(contract_text)\n", "text_dict = preprocess.split_text(contract_text)\n", @@ -110,14 +122,16 @@ " if len(td_dicts) == 1:\n", " td_dict = td_dicts[0]\n", " bud.update(td_dict)\n", - " elif len(td_dicts) > 1: # Case 2: More than one option\n", + " # Case 2: More than one option\n", + " elif len(td_dicts) > 1: \n", " td_check_formatted = format_td_check(td_dicts)\n", " prompt = LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']])\n", " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", " print(answer)\n", " answer_idx = ''.join(re.findall(r'\\d', answer))\n", " bud.update(td_dicts[int(answer_idx)-1])\n", - " else: # Case 3: No option for page\n", + " # Case 3: No option for page\n", + " else:\n", " continue\n", " combined_list.append(bud)\n", " return combined_list\n", @@ -230,22 +244,2335 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Document Index\n", + "CALIFORNIA 1PROVIDER PARTICIPATION AGREEMENT 1FEE-FOR-SERVICE DIRECT NETWORK TEMPLATE 1\n", + "Table of ContentsSection/Addendum: Section I, Title: Definitions, Page: 1, |\n", + "Section/Addendum: Section II, Title: Duties of Provider, Page: 4, |\n", + "Section/Addendum: Section III, Title: Duties of Health Net, Page: 7, |\n", + "Section/Addendum: Section IV, Title: Financial Obligations, Page: 8, |\n", + "Section/Addendum: Section V, Title: Term and Tennination, Page: 12, |\n", + "Section/Addendum: Section VI, Title: Records, Audits and Regulatory Requirements, Page: 13, |\n", + "Section/Addendum: Section VII, Title: General Provisions, Page: 14, |\n", + "Section/Addendum: Exhibit I, Title: List of Facilities, Page: 19, |\n", + "Section/Addendum: Exhibit Il, Title: List of Professional Providers, Page: 20, |\n", + "Section/Addendum: Addendum A, Title: Commercial Benefit Programs; Provisions, Page: 21, |\n", + "Section/Addendum: Exhibit A-1, Title: Fee-For-Service Rate Exhibit - Commercial Benefit Programs, Page: 22, |\n", + "Section/Addendum: Addendum B, Title: Medicare Advantage Provisions, Page: 23, |\n", + "Section/Addendum: Exhibit B-1, Title: Medicare Advantage Fee-For-Service Rate Exhibit, Page: 27, |\n", + "Section/Addendum: Addendum C, Title: Medi-Cal Benefit Program; Provisions, Page: 28, |\n", + "Section/Addendum: Exhibit C-1, Title: Medi-Cal Fee-For-Service Rate Exhibit, Page: 35, |\n", + "Section/Addendum: Exhibit C-2, Title: Disclosure Form, Page: 36, |\n", + "Section/Addendum: Addendum D, Title: Healthy Families and Healthy Kids Benefit Program: Provisions, Page: 37, |\n", + "Section/Addendum: Exhibit D-1, Title: Healthy Families and Healthry Kids Benefit Program Rate Exhibit, Page: 39, |\n", + "Section/Addendum: Addendum E, Title: Direct Network Fee-For-Service Payment Conditions, Page: 40, |\n", + " 1\n", + "CALIFORNIA 2PROVIDER PARTICIPATION AGREEMENT 2FEF-FOR-SERVICE DIRECT NETWORK TEMPLATE 2\n", + "RECITALS 2\n", + "AGREEMENT 2\n", + "I. 2DEFINITIONS 2\n", + "II. 5DUTIES OF PROVIDER 5\n", + "III. 8DUTIES OF HEALTHNET 8\n", + "4.2 9Billing and Payment. 9\n", + "4.3 10Recompment of Overpayments; Right of Offset. 10\n", + "Net or a Payor. 12\n", + "V. 13TERM AND TERMINATION 13\n", + "VI. 14RECORDS, AUDITS AND REGULATORY REQUIREMENTS 14\n", + "VII. 15GENERAL PROVISIONS 15\n", + "7.9 18Indemnification. 18\n", + "THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED 19BY THE PARTIES, 19\n", + "EXHIBIT I 20LIST OF FACILITIES 20\n", + "Primary Location 20\n", + "Secondary Location 20\n", + "Remit Address 20\n", + "Baz Allergy, Asthma & Sinus Center 21\n", + "OFFICE LOCATIONS 21\n", + "Second Office Address: NPI# 1508885781 21\n", + "Third Office Address: NPI# 1023037108 21\n", + "Fourth Office Address: NPI # 1811916992 21\n", + "Fifth Office Address: NPI# 1215956396 21\n", + "Sixth Office Address NPI# 1336399013 21\n", + "EXHIBIT II 22LIST OF PROFESSIONAL PROVIDERS 22\n", + "INSERT ROSTER HERE 22\n", + "ADDENDUM A 23\n", + "COMMERCIAL BENEFIT PROGRAMS 23\n", + "NOT APPLICABLE 23\n", + "EXHIBIT A-1 24\n", + "DIRECT NETWORK FEE-FOR-SERVICE 24RATE EXHIBIT 24\n", + "COMMERCIAL BENEFIT PROGRAMS 24\n", + "ADDENDUMB 25\n", + "MEDICARE ADVANTAGE PROGRAM 25\n", + "EXHIBIT B-1 29\n", + "MEDICARE ADVANTAGE PROGRAM 29DIRECT NETWORK FEE-FOR-SERVICE 29RATE EXHIBIT 29\n", + "I. Payment Rates 29\n", + "ADDENDUM O 30\n", + "MEDI-CAL BENEFIT PROGRAM 30\n", + "A. 30COMPENSATION PROVISIONS. 30\n", + "B. 31GENERAL PROVISIONS 31\n", + "EXHIBIT C-2 38\n", + "DISCLOSURE FORM 38\n", + "ADDENDUMI 39\n", + "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS 39\n", + "A. 39COMPENSATION PROVISIONS 39\n", + "B. 39GENERAL PROVISIONS 39\n", + "EXHIBIT D-1 41\n", + "ADDENDUMI 42DIRECT NETWORK FEE-FOR-SERVICE PAYMENT CONDITIONS 42\n", + "\n", + "\n", + "\n", + "Start of Page No. = 1\n", + "CALIFORNIA\n", + "PROVIDER PARTICIPATION AGREEMENT\n", + "FEE-FOR-SERVICE DIRECT NETWORK TEMPLATE\n", + "Table of ContentsSection/Addendum: Section I, Title: Definitions, Page: 1, |\n", + "Section/Addendum: Section II, Title: Duties of Provider, Page: 4, |\n", + "Section/Addendum: Section III, Title: Duties of Health Net, Page: 7, |\n", + "Section/Addendum: Section IV, Title: Financial Obligations, Page: 8, |\n", + "Section/Addendum: Section V, Title: Term and Tennination, Page: 12, |\n", + "Section/Addendum: Section VI, Title: Records, Audits and Regulatory Requirements, Page: 13, |\n", + "Section/Addendum: Section VII, Title: General Provisions, Page: 14, |\n", + "Section/Addendum: Exhibit I, Title: List of Facilities, Page: 19, |\n", + "Section/Addendum: Exhibit Il, Title: List of Professional Providers, Page: 20, |\n", + "Section/Addendum: Addendum A, Title: Commercial Benefit Programs; Provisions, Page: 21, |\n", + "Section/Addendum: Exhibit A-1, Title: Fee-For-Service Rate Exhibit - Commercial Benefit Programs, Page: 22, |\n", + "Section/Addendum: Addendum B, Title: Medicare Advantage Provisions, Page: 23, |\n", + "Section/Addendum: Exhibit B-1, Title: Medicare Advantage Fee-For-Service Rate Exhibit, Page: 27, |\n", + "Section/Addendum: Addendum C, Title: Medi-Cal Benefit Program; Provisions, Page: 28, |\n", + "Section/Addendum: Exhibit C-1, Title: Medi-Cal Fee-For-Service Rate Exhibit, Page: 35, |\n", + "Section/Addendum: Exhibit C-2, Title: Disclosure Form, Page: 36, |\n", + "Section/Addendum: Addendum D, Title: Healthy Families and Healthy Kids Benefit Program: Provisions, Page: 37, |\n", + "Section/Addendum: Exhibit D-1, Title: Healthy Families and Healthry Kids Benefit Program Rate Exhibit, Page: 39, |\n", + "Section/Addendum: Addendum E, Title: Direct Network Fee-For-Service Payment Conditions, Page: 40, |\n", + "\n", + "California Provider Participation Agreement\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA- 04-08-11\n", + "\n", + "\n", + "\n", + "\n", + "Start of Page No. = 2\n", + "CALIFORNIA\n", + "PROVIDER PARTICIPATION AGREEMENT\n", + "FEF-FOR-SERVICE DIRECT NETWORK TEMPLATE\n", + "This Provider Participation Agreement (\"Agreement\") is made and entered into by and between the\n", + "Provider identified on the signature page of this Agreement (\"Provider\"), and Health Net of California, Inc. on\n", + "behalf of itself and the subsidiaries and affiliates of Health Net, Inc. (collectively, \"Health Net\"). This Agreement is\n", + "effective as of the effective date shown on the signature page of this Agreement. subject to the completion of any\n", + "applicable credentialing requirements.\n", + "RECITALS\n", + "A.\n", + "Provider has the legal authority to enter into this Agreement, and to deliver or arrange for the\n", + "delivery of Contracted Services.\n", + "B.\n", + "Health Net has the legal authority to enter into this Agreement, and to perform the obligations of\n", + "Health Net hereunder with respect to the Benefit Programs.\n", + "C.\n", + "The parties desire to enter into this Agreement to arrange for Provider to participate in one or more\n", + "of Health Net's networks of Participating Providers that render Contracted Services to Beneficiaries of various\n", + "Benefit Programs.\n", + "D.\n", + "Provider's primary consideration shall be the quality of the health care services rendered to\n", + "Beneficiaries, pursuant to Title 10 CCR 2240.4\n", + "AGREEMENT\n", + "NOW, THEREFORE in consideration of the above recitals and the covenants contained herein, the parties hereby\n", + "agree as follows:\n", + "I.\n", + "DEFINITIONS\n", + "Many words and terms are capitalized throughout this Agreement to indicate that they are defined as set forth in this\n", + "Article I.\n", + "1.1\n", + "Beneficiary. A person who is properly enrolled in and eligible to receive Covered Services under\n", + "a Benefit Program at the time Covered Services are rendered. The parties acknowledge that the term 'member' may\n", + "be used by Health Net in certain related materials, such as Benefit Program documents covering various products,\n", + "marketing materials, and Health Net Policies. For reference purposes, the term Beneficiary includes the term\n", + "\"member\" wherever used.\n", + "1.2\n", + "Benefit Program. The group agreement, evidence of coverage. certificate of insurance, summary\n", + "plan description or similar documents in effect at the time Covered Services are rendered for lines of business\n", + "offered through Health Net. The Benefit Programs in which Provider participates and terms and conditions such as\n", + "payment rates relating to such Benefit Programs, are set forth in the Addenda to this Agreement.\n", + "1.3\n", + "Coinsurance. That portion of the cost of Covered Services that a Beneficiary is obligated to pay\n", + "under a particular Benefit Program which is calculated as a percentage of the contracted reimbursement rate for such\n", + "services. Coinsurance does not include Copayments or Deductibles.\n", + "California Provider Participation Agreement\n", + "I\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 3\n", + "1.4\n", + "Complete Claim. A Complete Claim means a claim or portion thereof, if separable, including\n", + "attachments and supplemental information or documentation, which provides reasonably relevant information as\n", + "defined by applicable State or federal statutes and regulations, and which is submitted to Health Net or a Payor by\n", + "Provider for payment of Contracted Services that may be processed by Health Net or a Payor without obtaining\n", + "additional information from Provider or from a third party.\n", + "1.5\n", + "Contracted Services. Covered Services that are (i) those services which Provider is licensed to\n", + "provide and which Provider customarily provides to its patients, and (ii) to be provided to a Beneficiary under the\n", + "terms of the applicable Benefit Program in effect at the time such services are rendered or as required by State or\n", + "federal law, and (iii) compensated in accordance with this Agreement except as otherwise may be required by State\n", + "or federal law.\n", + "1.6\n", + "Coordination of Benefits. The allocation of financial responsibility between two or more payors\n", + "of health care services, each with a legal duty to pay for or provide Covered Services to a Beneficiary at the same\n", + "time.\n", + "1.7\n", + "Copayment. That portion of the cost of Covered Services that a Beneficiary is obligated to pay\n", + "under a particular Benefit Program, which generally is a fixed dollar amount and is paid at the time Covered\n", + "Services are rendered. Copayments do not include Coinsurance or Deductibles.\n", + "1.8\n", + "Covered Services. The health care services, equipment and supplies that are covered as\n", + "determined by the Benefit Program and by applicable State and federal law and regulations, including without\n", + "limitation decisions issued as a result of independent medical review conducted under applicable State or federal\n", + "law.\n", + "1.9\n", + "Deductible. The amount of money that a Beneficiary must pay before the Benefit Program pays\n", + "certain benefits for Covered Services. Deductibles do not include Coinsurance or Copayments.\n", + "1.10\n", + "Dispute. The term \"Dispute\". as used in this Agreement, including Sections 7.5 and 7.6, shall\n", + "mean any controversy or disagreement that may arise our of or relate to this Agreement, or the breach thereof,\n", + "whether involving a claim in tort, contract or other applicable area of law.\n", + "1.11\n", + "Emergency. The term \"Emergency\" shall mean a medical condition inanifesting itself by acute\n", + "symptoms of sufficient severity (including severe pain) such that a prudent layperson who passesses average\n", + "knowledge of health and medicine could reasonably expect the absence of immediate medical attention to result in:\n", + "(i) placing the health of the individual (and in the case of a pregnant woman, her health or that of her unborn child)\n", + "in serious jeopardy, or (ii) serious impairment to bodily functions, or (iii) serious dysfunction of any bodily organ\n", + "or\n", + "part.\n", + "1.12\n", + "Excluded Services. Those health care services, equipment and supplies that are determined by\n", + "Health Net or a Payor to be non-Covered Services under the applicable Benefit Program in effect at the time such\n", + "services are rendered and for which Provider may bill the Beneficiary.\n", + "1.13\n", + "Facility(ies). All service locations owned, operated, leased. or subcontracted by Provider at which\n", + "Contracted Services are provided under this Agreement. Provider's service locations as of the date this Agreement\n", + "is executed by the parties are listed on an exhibit to this Agreement.\n", + "1.14\n", + "Health Net. A network of managed health care delivery or indemnity companies, owned,\n", + "controlled, controlling, under common control with. managed or administered in whole or in part now or hereafter,\n", + "by Health Net. Inc., a Delaware Corporation, its successors and assigns.\n", + "1.15\n", + "Health Net Policies. The policies, procedures and programs established by Health Net and\n", + "applicable to Participating Providers in effect at the time Covered Services are rendered, including without\n", + "limitation Health Net's grievance and appeal procedures, provider dispute and/or appeal process, drug formulary\n", + "or\n", + "preferred drug list, fraud detection, recovery procedures, eligibility verification, billing and coding guidelines,\n", + "payment and review policies, anti-discrimination requirements, medical management programs, continuity of care\n", + "California Provider Participation Agreement\n", + "2\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 4\n", + "policies, provider manuals and/or operations manuals. The medical management program includes policies\n", + "regarding topics such as credentialing, ntilization management, quality improvement, catastrophic care management,\n", + "peer review, medical and other record reviews, outcome rate reviews, Prior Authorization, and Referral.\n", + "1.16\n", + "Medically\n", + "Necessary. A Medically Necessary service or supply is one that meets the following\n", + "criteria: it is an otherwise covered category of service, not specifically excluded and is recommended by the treating\n", + "physician and determined by Health Net's Medical Director or physician designee to be: (i) for the purpose of\n", + "treating a medical condition: (ii) the most appropriate supply or level of service, considering potential benefits and\n", + "harm to the Beneficiary; not furnished primarily for the convenience of the Beneficiary or Provider; not required\n", + "solely for custodial, comfort or maintenance reasons; consistent with Health Net Policies and furnished in the most\n", + "appropriate place of service consistent with nationally recognized review criteria and/or guidelines, such as, for\n", + "example, Milliman or Interqual criteria; and (iii) known to be effective and safe in improving health outcomes. For\n", + "new treatments. services or supplies, effectiveness is determined by scientific evidence. For existing treatments,\n", + "services or supplies, effectiveness is determined first by scientific evidence, then by professional standards, then by\n", + "expert opinion. The fact that a physician or other provider may prescribe, order. recommend or approve a service,\n", + "supply or hospitalization does not, in itself. make it Medically Necessary or make it a Covered Service.\n", + "1.17\n", + "Participating Provider. A facility, physician, physician organization, physician group,\n", + "independent practice association, health care provider, supplier, or other organization which has met applicable\n", + "credentialing requirements, if any, and has, or is governed by, an effective written agreement directly with Health\n", + "Net or indirectly through another entity, such as a PPG, to provide Covered Services.\n", + "1.18\n", + "Pavor. Any public or private entity contracted with Health Net which provides, administers,\n", + "funds, insures or is responsible for paying Participating Providers for Covered Services rendered to Beneficiaries\n", + "under a Benefit Program, including self-funded health plaus.\n", + "1.19\n", + "PPG. A participating physician group that has entered into an agreement with Health Net to\n", + "deliver or arrange for the delivery of certain Covered Services to Beneficiaries.\n", + "1.20\n", + "Primary Care Physician (PCP). A Doctor of Medicine (M.D.) or Doctor of Osteopathy (D.O.)\n", + "who: (i) is duly licensed and qualified under the laws of the relevant jurisdiction to render certain Covered Services;\n", + "(ii) is a Participating Provider; and (iii) meets the credentialing standards of Health Net for designation as a PCP;\n", + "and (iv) is responsible for coordinating the provisions of health care services and providing for continuity of care\n", + "and twenty-four (24) hours a day, seven (7) days a week availability to Beneficiaries.\n", + "1.21\n", + "Prior Authorization. The written or electronically issued prior approval by Health Net or its\n", + "designee for the rendition of Covered Services that may be required under a Benefit Program or a Health Net Policy.\n", + "1.22\n", + "Professional Provider. The physicians, allied health professionals and other health care providers\n", + "who contract with Provider, or are employed by Provider, and who have been accepted by Health Net to provide\n", + "Contracted Services to Beneficiaries under the terms and conditions of this Agreement, and billed through\n", + "Provider's federal tax identification number and/or national provider identifier. Professional Providers covered by\n", + "this Agreement as of the date this Agreement is executed by the parties are listed on an exhibit to this Agreement.\n", + "1.23\n", + "Records.\n", + "Books, documents, contracts, subcontracts, and records prepared and/or\n", + "maintained by a party that relate to this Agreement whether in written or electronic format, including without\n", + "limitation medical records, Beneficiary billing and payment records, financial records, policies and procedures, and\n", + "other books and records that may be required by applicable federal and State law\n", + "1.24\n", + "Referral. The written or electronically issued referral of a Beneficiary by a Participating Provider\n", + "to another health care provider that may be required under a Benefit Program or a Health Net Policy prior to the\n", + "rendition of Covered Services, usually for a specified number of visits or type or duration of treatment.\n", + "1.25\n", + "State. The State of California.\n", + "California Provider Participation Agreement\n", + "3\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 5\n", + "1.26\n", + "Surcharge. An additional fee which is charged to a Beneficiary for a Covered Service, but which\n", + "is not approved by the applicable State and federal regulatory authority, and is neither disclosed nor provided for in\n", + "a Benefit Program.\n", + "II.\n", + "DUTIES OF PROVIDER\n", + "2.1 General Obligations. Provider agrees on behalf of itself, and each of its Facilities and Professional\n", + "Providers, as applicable, that during the term of this Agreement and any renewal terms, each of them is:\n", + "2.1.1\n", + "licensed without restriction or limitation by the State to provide Contracted Services to the extent\n", + "required by the State;\n", + "2.1.2\n", + "operating and providing Contracted Services in compliance with applicable local, State, and\n", + "federal laws, rules, regulations and legal standards of care;\n", + "2.1.3\n", + "delivering Contracted Services to Beneficiaries in the same manner and with the same availability,\n", + "as services are delivered to other patients;\n", + "2.1.4\n", + "maintaining such physical plant, equipment, patient service personnel and allied health personnel\n", + "as may be necessary to provide Contracted Services:\n", + "2.1.5\n", + "available to Beneficiaries twenty-four (24) hours per day, seven (7) days per week on an\n", + "Emergency basis.\n", + "2.1.6\n", + "Provider shall notify Health Net in writing, thirty (30) days in advance, of any changes to federal\n", + "tax identification numbers and/or national provider identifier numbers.\n", + "2.2\n", + "Provision of Services. Provider agrees to render Contracted Services to Beneficiaries of Benefit\n", + "Programs under the terms and conditions of this Agreement. Notwithstanding the foregoing, Provider understands\n", + "and agrees that Health Net or a Payor does not have an obligation under this Agreement to assign or refer to\n", + "Provider any minimum amount of Beneficiaries. Health Net has not represented or guaranteed to Provider that any\n", + "Beneficiaries shall receive Covered Services from Provider or that Provider shall participate in all networks of\n", + "Participating Providers offered by or through Health Net.\n", + "Provider acknowledges that Health Net or a Payor shall not be liable for, nor will exercise control or\n", + "direction over, the manner or method by which Provider, Facilities, and/or Professional Providers reuder any\n", + "Covered Services to Beneficiaries under this Agreement.\n", + "2.3\n", + "Verification of Eligibility. Except in an Emergency, Provider shall verify the eligibility of\n", + "Beneficiaries using Health Net's telephonically or electronically available system before providing Contracted\n", + "Services, in compliance with the timeframes and procedures set forth in Health Net Policies.\n", + "2.4\n", + "Non-Discrimination. Provider shall not discriminate against any Beneficiary in the provision of\n", + "Contracted Services hereunder, whether on the basis of the Beneficiary's coverage under a Benefit Program, age,\n", + "sex. marital status, sexual orientation, race, color, religion, ancestry, national origin, disability, handicap, health\n", + "status, source of payment, utilization of medical or mental health services, equipment, pharmaceuticals or supplies,\n", + "or other unlawful basis including, without limitation. the filing by such Beneficiary of any complaint, grievance or\n", + "legal action against Provider, Health Net or Payor. Provider agrees to make reasonable accommodations for\n", + "Beneficiaries with disabilities or handicaps, including but not limited to, providing such auxiliary aides and services\n", + "to Beneficiaries as are reasonable, necessary and appropriate for the proper rendering of Contracted Services at the\n", + "Provider's expense.\n", + "California Provider Participation Agreement\n", + "4\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 6\n", + "2.5\n", + "Professional Providers and Facilities. The following provisions apply when Provider utilizes\n", + "Professional Providers or Facilities to deliver Contracted Services to Beneficiaries:\n", + "2.5.1\n", + "Provider binds its Facilities and Professional Providers, if any, covered by this Agreement, to the\n", + "terms and conditions of this Agreement, to the extent Contracted Services and/or contractual\n", + "provisions are performed by, or apply to, such Facilities and Professional Providers:\n", + "2.5.2\n", + "No new or satellite facility shall be added to this Agreement, or be allowed to deliver Covered\n", + "Services under this Agreement until Health Net has approved such Facility. Health Net reserves\n", + "the right to deny participation under this Agreement to any new or satellite facility without any\n", + "obligation to provide a right to appeal except as may be required by applicable State and federal\n", + "law.\n", + "2.5.3\n", + "In the event Provider desires to add a new Participating Provider, Provider shall notify Health Net\n", + "in writing as soon as possible but no later than sixty (60) days before such proposed addition is to\n", + "become effective with Health Net. Provider agrees that no new Professional Provider shall be\n", + "added to this Agreement, or be allowed to render Covered Services under this Agreement, unless\n", + "and until Health Net has approved the addition of such Professional Provider. Health Net reserves\n", + "the right to deny participation under this Agreement to any proposed new Participating Provider\n", + "without any obligation to provide a right to appeal except as may be required under State or\n", + "federal law.\n", + "Provider additionally shall comply with the terms of Section 2,6 hereof with respect to its Facilities and\n", + "Professional Providers. to the extent Facilities are not owned, and/or Professional Providers are not employed, by\n", + "Provider.\n", + "2.6 Subcontracting The following requirements shall survive termination of this Agreement with respect\n", + "to Contracted Services rendered during the term of the Agreement and apply when Contracted Services are provided\n", + "by a subcontractor, such as a reference laboratory:\n", + "2.6.1\n", + "Provider shall fornish Health Net with copies of its subcontracts within ten (10) days of Health\n", + "Net's written request\n", + "2.6.2\n", + "Every subcontract shall comply with all applicable local, State and federal laws, including\n", + "privacy/confidentiality and medical record accuracy laws, be consistent with the terms and\n", + "conditions of this Agreement, and shall not be used by Provider with respect to Beneficiaries,\n", + "Benefit Programs and/or Contracted Services upon the reasonable request of Health Net.\n", + "2.6.3\n", + "Provider shall not subcontract either directly or indirectly, with any provider that has been\n", + "excluded from participation in the Medicare Advantage Program under Section 1128 or 1128A.[42\n", + "U.S.C. 1320a-7] of the Social Security Act or in the State Medi-Cal program.\n", + "2.6.4\n", + "Each such subcontractor shall meet applicable Health Net credentialing requirements, if any, prior\n", + "to the subcontract becoming effective with respect to Contracted Services.\n", + "2.6.5\n", + "(i) Provider shall be solely responsible to pay the subcourtractor and (ii) Provider shall hold Health\n", + "Net, Payors and Beneficiaries harmless from and against any and all claims which may be made\n", + "by subcontractors in connection with Covered Services provided to Beneficiaries by the\n", + "subcontractor; and (iii) Provider shall require that the subcontractor hold Health Net. Payors, and\n", + "Beneficiaries harmless from and against any and all claims for payment for such services and shall\n", + "not attempt to collect any sums owed by Provider from Health Net or a Beneficiary.\n", + "2.6.6\n", + "Subcontracts shall not restrict the rights and obligations of a healthcare provider to communicate\n", + "freely with Beneficiaries regarding their medical condition and treatment alternatives including\n", + "medication treatment options, regardless of benefit coverage limitations.\n", + "California Provider Participation Agreement\n", + "5\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-03-11\n", + "\n", + "Start of Page No. = 7\n", + "2.6.7\n", + "In the event that any of Provider's subcontracts fail to comply with the requirements set forth\n", + "herein, Health Net or Payor shall not be required to recognize the existence or validity of the\n", + "subcontract with respect to Beneficiaries, Benefit Programs and/or Covered Services. Health Net\n", + "or a Payor shall further have the right, but not the obligation, to directly pay subcontractors\n", + "submitting claims for Contracted Services, and to recoup any compensation otherwise due by\n", + "Health Net or a Payor to Provider pursuant to the terms and conditions of this Agreement.\n", + "Provider shall indemnify and hold harmless Health Net or a Payor for all such payments and\n", + "related costs.\n", + "2.7\n", + "Participating Providers. Except in an Emergency, as otherwise permitted in the applicable\n", + "Benefit Program Requirements, or as otherwise required by applicable federal or State law, if Provider refers a\n", + "Beneficiary for Covered Services, Provider shall refer Beneficiary only to Participating Providers and Provider shall\n", + "coordinate such referrals with Health Net or its designee to facilitate the utilization of the most appropriate\n", + "Participating Provider based upon the Medical Necessary level of care required for the Beneficiary.\n", + "2.8\n", + "Health Net Policies. Provider shall participate in and comply with all Health Net Policies in\n", + "effect on the effective date of this Agreement, and as modified periodically by Health Net in accordance with\n", + "Section 3.2 of this Agreement. Provider hereby acknowledges that it has had the opportunity to review Health Net\n", + "Policies regarding quality improvement and utilization management that pertain in Health Net and Provider's rights\n", + "and obligations under this Agreement at least fifteen (15) business days prior to the date Provider has executed this\n", + "Agreement.\n", + "2.9\n", + "Prior Authorization and Referrals. When either Prior Authorization and/or a Referral is\n", + "required for the rendition of a health care service, the receipt of the required Prior Authorization and/or the required\n", + "Referral, each being separate and distinct requirements, is a prerequisite to payment of Complete Claims for\n", + "Covered Services in addition to confirming eligibility prior to delivering service as required by this Agreement and\n", + "Health Net Policies. Health Net (or its desiguee as applicable) may rescind or modify its Prior Authorization, in a\n", + "manner consistent with Health Net Policies, based on variety of factors, including but not limited to the eligibility of\n", + "the Beneficiary and whether the rendered service is a Covered Service. However, when Health Net or its designee\n", + "issues a Prior Authorization for a specific service under a Benefit Program regulated by the California Department\n", + "of Managed Health Care or the California Department of Insurance, Health Net (or its designee as applicable) shall\n", + "not rescind or modify its Prior Authorization after Provider has rendered the specified and authorized service in\n", + "good faith and pursuant to the terms of the Prior Authorization for any reason, including, but not limited to, Health\n", + "Net's subsequent rescission, cancellation. or modification of the Beneficiary's contract or Health Net's subsequent\n", + "determination that it did not make an accurate determination of the Beneficiary's eligibility; provided, however that\n", + "this section shall not be construed to expand or alter benefits available to a Beneficiary under such Benefit Program.\n", + "2.10\n", + "Notification. Provider shall notify Health Net or 8 Payor and the appropriate PCP or PPG as\n", + "applicable, as soon as possible, but 110 later than 24 hours or by the next business day after a Beneficiary is admitted\n", + "to a Facility.\n", + "2.11\n", + "Credentialing Program. Provider shall submit to Health Net or its designee any applicable\n", + "Credentials Application, which meets minimum requirements of Health Net. Provider or any Professional Provider\n", + "or subcontractor shall not begin performing Provider's obligations under this Agreement, until Provider and/or\n", + "Professional Provider and/or Facility has satisfied applicable credentialing or re-credentialing requirements, if any,\n", + "2.12\n", + "Insurance. Provider shall maintain insurance in amounts and types as required by Health Net\n", + "Policies. Provider agrees to provide Health Net with a Certificate of Insurance from Provider's insurance carrier or\n", + "other mutually agreeable written evidence of such insurance coverage within three (3) days of such request by\n", + "Health Net. Provider also agrees to notify Health Net in writing at least thirty (30) days prior to any termination.\n", + "cancellation or material modification of any policy for all or any portion of the coverage required herein.\n", + "2.13\n", + "Trade names, Trademarks. Directories. Provider shall not use or display the trade names,\n", + "trademarks. or other identifying information of Health Net without Health Net's prior written approval of both form\n", + "and content, which approval shall not be unreasonably withheld. However, this provision shall not prohibit Provider\n", + "from posting a reasonable notice on its website or in its Facilities listing by name those insurance carriers that are\n", + "accepted by Provider so long as the notice lists each name in substantially similar format. Provider shall supply all\n", + "California Provider Participation Agreement\n", + "6\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 8\n", + "printed materials and other information requested by Health Net in connection with the production of provider\n", + "directories within seven (7) days of Health Net's request. Provider agrees that Health Net may list the name, address,\n", + "telephone number and other factual information of Provider, each Facility and Professional Provider. and of\n", + "Provider's subcontractors and their facilities in its provider directories, marketing and informational materials, and\n", + "electronic media.\n", + "2.14\n", + "Non-Solicitation. Neither Provider nor any employee, agent or subcontractor of Provider shall\n", + "solicit or attempt to convince or otherwise persuade any Beneficiary to discontinue participation in any Benefit\n", + "Program or in any other manner interfere with Health Net's contract and/or property rights; provided, however, that\n", + "this provision does not prohibit Provider from posting a reasonable notice on its website or in its Facilities listing by\n", + "name those insurance carriers that are accepted by Provider so long as the notice lists each name in substantially\n", + "similar format. Notwithstanding the foregoing, Health Net in no way restricts Provider from discussing medical\n", + "treatment options with Beneficiaries regardless of Benefit Program coverage options.\n", + "2.15\n", + "Language Assistance Program. Effective January 1, 2009, Provider shall comply with Health\n", + "Net's ongoing language assistance program to ensure Limited English Proficient (\"LEP\") Beneficiaries have\n", + "appropriate access to language assistance while accessing Provider services, pursuant to Health and Safety Code §§\n", + "1367(e)(3), 1367.04 and 1367.07 and Insurance Code §§ 10133.8 and 10133.9.\n", + "2.16\n", + "Additional Rights and Obligations. Any additional rights or obligations of Provider or Health\n", + "Net shall be set forth in the Addenda to this Agreement.\n", + "III.\n", + "DUTIES OF HEALTHNET\n", + "3.1\n", + "Payment. Health Net shall, or Health Net shall require Payor to, make payment to Provider for\n", + "Contracted Services in accordance with Article IV and the applicable addenda, schedules and exhibits of this\n", + "Agreement.\n", + "3.2\n", + "Health Net Policies. Health Net Policies are set forth in references and forms available to\n", + "Provider through the Provider Section, California Region, of Health Net's website at \"www.healthnet.com\" or by\n", + "other means which Health Net will communicate to Provider periodically. Health Net Policies in existence as of the\n", + "effective date of this Agreement are hereby incorporated into this Agreement by reference. Notwithstanding the\n", + "foregoing and/or any other provision of this Agreement, the parties agree that a formal amendment to this\n", + "Agreement shall not be required to effectuate modifications to Health Net Polices. Modifications to Health Net\n", + "Policies may be made periodically as determined by Health Net in accordance with the procedures set forth in\n", + "applicable State law (including without limitation the California Health Care Providers' Bill of Rights). Such\n", + "modifications shall be deemed incorporated in this Agreement as of the effective date of such modification unless\n", + "otherwise mutually agreed by the parties in writing at the time of the modification in accordance with applicable\n", + "State law (including without limitation the California Health Care Providers' Bill of Rights).\n", + "3.3\n", + "Insurance. Health Net shall maintain appropriate insurance programs or policies including bodily\n", + "injury and personal injury coverage, which includes persons serving on Health Net committees as insured by\n", + "definition. In the event that a policy or program is terminated or the coverage of committee persons is materially\n", + "changed, Health Net shall so notify Provider.\n", + "3.4\n", + "Reporting to Regulators. Health Net and/or Payor shall accept sole responsibility for filing\n", + "reports, obtaining approvais and complying with applicable laws and regulations of State, federal and other\n", + "regulatory agencies having jurisdiction over Health Net and/or Payor; provided, however, that Provider agrees to\n", + "cooperate in providing Health Net and/or Payor with any information and assistance reasonably required in\n", + "connection therewith, including without limitation, permitting the regulatory agencies to conduct periodic site\n", + "evaluations of Provider, Facilities, Professional Providers and any of their equipment, operations, and billing and\n", + "medical records of Beneficiaries. Such records shall be located in the State.\n", + "California Provider Participation Agreement\n", + "7\n", + "Fee-For-Service Direct Network Templare\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 9\n", + "3.5\n", + "Access To This Agreement.\n", + "3.5.1\n", + "Access by Health Net. As of the effective date of this Agreement, the following Health Net\n", + "companies may at their option access this Agreement: Health Net of California, Inc., Health Net\n", + "Life Insurance Company, Health Net Community Solutions, Inc., Health Net of Arizona, Inc.,\n", + "Health Net Health Plan of Oregon, Inc., Health Net of Pennsylvania, LLC, Health Net of the\n", + "Northeast, Inc., Health Net of Connecticut, Inc., Health Net of New York, Inc., Health Net of New\n", + "Jersey, Inc., Health Net One Payment Services, Inc., Health Net Insurance of New York, Inc.,\n", + "Health Net Insurance Services, Inc., Health Net Federal Services, LLC, and Managed Health\n", + "Network. Inc. Health Net may periodically modify the Health Net companies which may access\n", + "this Agreement. To the extent Health Net allows a Health Net company to access this Agreement,\n", + "Health Net binds such company to the terms and conditions of this Agreement.\n", + "3.5.2\n", + "Access by Payors. To the extent Health Net allows a self-funded Payor to access this Agreement,\n", + "Health Net has obligated such self-funded Payor to fund a claims payment account in a sufficient\n", + "and timely manner to pay claims for services provided by health care providers like Provider. In\n", + "the event a self-funded Payor accessing this Agreement fails to sufficiently and timely fund a\n", + "claims payment account to the material detriment of Provider. Provider may terminate this\n", + "Agreement as to such self-funded Payor in accordance with Section 5.3 hereof. and,\n", + "norwithstanding the provisions of Section 4.6 of this Agreement, take legal action against the self-\n", + "funded Payor and/or Beneficiary as may be permitted by law. Additional information regarding\n", + "Payors and conditions for accessing this Agreement is set forth in Addendum A of this\n", + "Agreement.\n", + "3.6\n", + "Notification to Beneficiaries: Termination of a Professional Provider. Health Net shall notify\n", + "Beneficiaries who are affected by the termination of a specialist. Professional Provider in writing, immediately upon\n", + "notification of such termination but no later than thirty (30) calendar days prior to the effective date of such\n", + "specialist's termination. Applicable to Commercial HMO Benefit Programs only: For Beneficiaries covered by an\n", + "HMO Benefit Program, Health Net shall be required to issue a notice regarding the termination of a specialist's\n", + "contract that contains the following language in not less than eight-point type: \"If you have been receiving care from\n", + "a health care Provider, you may have a right to keep your Provider for a designated time period. Please contact\n", + "Health Net's customer service department, and if you have further questions, you are encouraged to\n", + "contact the Department of Managed Health Care, which protects HMO customers. by telephone at its\n", + "toll-free number, 1-888-HMO-2219, or at a TDD number for the hearing impaired at 1-877-688-9891. or online at\n", + "www.hmohelp.ca.gov.\"\n", + "IV.\n", + "FINANCIAL OBLIGATIONS. The terms of this Article IV shall survive termination of this Agreement\n", + "with respect to Covered Services rendered during the term of this Agreement:\n", + "4.1\n", + "Payment Rates. Health Net shall pay (or shall require Payor to pay). and Provider shall accept as\n", + "payment in full for Contracted Services, the rates payable by Health Net or Payor under the terms and conditions of\n", + "this Agreement (including the payment conditions, chargemaster and other provisions set forth in the applicable\n", + "addenda, schedules and exhibits to this Agreement), less Copayments, Coinsurance and Deductibles payable by\n", + "Beneficiaries in accordance with the applicable Benefit Program or as otherwise permitted by the section of this\n", + "Agreement covering Third Party Lien Recoveries. Any overpayment, inaccurate payment or other payment error\n", + "made by Health Net or Payor shall not be deemed or construed or otherwise operate to change the payment terms or\n", + "rates provided for under this Agreement.\n", + "4.2\n", + "Billing and Payment.\n", + "4.21\n", + "Billing. Provider shall submit to Health Net/Payor, via Health Net's/Payor's electronic claims\n", + "submission program or hardcopy as determined by Health Net/Payor, Complete Claims within one\n", + "hundred twenty (120) days after Provider renders Contracted Services unless Provider\n", + "demonstrates good cause pursuant to applicable State law. Where Health Net and/or Payor is the\n", + "California Provider Participation Agreement\n", + "8\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 10\n", + "secondary payor under Coordination of Benefits, Provider shall submit Complete Claims for\n", + "Covered Services accompanied by the explanation of benefits (EOB) or explanation of payment\n", + "(EOP) from the primary payor to Health Net or a Payor within one hundred twenty (120) days of\n", + "the date of the EOB/EOP. If Provider fails to comply with the timely claims submission/filing\n", + "requirements set forth herein, Health Net shall have no obligation to pay for such claims, and\n", + "Provider shall be prohibited from billing the Beneficiary as set forth in Section 4.6 hereof.\n", + "Provider agrees that Health Net or a Payor shall have the right to determine the accuracy of all\n", + "Complete Claims submitted to it prior to payment. including verification of diagnostic codes,\n", + "DRG assignment, and whether Provider has delivered the Covered Service in good faith and\n", + "pursuant to the terms of an applicable Prior Authorization.\n", + "4.2.2\n", + "Payment. Health Net or Payor, as applicable, shall make payment on each of Provider's timely-\n", + "submitted Complete Claims in accordance with this Agreement and pursuant to the timeframes,\n", + "procedures and other requirements of applicable State and federal law, including without\n", + "limitation the calculation and payment of interest on overdue payments. Payment of interest plus\n", + "the amount of any Complete Claim payment deficiency shall be Provider's sole measure of\n", + "damages (i.e., claims for consequential or incidental damages do not apply) for failure of Health\n", + "Net or Payor to make timely and accurate payments. In no event shall Health Net be under any\n", + "obligation to pay Provider for any claim or expense, which is the responsibility of a self-funded\n", + "Payor.\n", + "4.2.3\n", + "Appeals. In addition to the dispute resolution and arbitration rights described in Section 7.5 and\n", + "Section 7.6 herein, Provider may dispute any Health Net action that adjusts, denies. or contests a\n", + "claim. billing practice, or other contractual provision so long as Provider submits a written dispute\n", + "to the Health Net Provider Appeals Unit. Unless Provider demonstrates good cause pursuant to\n", + "applicable State or federal law, Health Net or Payor shall not grant Provider reconsideration or\n", + "appeal of a claims payment for Covered Services that exceed three hundred sixty five (365) days\n", + "of Health Net's action or in the case of inaction, within three hundred sixty five (365) days after\n", + "the time for contesting or denying claims (as defined in applicable State or federal law) has\n", + "expired. Appeals shall be submitted by Provider in accordance with the procedures, and to the\n", + "address for Health Net's Provider Appeals Unit, listed in Health Net Policies. If Provider fails to\n", + "comply with the timely appeals submission/filing requirements set forth herein, Health Net shall\n", + "have no obligation to pay for such claims except as otherwise required by applicable State and\n", + "federal law, and Provider shall be prohibited from billing the Beneficiary as set forth in Section\n", + "4.6 hereof. Provider and Health Net agree to comply with all timeliness and procedural\n", + "requirements for submitting and responding to disputes submitted to Health Net's Provider\n", + "Appeals Unit as set forth in Health Net Policies.\n", + "4.3\n", + "Recompment of Overpayments; Right of Offset.\n", + "4.3.1\n", + "Provider shall inform Health Net of any overpayment made to Provider and shall return any such\n", + "overpayment to Health Net within thirty (30) business days from the date Provider first becomes\n", + "aware of any such overpayment.\n", + "4.3.2\n", + "In the event Health Net determines that it has overpaid a claim, either in connection with an audit\n", + "or otherwise, Health Net shall notify Provider in writing through a separate overpayment notice\n", + "clearly identifying the claim, the name of the Beneficiary, the date of service and explanation of\n", + "the basis upon which Health Net or Payor believes the amount paid on the claim was in excess of\n", + "the amount due, including any interest and penalties that may be due on the claim. Such\n", + "overpayment notice shall be issued within (i) three hundred sixty-five (365) days of the date of\n", + "payment on the overpaid amount for claims arising from Benefit Programs regulated by the\n", + "California Department of Managed Health Care or the California Department of Insurance, or\n", + "within (ii) three (3) years from the date of payment on the overpaid amount for claims arising from\n", + "other types of Benefit Programs that are not regulated by the California Department of Managed\n", + "Health Care or the California Department of Insurance, or (iii) at any time. in the event of fraud\n", + "California Provider Participation Agreement\n", + "9\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 11\n", + "and/or misrepresentation. Such notice shall be sent to Provider's address of record with Health\n", + "Net for the receipt of claim related correspondence and payments unless Provider informs Health\n", + "Net in writing of an alternative address to which such notices are to be sent at least thirty (30) days\n", + "in advance of the address change.\n", + "4.3.3\n", + "If Provider does not contest Health Net's overpayment notice, Provider shall reimburse Health Net\n", + "or Payor within thirty (30) business days from the date Provider receives the overpayment notice.\n", + "If Provider fails to reimburse Health Net or Payor within those thirty (30) business days, then,\n", + "beginning on the first calendar day after the expiration of this thirty (30) business day time period,\n", + "Health Net shall commence offsetting, as set forth herein and interest shall accrue on any and all\n", + "unpaid amounts at the rate of ten percent (10%) per annum.\n", + "4.3.4\n", + "In the event, Provider wishes to contest the overpayment notice, it must do so within thirty (30)\n", + "business days from the date Provider receives the overpayment notice, by sending to Health Net's\n", + "Provider Appeals Unit (at the address listed in Health Net Policies) a written appeal clearly stating\n", + "the basis upon which Provider believes that the claim was not overpaid. Health Net shall review\n", + "and make a decision with respect to Provider's appeal, and shall notify Provider of its decision in\n", + "writing within forty-five (45) business days from the date Health Net receives Provider's written\n", + "appeal. In the event Health Net denies Provider's appeal and upholds Health Net's determination\n", + "that an overpayment has been made, Provider shall reimburse Health Net or Payor for the\n", + "overpayment within thirty (30) business days from the date it receives the written notice of Health\n", + "Net's deuial of Provider's written appeal. If Provider fails to reimburse Health Net or Payor within\n", + "those thirty (30) business days, then beginning on the first calendar day after the expiration of this\n", + "thirty (30) business day time period, Health Net may commence offsetting as set forth herein, and\n", + "interest shall accrue on any and all unpaid amounts at the rate of ten percent (10%) per annum\n", + "4.3.5\n", + "If Health Net or Payor exercises offset rights hereunder against Provider's current claims\n", + "payments, Health Net or Payor shall give Provider a detailed written explanation identifying the\n", + "specific overpayments that have been offset against the specific current claims payments.\n", + "4.3.6\n", + "If Provider desires to continue to contest the overpayment, it shall do so by following the dispute\n", + "resolution process set forth in Sections 7.5 and 7.6 of this Agreement.\n", + "4.4\n", + "Eligibility.\n", + "The parties acknowledge that verification of eligibility by Health Net is based on\n", + "information available to Health Net from its customers on the date Provider seeks verification. Health Net shall use\n", + "reasonable efforts to discourage its customers from retroactively canceling or adding Beneficiaries to a Benefit\n", + "Program and encourage its customers to timely and accurately provide eligibility information.\n", + "Tu the event Contracted Services are provided to an individual who is not a Beneficiary, based on an\n", + "erroneous or delayed enrollment/eligibility list the following shall apply: (i) when the individual is enrolled in a\n", + "substitute or replacement health care service or insurance plan which is obligated under applicable law to make\n", + "payment to Provider for services delivered to the individual, Provider shall seek payment from the substimute or\n", + "replacement carrier; and (ii) when the individual does not have substitute or replacement coverage, Health Net shall\n", + "pay Provider for Contracted Services delivered to the individual by Provider prior to the time Provider received\n", + "notice of that individual's ineligibility pursuant to the terms and conditions of this Agreement, provided, however,\n", + "for those Benefit Programs that are not regulated by the California Department of Managed Health Care or the\n", + "California Department of Insurance, as an additional prerequisite for payment pursuant to this Section 4.4(ii),\n", + "Provider shall submit to Health Net evidence that Provider has unsuccessfully sought payment through two billing\n", + "cycles for all or a portion of such charges from the patient or the person having legal responsibility for the patient, or\n", + "from the entity having financial responsibility for such payment. In the event Health Net pays Provider pursuant to\n", + "this Section 4.4, Provider shall have no further right and shall not attempt to collect any additional payment from the\n", + "individual for said services (except for applicable Copayments, Coinsurance and Deductibles) and Provider hereby\n", + "assigns and transfers all legal rights of collection and Coordination of Benefits for services to Health Net.\n", + "4.5\n", + "Collection of Copayments, Coinsurance and Deductibles. Provider shall collect all\n", + "Copayments, Coinsurance and Deductibles due from Beneficiaries, and shall not waive or fail to pursue such\n", + "California Provider Participation Agreement\n", + "10\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 12\n", + "collection except when otherwise permitted through Provider's established patient financial assistance program.\n", + "Provider shall not charge Beneficiary any fees or Surcharges for Contracted Services rendered pursuant to this\n", + "Agreement (except for Copayments, Coinsurance and Deductibles). In addition, Provider shall not collect a sales,\n", + "use or other applicable tax from Beneficiaries for the sale or delivery of Contracted Services unless required by\n", + "applicable State or federal law. If Health Net or any Payor receives notice of any attempt to collect or the receipt of\n", + "any inappropriate additional charges. including without limitation Surcharges, Health Net or Payor shall take\n", + "appropriate action. Provider shall cooperate with Health Net or such Payor to investigate such allegations, and shall\n", + "promptly refund to the party who made the payment, any payment reasonably determined to be improper by Health\n", + "Net or a Payor.\n", + "4.6\n", + "Beneficiary Held Harmless. Provider agrees that in no event, including, but not limited to. non-\n", + "payment by Health Net or a Payor, insolvency of Health Net or a Payor, or breach of this Agreement, shall Provider\n", + "bill, charge, collect a deposit from, seek compensation, remuneration. or reimbursement from. or have any recourse\n", + "against Beneficiaries or persons acting on their behalf other than Health Net or a. Payor for Contracted Services\n", + "provided pursuant to this Agreement except for Copayments, Coinsurance, Deductibles, Excluded Services or\n", + "permitted third party liens under this Agreement and as permitted under Section 3.5.2 hereof. This provision shall\n", + "not prohibit collection of Copayments, Coinsurance or Deductibles or Excluded Services or permitted third party\n", + "liens under this Agreement made in accordance with applicable Benefit Program Requirements. Provider agrees\n", + "that: (i) this provision shall survive the termination of this Agreement regardless of the cause giving rise to\n", + "termination and shall be construed to be for the benefit of Beneficiaries; and (ii) this provision supersedes any oral\n", + "or written contrary agreement now existing or hereafter entered into between Provider and Beneficiaries or persons\n", + "acting on their behalf. Provider agrees to (iii) address any and all concerns it has with claims payment through\n", + "Health Net's provider appeal process pursuant to Health Net Policies and (iv) give the Beneficiary and Health Net\n", + "confirmation that Provider has rescinded the collection notice and taken any other actions necessary to clear the\n", + "Beneficiary's credit record of the collection matter.\n", + "4.7\n", + "Conditions for Compensation for Excluded Services. Provider may bill a Beneficiary for\n", + "Excluded Services rendered by Provider to such Beneficiary only if the Beneficiary is notified in advance that the\n", + "services to be provided are not Covered Services under the Beneficiary's Benefit Program, and the Beneficiary\n", + "requests in writing that Provider reuder the Excluded Services, prior to Provider's rendition of such services.\n", + "4.8\n", + "Coordination of Benefits. Provider agrees to conduct Coordination of Benefits in accordance\n", + "with federal and State laws and regulations and Health Net Policies (\"Coordination of Benefit Rules\"). including but\n", + "not limited to, the prompt notification to Health Net or a Payor of any third party entity who may be responsible for\n", + "payment and collection of Copayments. Provider shall not bill Beneficiaries for any portion of Contracted Services\n", + "not paid by the primary carrier when Health Net or a. Payor is the secondary carrier, but shall seek payment from\n", + "Health Net/Payor. When Health Net or a Payor is secondary under the Coordination of Benefits Rules, Health Net or\n", + "a Payor shall pay Provider an amount up to Beneficiary's primary plan's copayment, coinsurance or deductibles as\n", + "applicable, where that payment does not exceed Health Net's contracted rate under this Agreement. In the event that\n", + "Medicare is the primary carrier and Health Net is secondary, Health Net shall pay Provider only up to Medicare's\n", + "allowable amount and/or the Beneficiary's Copayment, Coinsurance or Deductibles as applicable. Such recoveries\n", + "shall be performed in accordance with the applicable Benefit Program Requirements and Health Net Policies.\n", + "4.9\n", + "Third Party Recoveries: Workers Compensation. In the event Provider provides Covered\n", + "Services to Health Net Beneficiaries for injuries resulting from the acts of third parties, or resulting from work\n", + "related injuries, Provider shall have the right to recover from any settlement, award, or recovery from any\n", + "responsible third-party the reasonable and necessary charges for such Covered Services to the extent permitted by\n", + "applicable law. Provider shall notify Health Net of any such recovery and shall provide Health Net with an\n", + "accounting of all such sums recovered. In the event Provider has recovered sums from a third party, Provider agrees\n", + "to pay such recovered sums to Health Net up to the fee-for-service amounts that Health Net paid to Provider, to the\n", + "extent that Health Net has not recovered such amounts from its own third party recovery efforts. Provider shall pay\n", + "these amounts to Health Net within sixty (60) days of Health Net informing Provider of the amounts Health Net\n", + "recovered from its own third party recovery efforts, if any This section does not obligate, nor does it prohibit, either\n", + "Health Net or Provider to undertake such third party recovery efforts.\n", + "California Provider Participation Agreement\n", + "11\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 13\n", + "4.10\n", + "Reciprocity.\n", + "Provider agrees that Health Net may allow the payment rates set forth in this\n", + "Agreement to be used by Participating Providers and PPGs who may periodically be responsible for compensating\n", + "Provider for Covered Services reudered by Provider to a Beneficiary.\n", + "V.\n", + "TERM AND TERMINATION\n", + "5.1\n", + "Term. The term of this Agreement shall commence on the Effective Date and shall continue for a\n", + "period of one (T) year thereafter (the \"Ininal Term\"). Either party may terminate this Agreement effective as of the\n", + "end of the Initial Term by providing at least one hundred twenty (120) days prior written notice to the other party.\n", + "This Agreement shall automatically renew for successive one (1) year periods (the \"Renewal Terins\").\n", + "5.2\n", + "Immediate Termination. Either party may terminate this Agreement immediately upon notice to\n", + "the other party, in the event of: (i) a party's violation of material law, rule or regulation; (ii) a party's failure to\n", + "maintain the insurance coverage specified hereunder; or (iii) a felony conviction related to the medical and/or\n", + "financial practices of a party. Health Net may terminate this Agreement immediately upon notice to Provider in the\n", + "event of (iv) action taken by a State or federal regulator that results in a material restriction upon Provider's ability\n", + "to perform Covered Services, including if applicable, operate a Facility or reportable discipline against Provider's\n", + "license, accreditation, or certification: (v) Health Net's determination that the health, safety or welfare of any\n", + "Beneficiary may be in jeopardy if this Agreement is not terminated; (vi) any material adverse finding as a result of a\n", + "lawsuit or claim, related to the medical and/or financial practices of Provider.\n", + "5.3\n", + "Termination Due to Material Breach. In the event either party believes the other party has\n", + "committed a material breach of this Agreement, the non-breaching party shall send the other party a written Notice\n", + "Of Breach and Demand to Cure (\"Notice\"). Without limiting either party's other termination rights under this\n", + "Article V. in the event that either party fails to cure a material breach of this Agreement within thirty (30) days of\n", + "receipt of the Notice from the other party (the \"Cure Period\"). the non-defaulting party may terminate this\n", + "Agreement by providing the defaulting party thirty (30) days prior written notice of termination. The non-defaulting\n", + "party may exercise this termination option, if at all; within thirty (30) days of the date the Cure Period expires. If the\n", + "breach is cured within the Cure Period, or if the breach is one, which cannot reasonably be corrected within the Cure\n", + "Period, and the defaulting party is making substantial and diligent progress toward correction during the Cure Period\n", + "to the reasonable satisfaction of the non-defaulting party, this Agreement shall remain in full force and effect. The\n", + "provisions of this Section 5.3 shall not apply to Health Net claims payment timeliness issues which are governed by\n", + "Article IV of this Agreement, unless and until the parties have completed the dispute resolution process set forth in\n", + "Sections 7.5 and 7.6 of this Agreement, and the dispute relates to habitual, chronic and material claims payment\n", + "timeliness issues. In the event a Payor fails in its obligations under the terms of this Agreement to niake Complete\n", + "Claim payments to Provider when due. Provider may terminate the specific delinquent Payor without terminating\n", + "this Agreement in its entirety, but only after all of the following conditions have been met: (i) Payor has failed to\n", + "make a payment to Provider within the applicable time frame set forth in this Agreement; (ii) Provider provides\n", + "written notice to Payor that such payment has not been made: (iii) Payor fails to remit payment to Provider within\n", + "ten (10) days following Payor's receipt of Provider's written notice: (iv) Provider has made a good-faith attempt to\n", + "meet with Health Net and Payor to resolve the payment issue(s).\n", + "5.4\n", + "Termination Upon Notice. Either party may terminate this Agreement during a Renewal Term\n", + "for any reason or no reason upon one hundred twenty (120) days prior written notice to the other party. In the event\n", + "that either party provides the other party with such notice, and following Health Net's completion of any applicable\n", + "regulatory filing requirements, Health Net may, at its option, begin to transition Beneficiaries under this Agreement\n", + "to another Participating Provider.\n", + "5.5\n", + "Information to Beneficiaries. The parties each agree not to disparage the other in any information\n", + "supplied by either party to Beneficiaries or other third parties in connection with any expiration, termination or non-\n", + "renewal of this Agreement. Health Net shall assume sole responsibility for notifying Beneficiaries, and Health Net\n", + "may commence transferring Beneficiaries to alternate providers, prior to the effective date of any expiration,\n", + "termination or non-renewal of this Agreement in accordance with State and federal law. If Beneficiaries seek\n", + "services or Participating Providers order tests or seek services from Provider after the effective date of any\n", + "expiration, termination or non-renewal, Provider shall inform such Beneficiaries and Participating Providers only\n", + "California Provider Participation Agreement\n", + "12\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 14\n", + "that Provider no longer has an agreement with Health Net to render Covered Services and shall direct them to Health\n", + "Net's customer service department. Provider shall not otherwise initiate communications with Beneficiaries or other\n", + "third parties, verbally or in writing, concerning the expiration, termination or non-renewal of this Agreement and\n", + "Provider's participation in Health Net's Participating Provider network, unless the parties have agreed in writing to\n", + "the content of such communications in the context of 21 mutually agreed communication plan. Nothing in this\n", + "provision is intended nor shall it be construed to prohibit or restrict Provider, Professional Provider, or other\n", + "Participating Providers from (i) disclosing to any Beneficiary information regarding treatment options available, the\n", + "risks, benefits and alternatives thereto, or (ii) disclosing to any Beneficiary the decision or process of Health Net or a\n", + "Payor to Prior Authorize or deny benefits under a Benefit Program. or (iii) posting a reasonable notice on Provider\n", + "website or in Provider's Facilities listing by name those insurance carriers that are accepted by Provider, provided\n", + "that the notice lists each name in substantially similar format. The terms of this Section 5.5 shall survive termination\n", + "of this Agreement.\n", + "5.6\n", + "Effect of Termination. In the event that a Beneficiary is receiving Contracted Services on the date\n", + "this Agreement expires, non-renews, and/or terminates, upon the request of Beneficiary and Health Net, Provider shall\n", + "continue to provide Contracted Services to the Beneficiary until the later of: (i) treatment is completed; (ii) the\n", + "Beneficiary is discharged if Provider is an inpatient facility: (iii) the Beneficiary is assigned to another Participating\n", + "Provider; or (iv) the anniversary date of the Beneficiary's Benefit Program. Provider's compensation for such\n", + "Contracted Services shall be at the rates contained in the applicable Addendum hereto. If Provider's services are\n", + "continued beyond the expiration, non-renewal. and/or termination of this Agreement, Provider shall be subject to the\n", + "same contractual terms and conditions that were imposed on Provider prior to the expiration/non-\n", + "renewal/ternination, including, but not limited to, credentialing, hospital privileging, utilization review, peer review,\n", + "and quality assurance requirements.\n", + "VI.\n", + "RECORDS, AUDITS AND REGULATORY REQUIREMENTS\n", + "6.1\n", + "Medical and Other Records. Provider shall prepare and maintain Records in accordance with the\n", + "general standards applicable to such Record-keeping and in compliance with all applicable federal and State\n", + "confidentiality and privacy laws. Provider shall maintain such Records for at least ten (10) years after the reudition\n", + "of Contracted Services, and Records of a minor child shall be kept for at least three (3) years after the minor has\n", + "reached the age of cighteen (18), but in no event less than ten (10) years after the rendition of Contracted Services.\n", + "Additionally, Provider shall maintain such Records as may be necessary and reasonably requested by Health Net to\n", + "comply with applicable federal and State law, and accrediting agency reporting requirements, rules and regulations.\n", + "Provider shall comply with and require Professional Providers to comply with all confidentiality and Beneficiary\n", + "records accuracy requirements. Provider's Records shall be and remain the property of Provider.\n", + "6.2\n", + "Access to Records and Audits by Regulatory Agencies and Accreditation Agencies. Subject\n", + "only to applicable State and federal confidentiality or privacy laws, Provider shall permit designated representatives\n", + "of local, State, and federal regulatory agencies having jurisdiction over Health Net or Payor (\"Regulatory\n", + "Agencies\") and designated representatives of accreditation agencies having jurisdiction over Health Net or Payor\n", + "(\"Accreditation Agencies\"), access to Provider's Records, at Provider's place of business in this State during normal\n", + "business hours, in order to audit, inspect and review and make copies of such Records. Such Regulatory Agencies\n", + "shall include. but not be limited to, the State Department of Health Care Services, the State Department of\n", + "Insurance, the State Department of Managed Healthcare, the United States Justice Department, CMS and the United\n", + "States Department of Health and Human Services and any of their representatives. Such Accreditation Agencies\n", + "shall include, but not be limited to, the National Committee on Quality Assurance (NCQA). When requested by\n", + "Regulatory Agencies aud/or Accreditation Agencies, Provider shall produce copies of any such Records at no\n", + "charge. Additionally, Provider agrees to permit Regulatory Agencies and Accreditation Agencies or their\n", + "representatives, to conduct audits, site evaluations and inspections of Provider's Records, offices and service\n", + "locations. Provider shall make available the access, audits, evaluations, inspections, records, and/or copies of\n", + "Records required by this Section, at no cost to Health Net, Payor, Regulatory Agencies and/or Accreditation\n", + "Agencies, and within a reasonable time period, but not more than five (5) days after the request is submitted to\n", + "Provider.\n", + "California Provider Participation Agreement\n", + "13\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 15\n", + "6.3\n", + "Access to Records and Audits by Health Net. Subject only to applicable State and federal\n", + "confidentiality or privacy laws, Provider shall permit Health Net or its designated representative access to\n", + "Provider's Records, at Provider's place of business in this State during normal business hours, in order to audit.\n", + "inspect, review and duplicate such Records. Access to Records for the purpose of an audit shall be scheduled at\n", + "mutually agreed upon times, npon at least thirty (30) business days prior written notice by Health Net or its\n", + "designated representative, but not more than sixty (60) days following such written notice. Provider shall attend an\n", + "exit interview upon completion of the audit for the purpose of obtaining a mutually agreed upon reconciliation of\n", + "the initial audit findings. Such exit interview shall be conducted al a mutually agreeable time at Provider's place of\n", + "business in this State during normal business hours upon at least ten (10) days prior written notice by Health Net or\n", + "its designated representative, but not more than thirty (30) days following such written notice. In the event\n", + "Provider fails to attend the scheduled exit interview. Provider shall be deemed to have accepted the andit findings.\n", + "Provider may be reimbursed reasonable fees associated with the retrieval of Provider's Records and or duplication\n", + "and preparation of requested Provider Records pursuant to applicable State law, including California Health and\n", + "Safety code Section 123110. Audit findings relating to any audit of claims shall include adjustment for late\n", + "charges, overcharges and undercharges. Any such adjustments shall be the net amounts as reflected in the audit\n", + "findings. Any payments owed by one party to the other as the result of an audit shall be paid within thirty (30) days\n", + "of the exit interview for such audit.\n", + "6.4\n", + "Continuing Obligation. The obligations of Provider under this Article VI shall not be terminated\n", + "upon termination of this Agreement, whether by rescission, non-renewal or otherwise. After such termination of this\n", + "Agreement, Health Net, Payors and Regulatory Agencies shall continue to have access to Provider's Records as\n", + "necessary to fulfill the requirements of this Agreement and to comply with all applicable laws, rules and regulations.\n", + "6.5\n", + "Regulatory Compliance. Each party agrees to comply with all applicable local, State, and federal\n", + "laws, rules and regulations. now or hereafter in effect, regarding the performance of the party's obligations\n", + "hereunder, including without limitation, laws or regulations governing Beneficiary confidentiality. privacy, appeal\n", + "and dispute resolution procedures to the extent that they directly or indirectly affect Provider, a Beneficiary, Health\n", + "Net, or Payor, and bear upon the subject matter of this Agreement. If Health Net is sanctioned by any Regulatory\n", + "Agency for non-compliance that is caused by Provider, Provider shall compensate Health Net for amounts tied to\n", + "this sanction incurred by Health Net including Health Net's costs of defense and fees.\n", + "VII.\n", + "GENERAL PROVISIONS\n", + "7.1\n", + "Amendments. This Agreement may be amended by mutual written agreement of the parties.\n", + "Notwithstanding the foregoing, amendments required to comply with State or federal laws or regulations,\n", + "requirements of Regulatory Agencies, or requirements of Accreditation Agencies, shall not require the consent of\n", + "Provider or Health Net and shall be effective immediately OIL the effective date of the requirement. The parties\n", + "acknowledge that changes to Health Net Policies that may affect a party's rights or obligations under this Agreement\n", + "are addressed in Section 3.2 hereof.\n", + "7.2\n", + "Separate Obligations. The rights and obligations of Health Net under this Agreement shall\n", + "apply to each Health Net company and/or Payor accessing this Agreement only to the extent such Health Net\n", + "company and/or Payor has accessed this Agreement with respect to the Benefit Programs of such Health Net\n", + "company or Payor. Health Net or Payor shall not be responsible for the obligations of any other Health Net\n", + "company or Payor under this Agreement with respect to the other's Benefit Programs. The terms of this Section 7.2\n", + "shall survive termination of this Agreement.\n", + "7.3\n", + "Assignment. Neither this Agreement, nor any of Provider's rights or obligations hereunder, is\n", + "assignable by Provider without the prior written consent of Health Net which consent shall not be unreasonably\n", + "withheld. Health Nel expressly reserves the right to assign, delegate or transfer any or all of its rights, obligations or\n", + "privileges under this Agreement to an entity controlling, controlled by, or under common control with Health Net,\n", + "Inc.,\n", + "California Provider Participation Agreement\n", + "14\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 16\n", + "7.4\n", + "Confidentiality. Health Net. Payors and Provider agree to hold Beneficiary health information\n", + "and records, the terms of this Agreement, and all confidential or proprietary information or trade secrets of each\n", + "other, in trust and confidence. Health Net. Payors and Provider each agree to keep strictly confidential all terms,\n", + "including without limitation compensation rates, set forth in this Agreement and its Addenda. except that this\n", + "provision does not preclude disclosure by Health Net to potential customers, Beneficiaries, Regulatory Agencies and\n", + "Accreditation Agencies of the method of compensation used by Health Net with respect to Health Net's\n", + "Participating Provider networks, e.g., fee-for-service, capitation, shared risk pool, DRG or per diem. Health Net,\n", + "Payors and Provider agree that such information shall be used only for the purposes contemplated herein, and not for\n", + "any other purpose. Health Net. Payors and Provider Agree that nothing in this Agreement shall be construed as a\n", + "limitation of (i) Provider's rights or obligations to discuss with the Beneficiaries matters pertaining to the\n", + "Beneficiaries health regardless of Benefit Program coverage options or (ii) Health Net's rights or obligations with\n", + "respect to subcontractors, including without limitation delegated providers, or (iii) disclosures to counsel or a\n", + "consultant of a party for the purpose of monitoring regulatory compliance or rendering legal advice pertaining only\n", + "to this Agreement or disclosures to internal or independent auditors of a party for audit purposes pertaining to this\n", + "Agreement, provided that in either case the counsel or consultant agrees in writing to comply with the provisions of\n", + "this Section 7.4 and agrees that the terms of this Agreement may not be disclosed to any other person or entity or\n", + "used in any manner whatsoever in connection with any other agreement involving Health Net. The terms of this\n", + "Section 7,4 shall survive termination of this Agreement.\n", + "7.5\n", + "Provider Dispute Resolution Procedure. The parties agree to use the dispute resolution process\n", + "set forth in this Section 7.5, and binding arbitration as described in Section 7.6, as the final steps in resolving any\n", + "Dispute. The parties each understand and agree that any and all Health Net internal appeals processes (including\n", + "without limitation as set forth in Section 4.2.3 hereof) must be properly pursued and exhausted before engaging in\n", + "the dispute resolution process set forth in this Section 7.5.\n", + "(i) Meet and Confer Process:\n", + "Initiation: If the parties are unable to resolve any Dispute through applicable Health Net internal appeal\n", + "processes, if any, the parties agree to meet and confer within thirty (30) days of a written request by either\n", + "party in a good faith effort to informally settle any Dispute. The parties each agree and understand that the\n", + "meet and confer requirements set forth herein may be satisfied only by meeting each of the following\n", + "requirements: (a) an actual meeting must occur between executive level employees of the parties who have\n", + "authority to resolve the Dispute and are each prepared to discuss in good faith the Dispute and proposed\n", + "resolution(s) to the Dispute, and (b) such meeting may take place either in person or on the telephone at a\n", + "mutually agreeable time, and (c) unless otherwise murually agreed by the parties, neither party is allowed\n", + "to have legal counsel present at the meeting or to substitute legal counsel for the executive level employee,\n", + "and (d) such meeting and all related discussions between the parties shall be treated in the same\n", + "manner as confidential protected settlement discussions under the State Rules of Civil Procedure.\n", + "Confidentiality: All documents created for the purpose of, and exchanged during, the meet and confer process\n", + "and all meet and confer discussions. negotiations and proceedings shall be treated as compromise and\n", + "settlement negotiations subject to applicable State law. To the extent the parties produce or exchange any\n", + "documents, the parties agree that such production or exchange shall not waive the protected nature of those\n", + "documents and shall not otherwise affect their inadmissibility as evidence in any subsequent proceedings.\n", + "(ii) Voluntary Mediation:\n", + "if the parties are unable to resolve any Dispute through the meet and confer process set forth above, and\n", + "desire to utilize other impartial dispute settlement techniques such as mediation or fact-finding, a joint\n", + "request for such services may be made to the American Arbitration Association (\"AAA\"), or the Judicial\n", + "Arbitration and Mediation Services (\"JAMS\") prior to submitting a Dispute to arbitration, or the parties\n", + "may initiate such other procedures as they may mutually agree upon.\n", + "7.6\n", + "Binding Arbitration. If the parties are unable to resolve a Dispute through the dispute resolution\n", + "process set forth in Section 7.5. the parties agree that such Dispute shall be settled by final and binding arbitration,\n", + "upon the motion of either party, under the appropriate rules of the AAA or JAMS, as agreed by the parties. Any\n", + "California Provider Participation Agreement\n", + "15\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 17\n", + "Arbitrator nust be either a judge, or an attorney licensed to practice law in the State of California, who is in good\n", + "standing with the State Bar, and has at least ten (10) years of experience with health care matters and the arbitration\n", + "of managed care disputes. The parties each understand and agree that the exhaustion of any Health Net internal\n", + "appeals processes and the Meet and Confer Process set forth in Section 7.5 (i) hereof are conditions precedent to\n", + "binding arbitration under this Section 7.6. Notwithstanding the foregoing, nothing contained herein is intended to\n", + "require binding arbitration of disputes alleging medical malpractice between a Beneficiary and Provider or to\n", + "Disputes between the parties alleging breaches of confidentiality of Beneficiary information, trade secret or\n", + "intellectual property obligations. The arbitration shall be conducted in San Francisco, California or Los Angeles,\n", + "California, or another location murnally agreeable to the parties that is not more than one hundred miles from\n", + "Provider's principal office. The written demand shall contain a detailed statement of the matter and facts and\n", + "include copies of all material documents supporting the demand. Except as provided below, arbitration must be\n", + "initiated within one year after the date the Dispute arose by submitting a written notice to the other party.\n", + "For the purposes of filing for arbitration regarding a Dispute over Health Net's alleged non-payment or\n", + "underpayment of Complete Claims under this Agreement, the parties agree that an arbitration shall be filed within\n", + "one (1) year after the date of Health Net's notice of its final determination on Provider's internal appeal if any, on\n", + "such Complete Claim.\n", + "The parties expressly agree that the deadlines to file arbitration set forth above shall not be subject to waiver, tolling,\n", + "alteration or modification of any kind or for any reason except for fraud. The failure to initiate arbitration before\n", + "such deadlines shall mean the complaining party shall be barred forever from initiating such proceedings.\n", + "All such arbitration proceedings shall be administered by the AAA or JAMS, as agreed by the parties; however, the\n", + "arbitrator shall be bound by applicable State and federal law, and shall issue a written opinion setting forth findings\n", + "of fact and conclusions of law. The parties agree that the decision of the arbitrator shall be final and binding as to\n", + "each of them. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction.\n", + "The arbitrator shall have no authority to make material errors of law or to award punitive damages or to add to.\n", + "modify, or refuse to enforce any agreements between the parties. The arbitrator shall make findings of fact and\n", + "conclusions of law and shall have no authority to make any award, which could not have been made by a court of\n", + "law. The party against whom the award is rendered shall pay any monetary award and/or comply with any other\n", + "order of the arbitrator within sixty (60) days of the entry of judgment on the award. The parties waive their right to\n", + "a jury or court trial.\n", + "The parties recognize and agree that theirs is an ongoing business relationship, which may lead to sensitive issues\n", + "with respect to the exchange of information related to any Dispute. The parties agree, therefore, to enter into such\n", + "protective orders (including without limitation creating a category of discovery documents \"for attorney's eyes\n", + "only\" to the extent feasible given the nature of the evidence and the Dispute). All discovery information shall be\n", + "used solely and exclusively for arbitration of the Dispute between the parties and may not be used for any other\n", + "purpose. After the arbitration award becomes final, each party shall return or destroy all documents obtained from\n", + "the other party during the course of the arbitration that are subject to a protective order, and within thirty (30) days\n", + "of such date shall provide to the other party an officer's certificate signed under penalty of perjury indicating that all\n", + "such information has been returned or destroyed.\n", + "In all cases submitted to arbitration, the parties agree to share equally the administrative fee as well as the arbitrator's\n", + "fee, if any, unless otherwise assessed by the arbitrator. The administrative fees shall be advanced by the initiating\n", + "party subject to final apportionnent by the arbitrator in this award. The parties agree that the content and decision\n", + "of any arbitration proceeding shall be confidential unless disclosure is required by applicable State or federal statutes\n", + "or regulations. The terms of Section 7.5 and Section 7.6 shall survive termination of this Agreement.\n", + "7.7\n", + "Entire Agreement. This Agreement represents the entire agreement between the parties hereto\n", + "with respect to the subject maiter hereof and supersedes any and all other agreements, either oral or written, between\n", + "the parties with respect to the subject matter hereof. and no other agreement, statement or promise relating to the\n", + "subject matter of this Agreement shall be valid or binding.\n", + "California Provider Participation Agreement\n", + "16\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 18\n", + "7.8\n", + "Governing Law. This Agreement shall be governed by and construed and enforced in accordance\n", + "with the laws of the State, except to the extent such laws conflict with or are preempted by any federal law, in which\n", + "case such federal law shall govern. Health Net is subject to the requirements of various local, State, and federal\n", + "laws, rules and regulations including, but not limited to, the requirements of Chapter 2.2 of Division 2 of the\n", + "California Health & Safety Code (the Knox-Keene Health Care Service Plan Act) and of Chapters 1 and 2, of\n", + "Division 1 of Title 28 of the California Code of Regulations (\"C.C.R.\") and Title 10 of the C.C.R. Any provision\n", + "required to be in this Agreement by any of the above shall bind Provider and Health Net whether or not expressly set\n", + "forth herein.\n", + "7.9\n", + "Indemnification.\n", + "7.9.1\n", + "Responsibility for Own Acts. Each party shall be responsible for its own acts or omissions and\n", + "for any and all claims, liabilities, injuries, suits, demands and expenses of all kinds which may result or\n", + "arise out of any alleged malfeasance or neglect caused or alleged to have been caused by that party or its\n", + "employees or representatives in the performance or omission of any act or responsibility of that party under\n", + "this Agreement.\n", + "7.9.2\n", + "Provider agrees to indemnify, defend, and hold harmless Health Net, its agents, officers, and\n", + "employees from and against any and all liability expense including defense costs and legal fees incurred in\n", + "connection with claims for damages of any nature whatsoever, including but not limited to, bodily injury,\n", + "death, personal injury, or property damage arising from Provider's performance or failure to perform its\n", + "obligations hereunder.\n", + "7.9.3 Health Net agrees to indemnify, defend, and hold harmless Provider, its agents, officers, and\n", + "employees from and against any and all liability expense, including defense costs and legal fees incurred in\n", + "connection with claims for damages of any nature whatsoever, including but not limited to, bodily injury,\n", + "death, personal injury, or property damage arising from Health Net's performance or failure to perform its\n", + "obligations hereunder.\n", + "7.10\n", + "Non-Exclusive Contract. This Agreement is non-exclusive and shall not prohibit Provider or\n", + "Health Net or Payor from entering into agreements with other health care providers or purchasers of health care\n", + "services.\n", + "7.11\n", + "No Third Party Beneficiary. Nothing in this Agreement is intended to. or shall be deemed or\n", + "construed to, create any rights or remedies in any third party, including a Beneficiary. Nothing contained herein\n", + "shall operate (or be construed to operate) in any manner whatsoever to increase the rights of any such Beneficiary or\n", + "the duties or responsibilities of Provider or Health Net or Payor with respect to such Beneficiaries.\n", + "7.12\n", + "Notice. Notices regarding the breach, term, termination or renewal of this Agreement shall be\n", + "given in writing in accordance with this Section 7.12 and shall be deemed given five (5) days following depositi in\n", + "the U.S. mail. postage prepaid. If sent by hand delivery, overnight courier, or facsimile, notices shall be\n", + "deemed\n", + "given upon documentation of delivery. All notices shall be addressed as follows:\n", + "Health Net:\n", + "Direct Network Contracting\n", + "180 Grand Avenue\n", + "Oakland, CA 94612\n", + "Facsimile: (510) 891-6958\n", + "Provider:\n", + "See Provider Primary Location Address in Exhibit I. Listing of Facilities\n", + "The addresses to which notices are to be sent may be changed by written notice given in\n", + "accordance with this Section.\n", + "7.13\n", + "Severability. If any provision of this Agreement is rendered invalid or unenforceable by any\n", + "local, State, or federal law, rule or regulation, or declared null and void by any court of competent jurisdiction. the\n", + "remainder of this Agreement shall remain in full force and effect.\n", + "California Provider Participation Agreement\n", + "17\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 19\n", + "7.14\n", + "Status as Independent Entitles. NoneCategory of Service: Covered Services delivered or arranged by Provider, excluding Laboratory services, Compensation: 100% of CMS Allowable, |\n", + "Category of Service: Anesthesia Services when provided by an Anesthesiologist or Certified Registered Nurse Anesthetist (American Society of Anesthesiology (ASA) unit scale), Compensation: $39 / ASA unit, |\n", + "Category of Service: Medical/Surgical Services by an Anesthesiologist or Certified Registered Norse Anesthetist, Compensation: 100% of CMS Allowable, |\n", + "Category of Service: Laboratory Services performed in Provider or Professional Provider office, Compensation: 100% of CMS Allowable, |\n", + "Category of Service: Pharmaceuticals With an established Medicare Value Without an established Medicare Value, Compensation: 100% of CMS Allowable 90% of the Average Wholesale Price (AWP), |\n", + "Category of Service: OB Services CPT 59400: Global Obstetric care with vaginal delivery CPT 59510: Global Obstetric care with cesarean delivery CPT 59610: Vaginal Delivery after previous cesarean delivery CPT 59618: Attempted vaginal delivery, resulting in cesareau, Compensation: $1,700.00 $1,700.00 $1,700.00 $1,700.00, |\n", + "Category of Service: Immunizations With an established Medicare Value Without an established Medicare Value, Compensation: 100% of CMS Allowable 90% of the Average Wholesale Price (AWP), |\n", + "Category of Service: General Health Panel CPT 80050: General Health Panel CPT 80055: Obstetric Panel, Compensation: $20.00 $15.00, |\n", + "Category of Service: By Report (BR) Procedures, Procedures not Listed and Procedures with Relativities not Established, and Pharmaceuticals/Immumizations without an established Medicare or AWP value, Compensation: 75% of billed charges for Covered Services, |\n", + " of the provisions of this Agreement is intended to create,\n", + "nor shall be deemed or construed to create any relationship between Provider and Health Net or a Payor other than\n", + "that of independent entities contracting with each other solely for the purpose of effecting the provisions of this\n", + "Agreement. Neither Provider nor Health Net/Payor, nor any of their respective agents, employees or representatives\n", + "shall be construed to be the agent, employee or representative of the other.\n", + "7.15\n", + "Addenda. Each Addendum to this Agreement is made a part of this Agreement as though set\n", + "forth fully herein. Any provision of an Addendum that is in conflict with any provision of this Agreement shall take\n", + "precedence and superseda the conflicting provision of this Agreement with respect to the subject matter of the\n", + "Addendum.\n", + "7.16\n", + "Calculation of Time. The parties agree that for purposes of calculating time under this\n", + "Agreement, any time period of less than ten (10) days shall be deemed to refer to business days and any time period\n", + "of ten (10) days or more shall be deemed to refer to calendar days unless the term \"business\" precedes the term\n", + "\"days\".\n", + "7.17\n", + "Waiver of Breach. The waiver of any breach of this Agreement by either party shall not\n", + "constitute a continuing waiver of any subsequent breach of either the same or any other provision(s) of this\n", + "Agreement. Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict\n", + "compliance in the future and to exercise any right or remedy related thereto.\n", + "THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED\n", + "BY THE PARTIES,\n", + "IN WITNESS WHEREOF, the parties have executed this Agreement.\n", + "PROVIDER Love\n", + "HEALTHNE\n", + "Signature\n", + "Obly\n", + "Signature Hoens\n", + "Health\n", + "Malik N. BAZ , M.D.\n", + "Cathy Hoens\n", + "Print Name\n", + "Print Name\n", + "Chairman\n", + "Title\n", + "VP Provider Network Management & Strategy\n", + "Title\n", + "Baz Allergy Asthma & Sinus\n", + "Group Name (If Applicable)\n", + "LIR.\n", + "6/29/2011\n", + "Date\n", + "77-0508441\n", + "7/15/11\n", + "Tax Identification Number\n", + "10-6-2011\n", + "Effective Date\n", + "Date\n", + "California Provider Participation Agreement\n", + "Fee-For-Service Direct Network Template\n", + "18\n", + "HN-DIN-PPA-04-08-11\n", + "\n", + "This page has 2 signature.\n", + "\n", + "Start of Page No. = 20\n", + "EXHIBIT I\n", + "LIST OF FACILITIES\n", + "The information below is mandatory. Please complete all applicable fields.\n", + "\"Provider must submit all National Provider Identifiers (NPI number/information) and a sample billing form (CMS\n", + "1500 or successor form) for each as directed by Health Net In addition, as directed by Health Net, the\n", + "corresponding Tax Identification Number shall be indicated, with a completed W-9 for each TIN as directed by\n", + "Health Net. The Tax Identification Number on the billing form must be consistent with the TIN on the W-9 form.\"\n", + "Please enter individual information if sale provider.\n", + "Baz Allergy Asthma $ Sinus Center\n", + "Provider Name (use Group name if applicable)\n", + "770208441\n", + "Provider / Group Tax Identification Number\n", + "1710906995\n", + "Primary / Group NPI Number\n", + "Primary Location\n", + "STATES\n", + "Address: 7471 STREET N. Fresno Street\n", + "can\n", + "218 COSE:\n", + "Fresno\n", + "STATES CA\n", + "93720-2457\n", + "Telephone #: 559-4364500\n", + "Fax #: 559-261-1526\n", + "Secondary Location\n", + "Address:\n", + "STREET\n", + "SEE ATTached Officelocations\n", + "STATES\n", + "CODS\n", + "STATE\n", + "ZIPILIZE\n", + "Telephone #:\n", + "Fax #:\n", + "Remit Address\n", + "Address: 7471 STREET N. Fresno St\n", + "SUFIEX\n", + "CITY\n", + "STATE\n", + "ZIP XXE\n", + "Fresno\n", + "CA\n", + "93720-2457\n", + "Telephone #: 559-436-4500\n", + "Fax #: 059-261-1526\n", + "Note: Please attach a list of additional locations on a separate sheet of paper if required.\n", + "California Provider Participation Agreement\n", + "19\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 21\n", + "Baz Allergy, Asthma & Sinus Center\n", + "**Tax ID Number 77-0208441 for all locations**\n", + "OFFICE LOCATIONS\n", + "Main Office\n", + "Billing/Mailing/Correspondence FOR ALL LOCATIONS:\n", + "NPI # 1710906995\n", + "7471 N. Fresno Street\n", + "Fresno, CA 93720\n", + "Phone: 559-436-4500\n", + "Fax: 559-261-1526\n", + "Second Office Address: NPI# 1508885781\n", + "1420 Shaw Ave. #105\n", + "Clovis, CA 93611\n", + "Phone: 559-325-9990\n", + "Fax: 559-797-0004\n", + "Third Office Address: NPI# 1023037108\n", + "1560 W. Lacey #103\n", + "Hanford, CA 93230\n", + "Phone: 559-582-8500\n", + "Fax: 559-584-9133\n", + "Fourth Office Address: NPI # 1811916992\n", + "2311 W. Cleveland #1\n", + "Madera, CA 93637\n", + "Phone: 559-674-0075\n", + "Fax: 559-673-6058\n", + "Fifth Office Address: NPI# 1215956396\n", + "220 S. Akers Suite A\n", + "Visalia, CA 93291\n", + "Phone: 559-713-1600\n", + "Fax: 559-713-1602\n", + "Sixth Office Address NPI# 1336399013\n", + "2021 E. Herndon 2nd floor\n", + "Clovis, CA 93611\n", + "Phone: 559-292-8700\n", + "Fax: 559-324-8748\n", + "\n", + "Start of Page No. = 22\n", + "EXHIBIT II\n", + "LIST OF PROFESSIONAL PROVIDERS\n", + "INSERT ROSTER HERE\n", + "Complete the information below, or you may attach as an Exhibit a complete Physician Roster including\n", + "Physician Name, License number, Specialties, Individual NPI number, CAQH number (if applicable), and\n", + "provider's Medicare Certification status (yes/no).\n", + "Provider / Group Tax Identification #: 770208441\n", + "FIRST NAME\n", + "CAST NAME\n", + "LICENSE NUTURER\n", + "PRIMARY EFECIALIY\n", + "Malik N. Bag\n", + "M.D.\n", + "A35393\n", + "SECONDAR SPECIALS\n", + "Allergy Immunology\n", + "Pediatrics\n", + "CARRE\n", + "UNDUAL NPT 7\n", + "MEDICAIS ORIGINAL\n", + "0941870\n", + "1114923307\n", + "Yes\n", + "PIRSONAME\n", + "MI\n", + "LANI NAME\n", + "LICENSE NUMBER\n", + "Praveen\n", + "Baddiga M.D.\n", + "A90273\n", + "PRIMARY SERCIAL'S\n", + "SECURDARY SPECIALTY\n", + "CADER\n", + "Allergy / EVERVIDUAL # Internal Medicine\n", + "Immunology\n", + "MEDICARE CERTIFIED (YYOT)\n", + "11854817\n", + "1598703480\n", + "Yes\n", + "FIRST NAME\n", + "321\n", + "MANT NAME\n", + "LIVINSE B\n", + "Belle\n", + "FREDRY SPECIALIT\n", + "Peralejo SUNS. KADDARY M.D. SPECIALTY\n", + "A102716\n", + "Allergy/\n", + "CASHEL'S\n", + "immunology\n", + "Internal Medicine\n", + "INDIVERIAL\n", + "MEDICARE\n", + "11871444\n", + "1104031327\n", + "Yes\n", + "2001\n", + "LAST RATE\n", + "LICENSE NUMBER\n", + "Baloh\n", + "Diaz\n", + "M.D\n", + "A066901\n", + "PRIMARY SPECIAL\n", + "SECUNDARY SPECIALIY\n", + "Pediatrics\n", + "CARDH A\n", + "INDIVIDUAL NPIC\n", + "MEDICAES CERTIFIED (Y/N)\n", + "10947949\n", + "1083682629\n", + "Yes\n", + "PINKI NAME\n", + "Mail\n", + "LAST NAME\n", + "LICENSI. NURSER\n", + "Lauren\n", + "FRIMAR MEDICALLY\n", + "S. Hiyana M.D.\n", + "A84500\n", + "SCOMURRY SPECIALTY\n", + "CAURE\n", + "Allergy INDIVIDUAL NET\n", + "Immunology\n", + "Internal Medicine\n", + "MEDICARE ERINSIED (7/N)\n", + "11799303\n", + "1215025275\n", + "Yes\n", + "FUIS\n", + "MI\n", + "LICENSE NUMBER\n", + "Hawaed [NAME\n", + "LAST NAME\n", + "D.\n", + "FRMMARY SPECIALTY\n", + "Pettigrew SECURDARY M.D. SPECIALITY PhD\n", + "4104136\n", + "ALLERGY\n", + "IMMUNOLOGY\n", + "INTERNAL MEDICINE\n", + "CARD\n", + "INDIVIDUAL\n", + "CESTUFIES (VN)\n", + "12172296\n", + "1891961835\n", + "Yes\n", + "California Provider Participation Agreement\n", + "20\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 23\n", + "ADDENDUM A\n", + "COMMERCIAL BENEFIT PROGRAMS\n", + "I.\n", + "Applicability This Addendum A and accompanying exhibits apply to Covered Services\n", + "delivered to Beneficiaries covered by commercial Benefit Programs that include but are not limited, to HMO, PPO.\n", + "EPO, POS, AIM, and any leased networks. All Covered Services delivered to a Beneficiary covered by a\n", + "commercial Benefit Program shall be paid in accordance with this Addendum A regardless of product specific name\n", + "unless otherwise specifically agreed by the parties and set forth in a separate rate exhibit.\n", + "IL\n", + "Preferred Provider Organization (PPO), Exclusive Provider Organization (EPO), Point of Service\n", + "(POS), Leased PPO Benefit Programs. and Pavor Disclosures. Provider understands and agrees that Health Net\n", + "may sell, lease, transfer or convey a list, including Provider, to Payors.\n", + "Payors shall actively encourage subscribers to use the list of contracted providers when obtaining medical care.\n", + "Active encouragement includes offering subscribers direct financial incentives to use the list of contracted providers\n", + "when obtaining medical care (such as reduced Copayments, Coinsurance and Deductibles), or providing or causing\n", + "the provision of information to subscribers advising such subscribers of the existence of a list of contracted\n", + "providers through a variety of advertising or marketing approaches that supply the names, addresses and telephone\n", + "numbers of contracted providers to subscribers in advance of their selection of a health care provider. Nothing in\n", + "this Addendum A shall be construed to require a Payor to actively encourage such Payor's subscribers to use the list\n", + "of contracted providers, including Provider, when obtaining medical care in the event of an Emergency.\n", + "Health Net shall not permit Payors to access this Agreement and pay Provider's contracted rate for the Benefit\n", + "Programs covered by this Addendum unless Payor, or Health Net on Payor's behalf, has actively enconraged\n", + "Payor's subscribers to use the list of contracted providers in obtaining medical care.\n", + "Provider agrees that the following commercial Benefit Program Payors are eligible to pay Provider's contracted rate\n", + "under this Addendum A as of the effective date of this Agreement:\n", + "NOT APPLICABLE\n", + "Health Net may modify the above list periodically. Provider may request in writing, and Health Net shall have thirty\n", + "(30) days from the date of such request, to provide Provider with an updated listing of Payors.\n", + "Provider understands and agrees that any Health Net company, including, but not limited to, Health Net Life\n", + "Insurance Company, are not Payors under this Addendum A but shall access this Agreement as Health Net.\n", + "III.\n", + "Payment. As compensation for rendering Contracted Services to Beneficiaries covered by\n", + "commercial HMO, PPO, EPO, POS and Leased PPO Benefit Programs under this Addendum A, Health Net shall\n", + "pay and Provider shall accept as payment in full the rates set forth in Exhibit A-1, subject to the payment conditions\n", + "set forth in Addendum E, the terms of this Agreement and applicable State and federal law. Notwithstanding any\n", + "other provision in this Agreement, the parties acknowledge that each Payor is solely responsible for paying Provider\n", + "for Covered Services rendered to those individuals for whom Payor provides health care coverage. For self-insured\n", + "Payors, Health Net shall not be obligated to pay all or any portion of any Provider claim on a Payor's behalf unless\n", + "and until Health Net has received sufficient funds from the applicable Payor to cover such claim. In the event such\n", + "Payor fails to provide funds to Health Net. Provider may seek payment from Beneficiary up to the rates specified in\n", + "this Exhibit. unless prohibited by applicable law.\n", + "California Provider Participation Agreement\n", + "21\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 24\n", + "EXHIBIT A-1\n", + "COMMERCIAL BENEFIT PROGRAMS\n", + "DIRECT NETWORK FEE-FOR-SERVICE\n", + "RATE EXHIBIT\n", + "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", + "Health Net or Payor shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary\n", + "Covered Services delivered under commercial Benefit Programs pursuant to this Addendum, the lesser of: (i) the rates\n", + "listed below, or (ii) 75% of Provider's billed charges.\n", + "California Provider Participation Agreement\n", + "22\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "\n", + "\n", + "\n", + "Start of Page No. = 25\n", + "ADDENDUMB\n", + "MEDICARE ADVANTAGE PROGRAM\n", + "I.\n", + "Applicability. Provider and Health Net agree that this Addendum shall apply only to Contracted Services\n", + "delivered to Beneficiaries covered by the Medicare Advantage (MA) Program, and this Addendum B may be\n", + "accessed by those Health Net entities holding a valid Medicare Advantage agreement with CMS at the time\n", + "Contracted Services are delivered.\n", + "II.\n", + "Payment. Subject to the terms of this Agreement, including without limitation the Payment Conditions set\n", + "forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically\n", + "Necessary Covered Services delivered under Health Net's MA Program pursuant to this Addendum\n", + "III.\n", + "Definitions\n", + "For purposes of this Addendum, the definitions included herein shall have the meaning required by law\n", + "governing the Medicare Advantage (MA) Program.\n", + "(a)\n", + "Medicare Advantage (MA). The statutory name applicable to the federal program of prepaid\n", + "health care services for Medicare beneficiaries established by the Medicare Prescription Drug,\n", + "Improvement, and Modernization Act of 2003.\n", + "(b)\n", + "Medicare (MA) Member or Beneficiary. A Medicare Beneficiary who has elected to enroll in\n", + "and receive coverage through a Health Net MA or MAPD Benefit Program, under the terms and conditions\n", + "of the Benefit Program's MA or MAPD Evidence of Coverage, and whose enrollment has been confirmed\n", + "by the Centers for Medicare and Medicaid Services (\"CMS\"), an agency of the U.S. Department of Health\n", + "and Human Services (\"HHS\") that administers the Medicare Advantage program.\n", + "(c)\n", + "Contract Period. The term of the contract between Health Net and CMS.\n", + "(d)\n", + "Delegation. Delegation to Provider of a certain contractual obligation of Health Net under Health\n", + "Net's contract with CMS.\n", + "(e)\n", + "MA Benefit Program. A Health Net Medicare Advantage Benefit Program that provides\n", + "coverage for certain health care services.\n", + "(f)\n", + "MAPD Benefit Program. An MA Benefit Program that includes Medicare Part D prescription\n", + "drug coverage.\n", + "(g)\n", + "MA/MAPD Evidence of Coverage. A legally binding statement of coverage, revised annually,\n", + "between a Medicare Member and Health Net under which a Medicare Member is entitled to receive\n", + "coverage for certain hospital, medical and other associated health care services.\n", + "(h)\n", + "Downstream Providers. A Participating PROVIDER who or which is contracted with PROVIDER\n", + "to render services to Beneficiaries.\n", + "IV.\n", + "General Provisions.\n", + "4.1\n", + "Provider Obligations. Provider agrees to render Contracted Services to MA Members, in\n", + "accordance with the terms and conditions of Health Net's MA Programs. Health Net shall provide Provider\n", + "with the Benefit Program Requirements of such Benefit Programs not set forth in this Addendum. Such Benefit\n", + "Program Requirements include the provisions of the applicable MA or MAPD Evidence of Coverage. and\n", + "Health Net Policies. Provider acknowledges that the determination of Covered Services shall be governed by\n", + "coverage guidelines established by Health Net and the MA Program, with Health Net being solely responsible\n", + "for final coverage determinations, subject to applicable appeal procedures.\n", + "California Provider Participation Agreement\n", + "23\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 26\n", + "4.2\n", + "Member Services. Provider shall cooperate fully with Health Net in the investigation and\n", + "resolution of complaints by MA Members regarding Provider, or the services Provider provides or arranges. in\n", + "compliance with CMS and state regulatory requirements.\n", + "4.3\n", + "Reports and Administration. Health Net shall have sole responsibility for filing reports,\n", + "obtaining approval from, and complying with the applicable laws and regulations of federal, state and local\n", + "governmental agencies having jurisdiction over Health Net. Provider shall cooperate in providing Health Net\n", + "with such information and assistance regarding MA. Members and Provider's performance under this Agreement\n", + "and Addendum as Health Net may reasonably require in filing such reports and complying with applicable laws\n", + "and regulations.\n", + "4.4\n", + "Health Net Policies and Procedures. Health Net shall promptly communicate any material\n", + "change in Health Net's policies and procedures affecting Provider in accordance with Section 3.2 of this\n", + "Agreement.\n", + "4.5\n", + "Provider and Health Net agree to: (i) Process contracted Provider claims received at Health Net\n", + "within 60 calendar days of receipt, (ii) Process non-contracted provider clean claims received by Provider\n", + "within 30 calendar days and 60 calendar days for all other claims.\n", + "4.6.\n", + "Regulatory Obligations.\n", + "A.\n", + "Health Net and Provider agree and Provider will require its subcontractors to agree to:\n", + "(1)\n", + "For a period of ten (10) years from the final date of the Contract Period or the date of completion\n", + "of any audit, whichever is later, allow HHS, CMS, the Comptroller General, or their designees the right to audit,\n", + "evaluate, and inspect any books, contracts, records, including medical records and documentation of Provider and\n", + "any Provider subcontractor. involving transactions related to Health Net's contract with CMSi *See 42 CFR §§\n", + "422.504(i)(2)(i) and (ii); Ch. 11. 100.4, MMCM]\n", + "(2)\n", + "Cooperate in assist in, and provide information as requested for audits, evaluations and inspections\n", + "performed under Section A(1) above. [*See Ch. 11. $100.4, MMCM]\n", + "(3)\n", + "Safeguard MA Member's privacy and confidentiality. [*See Ch. 11. $100.4, Medicare Managed\n", + "Care Manual (\"MMCM\")]\n", + "(4)\n", + "Specify a prompt payment requirement in its written agreement with a Provider subcontractor.\n", + "[See Ch. 11, $100.4, MMCM]\n", + "(5)\n", + "Ensure the accuracy of MA Member's health and other records. [*See Ch. 11. $100.4, MMCM]\n", + "(6)\n", + "Hold MA Members harmless for payment of any fees that are the legal obligation of Health Net in\n", + "the event of but not limited to, insolvency of, breach by or billing of Provider by Health Net. [*See 42 CFR\n", + "422.504(g)(1)(i) and (h)(3)(i)]\n", + "(7)\n", + "Comply with all applicable Medicare laws, regulations, reporting requirements and CMS\n", + "instructions. [*See 42 CFR § 422.504(i)(4)(v)]\n", + "(S)\n", + "Perform each service or other activity under this Agreement in a manner consistent and in\n", + "compliance with Health Net's contractual obligations to CMS. (*See 42 CFR 422.504(i)(3)(iii)]\n", + "(9)\n", + "Maintain all records relating to this contract for a minimum of ten (10) years from the final date of\n", + "the Contract Period or the date of the completion date of any audit. whichever is later. [See Ch. 11. §100.4,\n", + "MMCM]\n", + "California Provider Participation Agreement\n", + "24\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 27\n", + "(10)\n", + "Health Net oversees and is ultimately responsible to CMS for any functions and responsibilities\n", + "described in the Medicare Advantage regulations. Provider understands that this accountability provision also\n", + "applies to this Addendum. [See CH. 11 $100.4, MMCM. See 42 CFR §§422.504 (i)(4)(iii)\n", + "(11)\n", + "Comply with Health Net Policies. [*See Ch. 11. $100.4, MMCM]\n", + "(12)\n", + "Effective January 1. 2010, not collect cost sharing from any Medicare Member that exceeds the\n", + "amount of cost sharing that would be permitted with respect to the Medicare Member under Title XIX of the Social\n", + "Security Act if the Medicare Member were not enrolled in the MA/MAPD Benefit Program. In no event shall\n", + "Provider or a Provider subcontractor hold a Medicare Member responsible for any cost sharing for Covered Services\n", + "when a State entity is responsible for paying such amount. Where the State is responsible for paying the cost share\n", + "amount. Provider shall either accept Health Net's contracted rate as payment in full or bill the appropriate State\n", + "source for the cost share amount. [See CMS 2010 Call Letter.]\n", + "B.\n", + "Provider and Health Net agree:\n", + "(1)\n", + "Provider shall bill and Health Net shall make payment for Covered Services in accordance with\n", + "the Agreement and Health Net Policies. [*See Ch. 11 $100.4, Medicare Managed Care Manual (\"MMCM\")]\n", + "C.\n", + "Provider and its subcontractors further agree to the following:\n", + "(a)\n", + "To pay for emergency and urgently needed services consistent with federal regulations, if such\n", + "services are Provider's liability.\n", + "(b)\n", + "To pay for renal dialysis services for Beneficiaries temporarily outside the service area, if such\n", + "services are Provider's responsibility.\n", + "(c)\n", + "To direct access to mammography screening and influenza vaccinations.\n", + "(d)\n", + "To direct access to in-network women's health specialist for women for routine and preventative\n", + "services.\n", + "(e)\n", + "To have approved procedures to identify, assess and establish a treatment plan for Beneficiaries with\n", + "complex or serious medical conditions.\n", + "(f)\n", + "To provide access to benefits in a manner described by CMS.\n", + "(g)\n", + "To protect Beneficiaries who are hospitalized from loss of benefits through the period of time CMS\n", + "premiums are paid.\n", + "(h)\n", + "To work with Health Net in conducting a health assessment of all new Beneficiaries within ninety\n", + "(90) days of the effective date of enrollment.\n", + "(i)\n", + "That Provider must notify any Professional Provider being terminated, in writing, of the reason(s) for\n", + "denial, suspension or termination determinations.\n", + "(j)\n", + "To not employ or contract with individuals excluded from participation in Medicare under Section\n", + "1128 or 1128A of the Social Security Act.\n", + "(k)\n", + "To adhere to Medicare's appeals, expedited appeals and expedited review procedures for Medicare\n", + "HMO Beneficiaries, including gathering and forwarding information on appeals to Health Net. as\n", + "necessary.\n", + "California Provider Participation Agreement\n", + "25\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 28\n", + "(I)\n", + "That Medicare Beneficiaries health services are being paid for with Federal funds, and as such,\n", + "payments for such services are subject to laws applicable to individuals or entities receiving Federal\n", + "funds.\n", + "(in)\n", + "To assume the financial responsibility for any failure to comply with Medicare Regulations,\n", + "including, but not limited to, failure to provide records when requested and failure to provide or\n", + "document Notice(s) of Non-Coverage to Beneficiaries.\n", + "(11)\n", + "That for purposes of Medicare Beneficiaries, retroactive eligibility changes shall be limited to\n", + "thirty six (36) months or as otherwise required by CMS\n", + "(o)\n", + "To comply with Medicare laws. regulations, reporting requirements and CMS instructions.\n", + "(p)\n", + "To not collect any co-payment or other cost sharing for influenza vaccine and pneumococcal vaccines.\n", + "(q)\n", + "To comply with and require that all Downstream Providers comply with applicable State and federal\n", + "laws and regulations, including Medicare laws and regulations and CMS instructions.\n", + "VI.\n", + "DELEGATION.\n", + "The parties acknowledge that none of Health Net's MA Program contract obligations to CMS have been\n", + "delegated by Health Net to Provider.\n", + "California Provider Participation Agreement\n", + "26\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 29\n", + "EXHIBIT B-1\n", + "MEDICARE ADVANTAGE PROGRAM\n", + "DIRECT NETWORK FEE-FOR-SERVICE\n", + "RATE EXHIBIT\n", + "I. Payment Rates\n", + "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", + "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", + "Services delivered under Medicare Advantage Benefit Programs pursuant to this Addendum. the lesser of: (i) the rates\n", + "listed below, or (ii) 75% of Provider's billed charges.\n", + "California Provider Participation Agreement\n", + "27\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "\n", + "\n", + "\n", + "Start of Page No. = 30\n", + "ADDENDUM O\n", + "MEDI-CAL BENEFIT PROGRAM\n", + "Provider understands and agrees that the obligations of Health Net. Inc. set forth in this Addendum shall be the\n", + "obligations of Health Net of California, Inc. (\"Health Net\"), an Affiliate of Health Net, Inc, and not the obligations\n", + "of Health Net, Inc. or any other Affiliate of Health Net, Inc. Health Net has entered into one or more Medi-Cal\n", + "prepaid health plan agreements with the California Department of Health Care Services (\"DHCS\"). For the\n", + "purposes of this Addendum, Health Net's Medi-Cal agreements with the DHCS and any subcontracts with Medi-Cal\n", + "prepaid health plans, are hereinafter collectively referred to as the Medi-Cal Agreement\". Health Net has agreed,\n", + "under the Medi-Cal Agreement. to provide medical services covered under California's Medi-Cal Program,\n", + "including Provider risk services, to Medi-Cal HMO Beneficiaries enrolled in or otherwise assigned to Health Net, on\n", + "a prepaid basis. The provisions of the Addendum are required to appear in all subcontracts under the Medi-Cal\n", + "Agreement by the terms of the Medi-Cal Agreement and by Medi-Cal law and may not be altered.\n", + "Provider understands and agrees that Health Net Community Solutions, Inc. is an Affiliate of Health Net and of\n", + "Health Net of California, Inc. and has entered into an agreement with the Fresno-Kings-Madera Regional Health\n", + "Authority (CalViva Health) to provide Covered Services to Medi-Cal beneficiaries in Fresno, Kings and Madera\n", + "Counties who enroll in CalViva Health. Provider understands and agrees that Health Net of California, Inc. is\n", + "providing Covered Services as a subcontractor to Health Net Community Solutions, Inc. for the CalViva Health\n", + "Medi-Cal membership. Provider further understands and agrees that it will provide Covered Services to CalViva\n", + "Health Medi-Cal members pursuant to the Agreement and that all terms and conditions of the Agreement, including\n", + "reimbursement, shall apply to the provision of Covered services to the CalViva Health Medi-Cal members.\n", + "Health Net has or may enter into contracts with certain Payors, including local initiatives such as Cal Viva Health in\n", + "Fresno, Kings and Madera Counties, to provide or arrange for Covered Services to Medi-Cal beneficiaries enrolled in\n", + "the Medi-Cal plans of such Payors. Provider understands and agrees that a Payor may have adopted policies and\n", + "procedures, including, but not limited to, quality assurance and quality improvement programs. Provider further\n", + "understands and agrees that Provider and its Participating Providers shall comply with all the policies and procedures\n", + "adopted by a Payor and shall participate in the Payor's quality assurance aud quality improvement programs.\n", + "A.\n", + "COMPENSATION PROVISIONS.\n", + "1.\n", + "Compensation. Provider shall arrange and provide Contracted Services to Medi-Cal HMO\n", + "Beneficiaries of Health Net's Benefit Programs covered under this Addendum on a fee-for-service basis. As\n", + "compensation for providing such Contracted Services, Provider shall be paid in accordance with the rates set forth\n", + "on Exhibit C-1. subject to the payment conditions set forth in Addendum E. Such compensation shall be paid within\n", + "forty-five (45) working days of receipt of a complete and accurate claim for Covered Services rendered to a Medi-\n", + "Cal HMO Beneficiary.\n", + "2.\n", + "Billing. Notwithstanding anything to the contrary to the Agreement, if Provider is compensated on\n", + "a fee-for-service basis, Provider shall submit to Health Net, via Health Net's electronic claims submission program\n", + "or hardcopy as determined by Health Net, Complete Claims within one hundred eighty (180) days after the month in\n", + "which the Covered Service is reudered unless Provider demonstrates good cause pursuant to applicable State law.\n", + "Where Health Net is the secondary payor under Coordination of Benefits, Provider shall submit Complete Claims\n", + "for Covered Services accompanied by the explanation of benefits (EOB) or explanation of payment (EOP) from the\n", + "primary payor to Health Net within one hundred eighty (180) days of the date of the EOB/EOP.\n", + "If the Provider fails to comply with the timely claims submission/filing requirements set forth above,\n", + "Health\n", + "Net shall reimburse the Provider at the following rates: 75% of usual allowance for claims submitted during\n", + "the seventh through ninth month after the month of service; 50% of usual allowance for claims submitted during the\n", + "tenth through the twelfth month after the month of service. Health Net shall not be liable for payment to Provider\n", + "for any Complete Claims received after the twelfth month after the month of service.\n", + "California Provider Participation Agreement\n", + "28\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 31\n", + "B.\n", + "GENERAL PROVISIONS\n", + "1.\n", + "Provider Certification. Provider is certified to participate in Medicaid/Medi-Cal under Title XIX\n", + "of the Social Security Act or other applicable State law pertaining to Title XIX of the Social Security Act.\n", + "2.\n", + "Provision\n", + "of\n", + "Covered Services. Provider shall arrange Covered Services for assigned\n", + "Beneficiaries. For the purposes of this Addendum, \"Covered Services\" means those health care services, supplies\n", + "and items that are specified as being covered under the Medi-Cal Agreement. Provider shall arrange Covered\n", + "Services for Beneficiaries. in accordance with the following, each of which is hereby incorporated by reference as if\n", + "set out in full herein:\n", + "a)\n", + "The terms and conditions of this Addendun and the Agreement.\n", + "b)\n", + "The terms and conditions of the Medi-Cal Agreement and the applicable Evidence of\n", + "Coverage.\n", + "e)\n", + "Health Net Medi-Cal policies and procedures and physician bulletins.\n", + "d)\n", + "DHCS Medi-Cal Managed Care Division (MMCD) Policy Letters.\n", + "e)\n", + "All laws applicable to Provider and Health Net.\n", + "1)\n", + "Health Net's Utilization Care Management Program and Quality Improvement Program.\n", + "g)\n", + "Standards requiring services to be provided in the same manner, and with the same\n", + "availability, as services are rendered to other patients.\n", + "b)\n", + "No less than the minimum clinical quality of care and performance standards that are\n", + "professionally recognized and/or adopted, accepted or established by Health Net.\n", + "3.\n", + "Preparation and Retention of Records; Access to Records: Audits. Provider shall prepare and\n", + "maintain medical and other books and records required by law in a form maintained in accordance with the general\n", + "standards applicable to such book or record keeping, Provider shall maintain such financial. administrative and\n", + "other records as may be necessary for compliance by Health Net with all applicable local, State and federal laws.\n", + "Provider shall retain such books and records and all encounter data for a term of at least five (5) years from the close\n", + "of the California State fiscal year in which the Agreement is in effect. Provider shall make Provider's books, records\n", + "and encounter data pertaining to the goods and services furnished under the terms of the Agreement, available for\n", + "inspection, examination or copying by Health Net, DHCS, the United States Department of Health and Human\n", + "Services (\"DHHS\"), the California Department of Managed Health Care (\"DMHC\"), the United States Department\n", + "of Justice (\"DOJ\"), and any other regulatory agency having jurisdiction over Health Net. The records shall be\n", + "available at Provider's place of business, or at such other innitually agreeable location in California. When such\n", + "entities request Provider's records, Provider shall produce copies of the requested records at no charge. Provider\n", + "shall permit Health Net, and its designated representatives, and designated representatives of local, State, and federal\n", + "regulatory agencies having jurisdiction over Health Net. to conduct site evaluations and inspections of Provider's\n", + "offices and service locations. [22 CCR § 53250(e)(1); W & § 14452(c); Medi-Cal Agreement ].\n", + "4.\n", + "Subcontracting Under the Agreement. Provider shall not subcontract for the performance of\n", + "services under the Agreement without the prior written consent of Health Net Every such subcontract shall provide\n", + "that it is terminable with respect to Medi-Cal HMO Beneficiaries by Provider upon Health Net's request. Provider\n", + "shall furnish Health Net with copies of such subcontracts. and amendments thereto, within ten (10) days of\n", + "execution. Each such subcontracting Provider shall meet Health Net's credentialing requirements, prior to the\n", + "subcontract becoming effective. Provider shall be solely responsible to pay any health care Provider pennitted\n", + "under the subcontract, and shall hold, and ensure that health care Providers hold, Health Net, Beneficiaries and the\n", + "State harmless from and against any and all claims which may be made by such subcontracting Providers in\n", + "connection with services rendered to Medi-Cal HMO Beneficiaries under the subcontract. Provider shall maintain\n", + "California Provider Participation Agreement\n", + "29\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 32\n", + "and make available to Health Net, DHCS, DHHS, DMHC, DOJ, and any other regulatory agency having jurisdiction\n", + "over Health Net, copies of all Provider's subcontracts under the Agreement and to ensure that all such subcontracts\n", + "are in writing and require that the subcontractor: (1) make all applicable books and records available for inspection,\n", + "examination or copying by said entities; (2) retain such books and records for a term of at least five (5) years from\n", + "the close of the fiscal year in which the subcontract is in effect: and (3) maintain such books and records in a form\n", + "maintained in accordance with the general standards applicable to such book or record keeping. [22 CCR §\n", + "53250(e)(3)].\n", + "5.\n", + "Federal Disclosure Form. Provider shall submit to Health Net a completed Disclosure Form,\n", + "attached to this Addendum for officers and other persons associated with Provider as required by California Welfare\n", + "and Institutions Code 14452(a).\n", + "6.\n", + "Medi-Cal HMO Beneficiary Education. Provider shall make health education materials and\n", + "programs available to Medi-Cal HMO Beneficiaries on the same basis that it makes such materials and programs\n", + "available to the general public, and shall use its best efforts to encourage Medi-Cal HMO Beneficiaries to participate\n", + "in such health education programs. [Medi-Cal Agreement].\n", + "7.\n", + "Medi-Cal HMO Beneficiaries and State Held Harmless. Provider agrees that in no event,\n", + "including, but not limited to, non-payment by Health Net. the insolvency of Health Net, or breach of the Agreement,\n", + "shall Provider or a subcontractor of Provider bill, charge, collect a deposit from, seek compensation, remomeration,\n", + "or reimbursement from, or have any recourse against Medi-Cal HMO Beneficiaries, the State of California, or\n", + "persons other than Health Net acting on their behalf for services provided pursuant to the Agreement. Provider\n", + "agrees: (1) this provision shall survive the termination of the Agreement regardless of the cause giving rise to\n", + "termination and shall be construed to be for the benefit of Medi-Cal HMO Beneficiaries: and (2) this provision\n", + "supersedes any oral or written contrary agreement now existing or hereafter entered into between Provider and\n", + "Medi-Cal HMO Beneficiaries or persons acting on their behalf. Any modification, addition, or deletion of or to the\n", + "provisions of this clause shall be effective on a date no earlier than fifteen (15) days after the DHCS has received\n", + "written notice of such proposed change and has approved such change. [22 CCR § 53250(e)(6)].\n", + "8.\n", + "No Surcharges and No Copayments. Provider shall not charge a Medi-Cal HMO Beneficiary\n", + "any fee, surcharge or Copayment for health care services rendered pursuant to the Agreement except when explicitly\n", + "allowed by the Medi-Cal Benefit Program, for covered services rendered pursuant to the Agreement. In addition,\n", + "Provider shall not collect a sales, use or other applicable tax from Medi-Cal HMO Beneficiaries for the sale or\n", + "delivery of medical services. If Health Net receives notice of any additional charge, Provider shall fully cooperate\n", + "with Health Net to investigate such allegations, and shall promptly refund any payment deemed improper by Health\n", + "Net to the party who made the payment. [Knox-Keene Act and Medi-Cal Agreement].\n", + "9.\n", + "Grievances and Appeals. Provider shall resolve all grievauces and appeals relating to the\n", + "provision of services to Medi-Cal HMO Beneficiaries in accordance with the Health Net Medi-Cal grievance and\n", + "appeal procedures.\n", + "10.\n", + "Provider Patient Relationship. Provider shall be solely responsible, without interference from\n", + "Health Net or its agent, for providing Hospital Services to Medi-Cal HMO Beneficiaries, and shall have the right to\n", + "object to treating any individual who makes enerous the relationship between Provider and Medi-Cal HMO\n", + "Beneficiary. In the event of a breakdown in such relationship, Health Net shall make reasonable efforts to assign the\n", + "Medi-Cal HMO Beneficiary to another Participating Provider. If reassignment is unsuccessful, a request may be\n", + "filed with the DHCS to permit termination of services to such Medi-Cal HMO Beneficiary. Approval from the\n", + "DHCS must be obtained before Provider terminates services to such Medi-Cal HMO Beneficiary.\n", + "11.\n", + "Fair Employment Requirements. During the term of this Agreement, Provider and its\n", + "subcontractors shall not unlawfully discriminate against any employee or applicant for employment because of race,\n", + "religious creed, color, national origin, ancestry, physical disability, mental disability, medical condition. marital\n", + "status, age (over 40) or sex. Provider and its subcontractors also shall ensure that the evaluation and treatment of\n", + "their employees and applicants for employment are free of such discrimination. Provider and its subcontractors shall\n", + "comply with the provisions of the Fair Employment & Housing Act (California Government Code, Section 12990 et\n", + "seg.) and the applicable regulations promulgated thereunder (California Code of Regulations, Title 2, Section 7285.0\n", + "California Provider Participation Agreement\n", + "30\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 33\n", + "et seq.). The applicable regulations of the Fair Employment & Housing Commission implementing Government\n", + "Code, Section 12990, set forth in Chapter 5 of Division 4 of Title 2 of the California Code of Regulations are\n", + "incorporated into this Agreement by reference and made a part hereof as if set forth in full Provider and its\n", + "subcontractors shall give written notice of their obligations under this clause to labor organizations with which they\n", + "have a collective bargaining or other agreements.\n", + "12.\n", + "Governing Law. The Agreement shall be governed by and construed and enforced in accordance\n", + "with all laws and contractual obligations incumbent upon Health Net. Provider shall comply with all applicable\n", + "local, State, and federal laws, now or hereafter in effect, to the extent that they directly or indirectly affect Provider\n", + "or Health Net. and bear upon the subject matter of the Agreement. Provider shall comply with the provisions of the\n", + "Medi-Cal Agreement, and Chapters 3 and 4 of Subdivision 1 of Division 3 of Title 22 of the California Code of\n", + "Regulations. In addition, Health Net is subject to the requirements of Chapter 2.2 of Division 2 of the California\n", + "Health and Safety Code and Subchapter 5.5 of Chapter 3 of Title 10 of the California Code of Regulations. Any\n", + "provision required to be in the Agreement by either of the above laws shall bind the parties whether or not provided\n", + "in the Agreement. [22 CCR § 53250(c)(2)]; § 14452(a): Knox-Keene Act].\n", + "13.\n", + "Notice. Provider shall notify DHCS in the event this Agreement is amended or terminated\n", + "Notice to DHCS is considered given when properly addressed and deposited with the United States Postal Service as\n", + "first class registered mail, postage attached. [Knox-Keene Act and Medi-Cal Agreement].\n", + "14.\n", + "Reports: Provider shall provide Health Net. within the time requested by Health Net, with all\n", + "such reports and information AS Health Net may require to allow to meet the reporting requirements under the Medi-\n", + "Cal Agreement or any applicable law, [22 CCR 53250(c)(5)].\n", + "15.\n", + "Confidentiality of Information. Names of persons receiving public social services are\n", + "confidential and are to be protected from unauthorized disclosure in accordance with Title 45, Code of Federal\n", + "Regulations, Section 205.50 and Section 14100.2 of the California Welfare and Institutions Code and the regulations\n", + "adopted thereunder. For the purposes of this Agreement, all information, records, data, and data elements collected\n", + "and maintained for or in connection with performance under this Agreement and pertaining to Medi-Cal HMO\n", + "Beneficiaries shall be protected by Provider from unauthorized disclosure, With respect to any identifiable\n", + "information concerning a Medi-Cal HMO Beneficiary under this Agreement that is obtained by Providers or its\n", + "subcontractors, Provider: (1) will not use any such information for any purpose other than carrying out the express\n", + "terms of this Agreement; (2) will promptly transmit to Health Net all requests for disclosure of such information; (3)\n", + "will not disclose, except as otherwise specifically permitted by this Agreement, any such information to any party\n", + "other than Health Net without Health Net's prior written authorization specifying that the information is releasable\n", + "under applicable law, and (4) will, at the expiration or termination of this Agreement, return all such information to\n", + "Health Net or maintain such information according to written procedures provided Provider by Health Net for this\n", + "purpose. Provider shall ensure that its subcontractors comply with the provisions of this paragraph.\n", + "16.\n", + "Third Party Tort Liability. Provider shall make no claim for recovery for health care services\n", + "reudered to a Medi-Cal HMO Beneficiary when such recovery would result from an action involving the tort\n", + "liability of a third party or casualty liability insurance, including workers' compensation awards and uninsured\n", + "motorist coverage. Within five (5) days of discovery, Provider shall notify Health Net of cases in which an action\n", + "by the Medi-Cal HMO Beneficiary involving the tort or workers' compensation liability of a third party could result\n", + "in a recovery by the Medi-Cal HMO Beneficiary. Provider shall promptly provide: (1) all information requested by\n", + "Health Net in connection with the provision of health care services to a Medi-Cal HMO Beneficiary who may have\n", + "an action for recovery from any such third party; (2) copies of all requests by subpoena from attorneys, insurers or\n", + "Medi-Cal HMO Beneficiaries for copies of bills, invoices or claims for health care services; and (3) copies of all\n", + "documents released as a result of such requests. Provider shall ensure that its subcontractors comply with the\n", + "requirements of this provision.\n", + "17.\n", + "Amendments.\n", + "17.1\n", + "When required under Medi-Cal law, Amendments to the Agreement shall be submitted by Health\n", + "Net to the DHCS for prior approval at least thirty (30) days before the effective date of any proposed changes\n", + "governing compensation, services or term. Proposed changes, which are neither approved nor disapproved by the\n", + "California Provider Participation Agreement\n", + "31\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 34\n", + "Department, shall become effective by operation of law thirty (30) days after the DHCS has acknowledged receipt.\n", + "or upon the date specified in the amendment, whichever is later. Subcontracts between a prepaid health plan and a\n", + "subcontractor shall be public records on file with the DHCS. [22 CCR § 53250(a), (c)(3), & (e)(4); W §\n", + "14452(a)j.\n", + "17.2\n", + "Notwithstanding the foregoing and any provisions to the contrary in this Agreement, the parties\n", + "understand and agree that an amendment to the material terms of this Agreement shall be permitted without the\n", + "consent of Provider if: (i) Provider is a non-institutional provider; (ii) the amendment applies to the Medi-Cal or\n", + "Healthy Fannlies products; (iii) Provider is compensated on a fee-for-service basis; (IV) Health Net gives the\n", + "Provider a minimum of ninety (90) business days' notice of its intent to amend the Agreement; (v) Provider has the\n", + "right to exercise its intent to negotiate and agree to the amendment within thirty (30) business days of Provider's\n", + "receipt of the notice of amendment; and (vi) Provider has the right to terminate the Agreement within ninety (90)\n", + "business days from the date of receipt of such notice if Provider does not exercise the right to negotiate the\n", + "amendment and no agreement is reached. In such event, the amendment becomes effective ninety (90) days from\n", + "the date of the notice set forth in this paragraph if Provider does not exercise its right to negotiate the amendment or\n", + "to terminate the Agreement as described in this paragraph.\n", + "18.\n", + "Notice of Change in Availability or Location of Covered Services. Health Net is obligated to\n", + "ensure Medi-Cal HMO Beneficiaries are notified in writing of any changes in the availability or location of Covered\n", + "Services at least thirty (30) days prior to the effective date of such changes, or within fourteen (14) days prior to the\n", + "change in cases of unforeseeable circumstances. Such notifications must be approved by DHCS prior to the release.\n", + "In order for Health Net to meet this requirement, Provider is obligated to notify Health Net in writing of any changes\n", + "in the availability or location of Covered Services at least forty (40) days prior to the effective date of such changes.\n", + "19.\n", + "Transfer of Care Upon Termination of the Agreement. Provider shall, pursuant to the\n", + "requirements of the Medi-Cal Agreement, assist in the orderly transfer of care of all Medi-Cal HMO Beneficiaries\n", + "under the care of Provider in the event of the termination of the Agreement.\n", + "20.\n", + "Assignment and Delegation. Assignment or delegation of the Agreement shall be void unless\n", + "prior written approval is obtained from the DHCS, in the instances where approval by the DHCS is required.\n", + "21.\n", + "Carve-out of California Children's Services (CCS) Program Services. The parties\n", + "acknowledge that health care services to treat CCS-eligible conditions are \"carved out\" of Health Net's coverage\n", + "obligations under the Medi-Cal Benefit Programs. Provider shall identify and timely refer Beneficiaries with\n", + "possible CCS-eligible conditions to the appropriate County CCS Program. Upon referral, Provider shall inform the\n", + "Beneficiary's parent or guardian and shall notify Health Net of any such referral. The CCS Program requires\n", + "eligible children to be treated at CCS-certified facilities by CCS-paneled providers. The CCS Program may require\n", + "transfer to CCS-certified facilities with CCS-paneled providers. The CCS Program is financially responsible for\n", + "payment of health care costs to treat a CCS-eligible condition. The parties understand and agree that Health Net is\n", + "not financially responsible for payment of services to treat CCS-eligible conditions. In the event Health Net\n", + "inadvertently pays a claim for such services, Health Net may recover the amount paid pursuant to Section 4.3 of the\n", + "Agreement.\n", + "22.\n", + "Cultural and Linguistic Services. Provider shall: (1) not require or encourage Beneficiaries to\n", + "utilize family Beneficiaries or friends as interpreters; (2) record the language needs of Beneficiaries in the medical\n", + "record; and (3) document Beneficiary requests or refusals of interpreter services in the Beneficiary's medical record.\n", + "Provider shall arrange interpreter services for Beneficiaries either through telephone language services or face-to-\n", + "face interpreters. Provider is encouraged to directly make these interpretive services available. However, upon\n", + "request. Health Net's Member Services Department is available to provide certain interpretive assistance to facilitate\n", + "communications.\n", + "23.\n", + "EPSDT Supplemental Services. Provider shall arrange for Early and Periodic Screening,\n", + "Diagnosis and Treatment Supplemental Services for Beneficiaries under the age of 21 in accordance with the\n", + "requirements of the Medi-Cal Agreement.\n", + "California Provider Participation Agreement\n", + "32\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-03-11\n", + "\n", + "Start of Page No. = 35\n", + "24.\n", + "CHDP (Children's Health and Disability Prevention) Program. Health Net requires that all\n", + "providers of CHDP services be certified by the CHDP Program and adhere to the CHDP Program requirements.\n", + "25.\n", + "CCS (California Children's Services). Provider is responsible for timely referral of children\n", + "with potential CCS eligible conditions. Failure to appropriately refer will result in Provider assuming financial\n", + "responsibility for any related charges.\n", + "26.\n", + "CPSP (Comprehensive Perinatal Services Program). Provider is required to refer members to\n", + "DHCS-certified CPSP providers to ensure that all pregnant women have access to care in accordance with DHCS\n", + "requirements.\n", + "27.\n", + "Sensitive Services. Sensitive Services are those health care services. which are covered by more\n", + "restrictive confidential treatment rules. In accordance with the Medi-Cal Agreement, Medi-Cal Members may self-\n", + "refer anywhere to obtain Sensitive Services and no referral or prior authorization shall be required.\n", + "Access\n", + "to\n", + "Sensitive Services shall not be limited in any way geographically or by provider network participation.\n", + "The Sensitive Services are as follows:\n", + "abortion (pregnancy termination) services,\n", + "family planning services, inclusive of all methods of birth control covered by the Department of\n", + "Health Care Services for the Medi-Cal Program.\n", + "sexually transmitted disease testing and treatment, and\n", + "Human Immunodeficiency Virus (HIV) testing and counseling\n", + "28.\n", + "Vaccines for Children Program (VFC)\n", + "Provider shall not seek reimbursement from Health\n", + "Net for immunizations covered by the State of California under the Vaccines for Children Program (VFC), where a\n", + "Provider, or Professional Provider who is a participant in the VFC program, rendered the immunization.\n", + "29.\n", + "Local Health Department Coordination. As more fully set out in the Medi-Cal Agreement,\n", + "Health Net or a contracting Medi-Cal plan has (or will) entered into agreements for specified public health services\n", + "with certain county health departments (Los Angeles, Fresno, Tulare. Riverside, San Bemardino, San Diego. and\n", + "Sacramento counties). The public health agreements specify the scope and responsibilities of the local health\n", + "departments and Health Net. billing and reimbursements, reporting responsibilities, and medical record management\n", + "to ensure coordinated health care services. The public health services specified under the agreements are as follows:\n", + "29.1\n", + "Family planning services;\n", + "29.2\n", + "Sexually transmitted disease (\"STD\") services diagnosis and treatment of disease episode of the\n", + "following STDs: syphilis, gouorrhea, chlamydia, herpes simplex, chancroid, trichomoniasis,\n", + "human papilloma virus, non-gonococcal urethritis, lynphogranuloma venereum and granuloma\n", + "inguinale;\n", + "29.3\n", + "Confidential HIV testing and counseling;\n", + "29.4\n", + "Immunizations:\n", + "29.5\n", + "Child Health and Disability Prevention Program;\n", + "29.6\n", + "California Children Services;\n", + "29.7\n", + "Maternal and Child Health:\n", + "29.8\n", + "Refugee assessments;\n", + "29.9\n", + "Tuberculosis Direct Observed Therapy:\n", + "California Provider Participation Agreement\n", + "33\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 36\n", + "29.10\n", + "Women, Infants, and Children Supplemental Food Program:\n", + "29.11 Population based Prevention Programs: collaborate in local health department community based\n", + "prevention programs.\n", + "Provider shall, in accordance with the terms and conditions of the public health agreements with the local\n", + "health departments and Health Net's related policies and procedures, be responsible for the coordination and\n", + "arrangement of the public health services for its assigned Beneficiaries. The services specified in Sections 29.1\n", + "through 29.5 above require reimbursement to the applicable local health department. The services specified in\n", + "Sections 29.6 through 29.11 above do not require reimbursement to the applicable local health department.\n", + "[Medi-Cal Agreement]\n", + "California Provider Participation Agreement\n", + "34\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 37\n", + "EXHIBIT C-1\n", + "MEDI-CAL BENEFIT PROGRAM\n", + "DIRECT NETWORK FEE-FOR-SERVICE\n", + "RATE EXHIBIT\n", + "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", + "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", + "Services delivered pursuant to this Addendum, the lesser of: 100% of the State of California Medi-Cal Fee Schedule\n", + "rates in effect at the time of service, subject to any adjustments made by the State of California under the applicable\n", + "Medi-Cal Fee-For-Service Program; (ii) Fee-for-service rates for the commercial Benefit Program set forth in\n", + "Addendum A, Exhibit A-1; or (iii) Provider's billed charges.\n", + "California Provider Participation Agreement\n", + "35\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 38\n", + "EXHIBIT C-2\n", + "DISCLOSURE FORM\n", + "(Required by California Welfare and Institutions Code Section 14452)\n", + "(Name of Provider)\n", + "The undersigned hereby certifies that the following information regarding:\n", + "(The \"Organization\") is true aud correct as of the date set forth below:\n", + "BagAllergy Asthma & Center, Inc\n", + "Officers/Directors/General Partners:\n", + "Malik N. Bag , M.D.\n", + "CHairman\n", + "Praveen Buddiga, M.D CEO\n", + "Co-Owner(s):\n", + "N/A\n", + "Stockholders owning more than ten percent of the stock of the Organization:\n", + "N/A\n", + "Major creditors holding more than five percent of Organization's debt:\n", + "NA\n", + "Form of Organization (Corporation, Partnership, Sole Proprietorship, Individual, etc.):\n", + "N/A\n", + "If not already disclosed above, is Organization, either directly or indirectly related to or affiliated with the\n", + "Contracting Health Plan? Please explain:\n", + "Dated: 5-17-2011\n", + "Signature:\n", + "Jay\n", + "Name: Praveen Buddga,MD.\n", + "(Please type or print)\n", + "Title:\n", + "CEO\n", + "(Please type or print)\n", + "California Provider Participation Agreement\n", + "Fee-For-Service Direct Network Template\n", + "36\n", + "HN-DN-PPA.-04-08-11\n", + "\n", + "Start of Page No. = 39\n", + "ADDENDUMI\n", + "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS\n", + "This Addendum D applies to Covered Services delivered to Beneficiaries covered by a Healthy Families Healthy\n", + "Kids or related benefit program. The term \"Beneficiary\" as used in this Addendum D shall be deemed to apply only\n", + "to Beneficiaries who are covered by a Healthy Families or a Healthy Kids Benefit Program. All Covered Services\n", + "delivered to such a Beneficiary shall be paid in accordance with this Addendum D regardless of product specific\n", + "name unless otherwise specifically agreed by the parties and set forth in a separate rate exhibit.\n", + "A.\n", + "COMPENSATION PROVISIONS\n", + "1.\n", + "Compensation and Applicability. Provider shall arrange and provide Contracted Services to\n", + "Beneficiaries of the Healthy Families and Healthry Kids Benefit Program covered under this Addendum on a fee-for-\n", + "service basis. As compensation for providing such Contracted Services, Provider shall be paid in accordance with\n", + "the rates set forth on Exhibit D-1, subject to the payment conditions set forth in Addendum E. This Addendum shall\n", + "be applicable to only those Healthy Families and Healthy Kids listed on the applicable remittance summaries. Health\n", + "Net will modify this Addendum to reflect a new rate structure for adults, pending regulatory approval of expanding\n", + "this program to parents. Provider shall submit claims for such services in accordance with the terms of this\n", + "Agreement and applicable State and federal law.\n", + "B.\n", + "GENERAL PROVISIONS\n", + "1.\n", + "Healthy Families Benefit Program. Health Net entered into an agreement with the California\n", + "Managed Risk Medical Insurance Board (\"MRMIB\") to arrange for the provision of Covered Services to persons\n", + "who are eligible under the California Children's Health Insurance Program \"Healthy Families Benefit Program\")\n", + "and enrolled in Health Net's Healthy Families Plan (\"Beneficiaries\"). Health Net arranges the Healthy Families\n", + "Program in certain California counties as a Health Maintenance Organization (\"HMO\") or an Exclusive Provider\n", + "Organization (\"EPO\"), as applicable to each county.\n", + "2.\n", + "Healthy Kids Benefit Program. Health Net participates in the Healthy Kids Programs (\"Healthy\n", + "Kids Benefit Program\") offered in certain California counties to provide medical coverage to children who qualify\n", + "for and are enrolled in the Healthy Kids Benefit Program in such California counties (\"Beneficiaries\"). Such\n", + "California counties have arranged for the provision of Covered Services to children who are eligible under the\n", + "individual county's program, collectively referred to herein as the Children's Health Insurance Collaborative\n", + "(\"CHIC\"). Health Net arranges the Healthy Kids Program in certain California counties as a Health Maintenance\n", + "Organization (\"HMO\") or an Exclusive Provider Organization (\"EPO\"). as applicable to each county.\n", + "3.\n", + "Participation and Provision of Covered Services. Provider understands and agrees that il shall\n", + "arrange and/or provide Covered Services to Beneficiaries in accordance with the terms and conditions of this\n", + "Agreement, applicable law, and the requirements of the Healthy Families and Healthy Kids Benefit Programs.\n", + "Provider understands that Benefit Program evidence of coverage documents are subject to review and approval by\n", + "appropriate regulatory or oversight entities including MRMIB, the California Department of Managed Health Care\n", + "and/or the CHIC.\n", + "4.\n", + "Carve-ont of California Children's Services (CCS) Program Services. The parties\n", + "acknowledge that health care services rendered to eligible Beneficiaries to treat CCS-eligible conditions are \"carved\n", + "out\" of Health Net's coverage obligations under the Healthy Families and Healthy Kids Benefit Programs. Provider\n", + "shall identify and timely refer Beneficiaries with possible CCS-eligible conditions to the appropriate County CCS\n", + "Program. Provider may ask Health Net to assist with this referral. Health Net shall also make a referral to CCS when\n", + "a Primary Care Physician refers a beneficiary to a specialist or where there is an inpatient admission which appears\n", + "to involve care for a CCS eligible condition. The CCS program will determine if the beneficiary's condition is\n", + "California Provider Participation Agreement\n", + "37\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 40\n", + "eligible for CCS services. Upon referral, Provider shall inform the Beneficiary's parent or guardian and shall notify\n", + "Health Net of any such referral. The CCS Program requires eligible children to be treated at CCS-certified facilities\n", + "by CCS-paneled providers. The CCS Program may require transfer to CCS-certified facilities with CCS-paneled\n", + "providers. The CCS Program is financially responsible for payment of health care costs to treat a CCS-eligible\n", + "condition. The parties understand and agree that Health Net is not financially responsible for payment of services to\n", + "treat eligible beneficiaries for CCS-eligible conditions. In the event Health Net inadvertently pays a claim for such\n", + "services, Health Net may recover the amount paid pursuant to Section 4.3 of the Agreement.\n", + "5.\n", + "Referral of Beneficiaries having possible mental health conditions to Managed Health\n", + "Network. Provider is required to identify and timely refer Beneficiaries with possible mental health conditions\n", + "(other than Severely Emotionally Disturbed as set out in the following section) to Health Net's affiliate and\n", + "subcontractor, Managed Health Network. Inc. Managed Health Network. Inc., is financially responsible for\n", + "payment of treatment of covered mental health services.\n", + "6.\n", + "Services for Severely Emotionally Disturbed (SED) Beneficiaries. For beneficiaries who are SED.\n", + "diagnosis and treatment are provided by the County Mental Health Department Health Net and the county mental\n", + "health department will coordinate to arrange or provide services so that all medically necessary services and\n", + "treatment are provided to Beneficiaries who are SED. Health care services to treat Beneficiaries who are SED are\n", + "the financial responsibility of the County Mental Health Department with the exception of the first thirty (30) days\n", + "of inpatient mental health care. Provider shall identify and timely refer Beneficiaries with possible SED to the\n", + "County Mental Health Department. Upon referral, Provider shall also inform the Beneficiary's parent or guardian.\n", + "The County Mental Health Department is exclusively responsible for the provision and payment of health care costs\n", + "to treat Beneficiaries who have been diagnosed as SED, with the exception of the provision and payment of health\n", + "care costs for the first thirty (30) days of inpatient mental health care. The Healthy Families Benefit Program defines\n", + "a Beneficiary as Severely Emotionally Disturbed (SED), if that beneficiary meets the criteria set forth in California\n", + "Welfare and Institutions Code 5600.3.\n", + "7.\n", + "Reports and Information. Provider shall provide to Health Net. within the time requested by\n", + "Health Net, all such reports and information as Health Net may require so that Health Net can meet the reporting\n", + "requirements hunder the Healthy Families, Healthy Kids or related benefit programs, and applicable law. Such reporting\n", + "obligations include, but are not limited to, monthly reporting of Beneficiary referrals to the following programs:\n", + "California Children's Services, referrals to Managed Health Network, and referrals of Beneficiaries with a diagnosis\n", + "of SED to the County Mental Health Department.\n", + "8.\n", + "Cultural and Linguistic Services. Provider shall: (1) not require or encourage Beneficiaries to\n", + "utilize family Beneficiaries or friends as interpreters; (2) record the language needs of Beneficiaries in the medical\n", + "record; and (3) document Beneficiary requests or refusals of interpreter services in the Beneficiary's medical record.\n", + "Provider shall arrange interpreter services for Beneficiaries either through telephone language services or face-to-\n", + "face interpreters. Provider is encouraged to directly make these interpretive services available. However, upon\n", + "request, Health Net's Member Services Department is available to provide certain interpretive assistance to facilitate\n", + "communications.\n", + "10.\n", + "Eligibility. Eligibility and commencement of enrollment under Healthy Families Benefit Program\n", + "is determined by MRMIB. Commencement of coverage for Beneficiaries can occur at any day of a month.\n", + "Eligibility and commencement of enrollment under Healthy Kids Benefit Program is determined by the appropriate\n", + "Healthy Kids county Children's Health Collaborative CIII, or its successor.\n", + "11.\n", + "Copayments. Copayments are subject to an animal limitation. Provider is encouraged to make\n", + "extended payment arrangements available to Beneficiaries experiencing an inability to pay a required copayment.\n", + "California Provider Participation Agreement\n", + "38\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-03-11\n", + "\n", + "Start of Page No. = 41\n", + "EXHIBIT D-1\n", + "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS\n", + "DIRECT NETWORK FEE-FOR-SERVICE\n", + "RATE EXHIBIT\n", + "Subject to the terms of this Agreement, including without limitation the Payment Conditions sei forth in Addendum E.\n", + "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", + "Services delivered to Beneficiaries under the Healthy Families and Healthy Kids Benefit Program pursuant to this\n", + "Addendum the lesser of 75% of Provider's Billed Charges or:\n", + "(a) 120% of the State of California Medi-Cal Fee Schedule rates in effect at the time of service, subject to\n", + "any adjustments made by the State of California under the applicable Medi-Cal Fee-For-Service Program;\n", + "or\n", + "(b) for those \"by report procedures, and/or unlisted procedures and relativities that are not established on\n", + "the State of California's Medi-Cal Fee Schedule:\n", + "80% of the CMS allowable.\n", + "For those \"By report\" procedures, and/or unlisted procedures and relativities that are not established on\n", + "both the State of California's Medi-Cal Fee Schedule or the CMS participating provider fee schedule for physician's\n", + "Locality, Health Net shall pay 60% of Provider's billed charges.\n", + "California Provider Participation Agreement\n", + "39\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n", + "\n", + "Start of Page No. = 42\n", + "ADDENDUMI\n", + "DIRECT NETWORK FEE-FOR-SERVICE PAYMENT CONDITIONS\n", + "The Payment Conditions set forth in this Addendum supplement Health Net Policies, and are applicable to\n", + "Commercial Benefit Programs, Medicare Advantage Benefit Program, Medi-Cal Benefit Program and the Healthy\n", + "Families and Healthy Kids Benefit Programs.\n", + "1.\n", + "Provider shall utilize valid CPT/HCPCS/ICD9 diagnosis_codes, or successor codes, when\n", + "submitting Complete Claims for Covered Services under this Agreement. The parties acknowledge that applicable\n", + "coding agencies periodically issue coding modifications. Such modifications may be implemented by Health Net\n", + "within sixty (60) days of the date Health Net receives the modification from the applicable coding agency. The\n", + "parties agree that in the event such coding modifications have the effect. of changing a payment amount in this\n", + "Agreement, the resulting payment amount change shall be effective on a prospective basis. The parties further agree\n", + "to reasonably and in good faith discuss any contract rate amendment that may be appropriate based on\n", + "comprehensive and substantive coding changes by the applicable coding agency within ninety (90) days of written\n", + "notification by either party. Any and all rate modifications that may result from such contract amendment shall be\n", + "effective on a prospective basis.\n", + "2.\n", + "Pharmaceuticals shall include inhaled/infused/injected medications provided by. and delivered in\n", + "PPG/Professional Provider office. Support drugs and injection supplies for patient self-administered injections shall\n", + "be supplied only through a Health Net preferred specialty pharmacy provider. AWP pricing shall be determined by\n", + "Health Net utilizing a nationally recognized source for pharmaceutical pricing to be updated at least quarterly.\n", + "Provider shall bill administered dosage in accordance with applicable HCPCS codes. Multi-dose vials shall be\n", + "reimbursed on a per dose basis.\n", + "3.\n", + "Immunizations shall include procedure codes for immune globulins, vaccines and toxoids that\n", + "identify the product only. The administration for immune globulins, vaccines and toxoids are separate from the\n", + "product itself and are not considered immunizations. AWP pricing shall be determined by Health Net utilizing a\n", + "nationally recognized source for pharmaceutical pricing to be updated at least quarterly. Provider shall bill\n", + "administered dosage in accordance with applicable HCPCS codes. Multi-dose vials shall be reimbursed on a per\n", + "dose basis.\n", + "1.\n", + "The rates set forth in this Agreement apply to all current and future locations billed under this and\n", + "future Tax Identification Numbers indicated by Provider through a signed W-9 form and subject to terms of this\n", + "Agreement.\n", + "California Provider Participation Agreement\n", + "40\n", + "Fee-For-Service Direct Network Template\n", + "HN-DN-PPA-04-08-11\n" + ] + } + ], + "source": [ + "print(table_edit['Baz Allergy Asthma & Sinus_MU.txt'])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def extract_all_tables(input_dict):\n", + " tables_dict = {}\n", + " for filename, filetext in input_dict.items():\n", + " tables_dict[filename] = extract_tables(filetext)\n", + " return tables_dict\n", + "\n", + "t = extract_all_tables(input_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "dict_keys(['Baz Allergy Asthma & Sinus_MU.txt'])" + "{'Baz Allergy Asthma & Sinus_MU.txt': []}" ] }, - "execution_count": 10, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "table_edit['Baz Allergy Asthma & Sinus_MU.txt']" + "t" ] }, { diff --git a/src/utils.py b/src/utils.py index 129c6bd..2f78503 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,5 +1,6 @@ import os +import re import pandas as pd import shutil @@ -93,3 +94,16 @@ def preprocess_text_file(file_path): # Split the text into pages based on a specific marker pages = re.split(r'Start of Page No\. = \d+', text) return pages + + +def format_td_check(td_dicts): + final_str = "" + dict_count = 1 + for td_dict in td_dicts: + final_str += str(dict_count) + '. ' + for k in td_dict.keys(): + if k not in ['Filename', 'page_num']: + final_str += k + ': ' + td_dict[k] + ', ' + final_str += '\n' + dict_count += 1 + return final_str From 1d9a3b0292d9c1a783142711066ef3174eb05172 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 21 May 2024 16:20:35 -0400 Subject: [PATCH 25/78] Integrated TD approach --- src/config.py | 2 +- src/prompt_funcs.py | 42 +- src/prompts.py | 26 + src/test_notebook.ipynb | 2686 +++------------------------------------ src/utils.py | 4 +- 5 files changed, 254 insertions(+), 2506 deletions(-) diff --git a/src/config.py b/src/config.py index 3a42fdf..d7e76b7 100644 --- a/src/config.py +++ b/src/config.py @@ -33,7 +33,7 @@ AWS_SECRET_ACCESS_KEY="f/mOlRpK7+iSlm1P5ZK1LOdS2UjnsbAQ2ysL7Mms" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEM3//////////wEaCXVzLWVhc3QtMiJGMEQCIGqPr40CU3Fz4nvSQJ1jaPgkOGIWWyDXBVLUU+VmsfIjAiAgluQOwSGIYph5hLcKVVsiFQ7OpLOs3lLsumxzeyw6nyqDAwhGEAAaDDY2MDEzMTA2ODc4MiIM5odYPf2eQP8/f9NBKuAC7eMH0qjb6ASoeBhtKNFzMWXpR5snAahfBheyiUo1my8q8VFbZem8m0n5c0vWnKsPhm9+K+qpVqM6ohhjVMIXCta/ZlGNizatHJt7gMpDOGQNyBI4rpXF9/urQMgN8PkjMG2hkD+XefkPYGmvgz9ZVKFs/oAnP/buG21Ju54F/D0qqnxhR/90pVM6Z48sgzkQThHHjB7wNgfy1hIA7CL9yeM0k+sdzlYvriDogDi3lcP3UkXMUJhQl5xQb7t8iLR70L/0H74ynfldpbaBfktWPred4ehoRMv2Zd/Jh6JNBIAeDQyfIKQTYn2FF/RuPBmqa+iLS9lJh/SnhwXNuMYc9JOERwMHaijgqQjenISnysSEr10u1Y+XiUI7waXRO6e8CBS/LygUKZZf8+cSeXIE3VN2XrMW72Ix5sg16QHAMvR3oUDJdYdf8jnWET995kocK6UctJdVKEna/FUnIdNgwzDirrKyBjqnAZjoQpVVuEh8p9tuGrjdWsjzLsXRkNRyqm/6xxIGtqITzvAv+TKQLSgJun/rRTOncaMLDwt0PAKL1Y8cVnOcuSIfVFUtpeZ+p27MgByL/fBQGd0yfZGHy15ZMPw51viYhjQo3MeA4y0YLRW+12v2ilrMxGHYlwTUT19Zj+qQNobTrlcJWQ7737lXmVL2WOZzmtduH7wf/fyAn2ZhZT/lcnSSQsef9ORk" # File Paths -LOCAL_PATH = '../docs/test/' # Replace with local +LOCAL_PATH = 'docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index bd91010..6bb3653 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -120,8 +120,39 @@ def run_bottom_up(filename, text_dict): return results_dicts # List of dictionaries +def run_top_down_metal_level(d, page): + if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper(): + prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + else: + answer = 'N/A' + return answer + +def run_top_down_date(type_, d, page): + formatted_d = utils.format_td_check([d], ['Filename', 'page_num']) + prompt = prompts.TOP_DOWN_DATE('EFFECTIVE', formatted_d, page) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + return answer + + +def top_down_secondary(td_results, text_dict): + updated_dicts = [] + for d in td_results: + # Run Metal Level + d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']]) + + # Dates + d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) + d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) + + updated_dicts.append(d) + return updated_dicts + + def run_top_down(filename, text_dict): all_results = [] + + # Primary for page_num, page_text in text_dict.items(): if not page_num.isdigit(): continue @@ -129,9 +160,12 @@ def run_top_down(filename, text_dict): answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries all_results.append(answer_dicts) - all_results_final = [i for d in all_results for i in d] # Consolidate to one list - return all_results_final # List of list of dictionaries + td_primary = [i for d in all_results for i in d] # Consolidate to one list + # Secondaries + td_final = top_down_secondary(td_primary, text_dict) + + return td_final # List of list of dictionaries def td_bu_combine(td, bu, text_dict): combined_list = [] @@ -143,10 +177,9 @@ def td_bu_combine(td, bu, text_dict): bud.update(td_dict) # Case 2: More than one option elif len(td_dicts) > 1: - td_check_formatted = utils.format_td_check(td_dicts) + td_check_formatted = utils.format_td_check(td_dicts, ['Filename', 'page_num']) prompt = prompts.LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']]) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - print(answer) answer_idx = ''.join(re.findall(r'\d', answer)) bud.update(td_dicts[int(answer_idx)-1]) # Case 3: No option for page @@ -175,7 +208,6 @@ def run_prompts(file_object): #### COMBINE #### combined_results = td_bu_combine(td_results, bu_results, text_dict) - answer_df = pd.DataFrame(combined_results) ### OUTPUT #### diff --git a/src/prompts.py b/src/prompts.py index 5a89ee3..14bce4c 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -152,7 +152,33 @@ def LOB_CHECK(bu_dict, td_dicts, page): """ +def TOP_DOWN_METAL_LEVEL(LOB, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above page was identified as applying to a Commercial or Marketplace Line of Business. +What is the metal level(s) of the line of business. It might be Bronze, Silver, Gold, or Platinum. If multiple are found, return them all in a comma-separated list. If no metal level is found, return 'N/A'. + +Only write what is written on the page. Do not make up new phrases or words. + +ONLY return the metal level, with no other commentary or explanation. +""" + + +def TOP_DOWN_DATE(type_, d, page): + return f"""### PAGE START ### {page} ### PAGE END + +Above is a page of a contract. Identify the {type_} Date associated with the information below: + +{d} + +Only write what is written on the page. Do not make up new phrases or words. If no {type_} date is listed, return 'N/A'. + +Ensure that you do not mistake EFFECTIVE date for TERMINATION date, or vice versa. They are not the same thing. +The TERMINATION date will NOT be labelled as the EFFECTIVE date, and the EFFECTIVE date will NOT be labelled as the TERMINATION date. This is very important, so pay extra close attention. + +ONLY return the answer, with no other commentary or explanation. +""" diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb index 19add88..6e9343e 100644 --- a/src/test_notebook.ipynb +++ b/src/test_notebook.ipynb @@ -33,19 +33,7 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'6.2000_Omni Transport Systems_ServicesAgreement MU.txt'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[3], line 2\u001b[0m\n\u001b[0;32m 1\u001b[0m filename \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m6.2000_Omni Transport Systems_ServicesAgreement MU.txt\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m----> 2\u001b[0m contract_text \u001b[38;5;241m=\u001b[39m \u001b[43minput_dict\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m6.2000_Omni Transport Systems_ServicesAgreement MU.txt\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[0;32m 4\u001b[0m contract_text \u001b[38;5;241m=\u001b[39m preprocess\u001b[38;5;241m.\u001b[39mclean_newlines(contract_text)\n\u001b[0;32m 5\u001b[0m text_dict \u001b[38;5;241m=\u001b[39m preprocess\u001b[38;5;241m.\u001b[39msplit_text(contract_text)\n", - "\u001b[1;31mKeyError\u001b[0m: '6.2000_Omni Transport Systems_ServicesAgreement MU.txt'" - ] - } - ], + "outputs": [], "source": [ "filename = 'Baz Allergy Asthma & Sinus_MU.txt'\n", "contract_text = input_dict['Baz Allergy Asthma & Sinus_MU.txt']\n", @@ -60,82 +48,204 @@ "cell_type": "code", "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{\"CONTRACT_LOB\": \"Commercial, Medicare Advantage, Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service, Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[\n", + " {\n", + " \"CONTRACT_LOB\": \"N/A\",\n", + " \"CONTRACT_PROGRAM\": \"N/A\",\n", + " \"CONTRACT_NETWORK\": \"FFS\",\n", + " \"PRODUCT\": \"Health Net\"\n", + " }\n", + "]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[\n", + " {\n", + " \"CONTRACT_LOB\": \"Commercial\",\n", + " \"CONTRACT_PROGRAM\": \"N/A\",\n", + " \"CONTRACT_NETWORK\": \"HMO\",\n", + " \"PRODUCT\": \"Health Net\"\n", + " }\n", + "]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"N/A\"}]\n", + "[{\"CONTRACT_LOB\": \"Commercial\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO, PPO, EPO, POS\", \"PRODUCT\": \"AIM, leased networks\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"PPO, EPO, POS, Leased PPO\", \"PRODUCT\": \"N/A\"}]\n", + "[{\"CONTRACT_LOB\": \"Commercial\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Direct Network, Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Direct Network, Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Health Net, CalViva Health, Medi-Cal\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net Medi-Cal\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Health Net, Medi-Cal HMO\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Medi-Cal, Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Medi-Cal, Healthy Families\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"CHDP (Children's Health and Disability Prevention) Program, CCS (California Children's Services), CPSP (Comprehensive Perinatal Services Program), Vaccines for Children Program (VFC)\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Medi-Cal Benefit Program, Health Net\"}]\n", + "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"N/A\"}]\n", + "[\n", + " {\n", + " \"CONTRACT_LOB\": \"Medicaid\",\n", + " \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids\",\n", + " \"CONTRACT_NETWORK\": \"HMO, EPO\",\n", + " \"PRODUCT\": \"Healthy Families Plan\"\n", + " }\n", + "]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net, Managed Health Network\"}]\n", + "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids Benefit Program\", \"CONTRACT_NETWORK\": \"Direct Network, FFS\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Medi-Cal Fee-For-Service Program, CMS\"}]\n", + "[{\"CONTRACT_LOB\": \"Commercial, Medicare Advantage, Medicaid, Healthy Families, Healthy Kids\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n" + ] + } + ], "source": [ - "td_results = prompt_funcs.run_top_down(filename, text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "bu_results = prompt_funcs.run_bottom_up(filename, text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "td_results = [i for d in td_results for i in d] # Turn list of list of dicts into list of dicts" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": {}, - "outputs": [], - "source": [ - "def LOB_CHECK(bu_dict, td_dicts, page):\n", - " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + "import prompts\n", + "import dict_operations\n", + "\n", + "def run_top_down(filename, text_dict):\n", + " all_results = []\n", + "\n", + " # Primary\n", + " for page_num, page_text in text_dict.items():\n", + " if not page_num.isdigit():\n", + " continue\n", + " prompt = prompts.TOP_DOWN_PRIMARY(page_text)\n", + " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries\n", + " all_results.append(answer_dicts)\n", + " td_primary = [i for d in all_results for i in d] # Consolidate to one list\n", + "\n", + " # Secondaries\n", " \n", - " Service: {bu_dict['SERVICE']}\n", - " Methodology: {bu_dict['FULL_METHODOLOGY']}\n", "\n", - " Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects:\n", - "\n", - " {td_dicts}\n", - "\n", - " Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", - " \"\"\"\n", - "\n", - "def format_td_check(td_dicts):\n", + " return td_primary # List of list of dictionaries" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": {}, + "outputs": [], + "source": [ + "def format_td_check(td_dicts, dont_include_list):\n", " final_str = \"\"\n", " dict_count = 1\n", " for td_dict in td_dicts:\n", " final_str += str(dict_count) + '. '\n", " for k in td_dict.keys():\n", - " if k not in ['Filename', 'page_num']:\n", + " if k not in dont_include_list:\n", " final_str += k + ': ' + td_dict[k] + ', '\n", " final_str += '\\n'\n", " dict_count += 1\n", " return final_str\n", "\n", - "def td_bu_combine(td, bu):\n", - " combined_list = []\n", - " for bud in bu:\n", - " td_dicts = [d for d in td if d['page_num'] == bud['page_num']]\n", - " # Case 1: Only one option\n", - " if len(td_dicts) == 1:\n", - " td_dict = td_dicts[0]\n", - " bud.update(td_dict)\n", - " # Case 2: More than one option\n", - " elif len(td_dicts) > 1: \n", - " td_check_formatted = format_td_check(td_dicts)\n", - " prompt = LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']])\n", - " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", - " print(answer)\n", - " answer_idx = ''.join(re.findall(r'\\d', answer))\n", - " bud.update(td_dicts[int(answer_idx)-1])\n", - " # Case 3: No option for page\n", - " else:\n", - " continue\n", - " combined_list.append(bud)\n", - " return combined_list\n", - " " + "def TOP_DOWN_METAL_LEVEL(LOB, page):\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "The above page was identified as applying to a Commercial or Marketplace Line of Business. \n", + "\n", + "What is the metal level(s) of the line of business. It might be Bronze, Silver, Gold, or Platinum. If multiple are found, return them all in a comma-separated list. If no metal level is found, return 'N/A'.\n", + "\n", + "Only write what is written on the page. Do not make up new phrases or words. \n", + "\n", + "ONLY return the metal level, with no other commentary or explanation.\n", + "\"\"\"\n", + "\n", + "\n", + "def TOP_DOWN_DATE(type_, d, page):\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + "\n", + "Above is a page of a contract. Identify the {type_} Date associated with the information below:\n", + "\n", + "{d}\n", + "\n", + "Only write what is written on the page. Do not make up new phrases or words. If no {type_} date is listed, return 'N/A'.\n", + "\n", + "Ensure that you do not mistake EFFECTIVE date for TERMINATION date, or vice versa. They are not the same thing. \n", + "The TERMINATION date will NOT be labelled as the EFFECTIVE date, and the EFFECTIVE date will NOT be labelled as the TERMINATION date. This is very important, so pay extra close attention.\n", + "\n", + "ONLY return the answer, with no other commentary or explanation.\n", + "\"\"\"\n", + "\n", + "\n", + "def run_top_down_metal_level(d, page):\n", + " if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper():\n", + " prompt = TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page)\n", + " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " else:\n", + " answer = 'N/A' \n", + " return answer\n", + "\n", + "def run_top_down_date(type_, d, page):\n", + " formatted_d = format_td_check([d], ['Filename', 'page_num'])\n", + " prompt = TOP_DOWN_DATE('EFFECTIVE', formatted_d, page)\n", + " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " return answer\n", + "\n", + "\n", + "def top_down_secondary(td_results, text_dict):\n", + " updated_dicts = []\n", + " for d in td_results:\n", + " # Run Metal Level\n", + " d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']])\n", + " \n", + " # Dates\n", + " d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']])\n", + " d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']])\n", + "\n", + " updated_dicts.append(d)\n", + " return updated_dicts\n", + "\n", + "td_dicts = top_down_secondary(td_results, text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'CONTRACT_LOB': 'N/A',\n", + " 'CONTRACT_PROGRAM': 'N/A',\n", + " 'CONTRACT_NETWORK': 'Fee-For-Service Direct Network',\n", + " 'PRODUCT': 'Health Net',\n", + " 'page_num': '19',\n", + " 'Filename': 'Baz Allergy Asthma & Sinus_MU.txt',\n", + " 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A',\n", + " 'LOB_PRICING_TERMS_EFFECTIVE_DATE': '10-6-2011',\n", + " 'LOB_PRICING_TERMS_TERMINATION_DATE': '10-6-2011'}" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "td_dicts[18]" ] }, { @@ -147,2432 +257,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "1\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n", - "2\n", - "2\n" + "19 7.14 Status as Independent Entitles. None of the provisions of this Agreement is intended to create, nor shall be deemed or construed to create any relationship between Provider and Health Net or a Payor other than that of independent entities contracting with each other solely for the purpose of effecting the provisions of this Agreement. Neither Provider nor Health Net/Payor, nor any of their respective agents, employees or representatives shall be construed to be the agent, employee or representative of the other. 7.15 Addenda. Each Addendum to this Agreement is made a part of this Agreement as though set forth fully herein. Any provision of an Addendum that is in conflict with any provision of this Agreement shall take precedence and superseda the conflicting provision of this Agreement with respect to the subject matter of the Addendum. 7.16 Calculation of Time. The parties agree that for purposes of calculating time under this Agreement, any time period of less than ten (10) days shall be deemed to refer to business days and any time period of ten (10) days or more shall be deemed to refer to calendar days unless the term \"business\" precedes the term \"days\". 7.17 Waiver of Breach. The waiver of any breach of this Agreement by either party shall not constitute a continuing waiver of any subsequent breach of either the same or any other provision(s) of this Agreement. Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict compliance in the future and to exercise any right or remedy related thereto. THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED BY THE PARTIES, IN WITNESS WHEREOF, the parties have executed this Agreement. PROVIDER Love HEALTHNE Signature Obly Signature Hoens Health Malik N. BAZ , M.D. Cathy Hoens Print Name Print Name Chairman Title VP Provider Network Management & Strategy Title Baz Allergy Asthma & Sinus Group Name (If Applicable) LIR. 6/29/2011 Date 77-0508441 7/15/11 Tax Identification Number 10-6-2011 Effective Date Date California Provider Participation Agreement Fee-For-Service Direct Network Template 18 HN-DIN-PPA-04-08-11 This page has 2 signature.\n" ] } ], "source": [ - "combined_results = td_bu_combine(td_results, bu_results)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "import re\n", - "import pandas as pd\n", - "import ast\n", - "\n", - "def extract_tables(text):\n", - " pattern = re.compile(r\"-------Table Start--------(.*?)-------Table End--------\", re.DOTALL)\n", - " tables = pattern.findall(text)\n", - " table_list = [table.strip() for table in tables]\n", - " table_list = ['{'+table.split('{')[1] for table in table_list]\n", - " df_list = [pd.DataFrame(ast.literal_eval(table)) for table in table_list]\n", - " for df in df_list:\n", - " print(df.columns)\n", - " return df_list\n", - "\n", - "def extract_all_tables(input_dict):\n", - " tables_dict = {}\n", - " for filename, filetext in input_dict.items():\n", - " tables_dict[filename] = extract_tables(filetext)\n", - " return tables_dict\n", - "\n", - "def format_table(table):\n", - " table = table.replace('>>>', '').replace('<<<', '')\n", - " table_dict = eval(table)\n", - " \n", - " keys = list(table_dict.keys())\n", - " # If values are lists\n", - " if isinstance(table_dict[keys[0]], list):\n", - " final_string = \"\"\n", - " for i in range(len(table_dict[keys[0]])):\n", - " for k in keys:\n", - " key_string = k + ': ' + table_dict[k][i] + ', '\n", - " final_string += key_string\n", - " final_string += '|\\n'\n", - " else:\n", - " # If values are not lists\n", - " final_string = \"\"\n", - " for k in keys:\n", - " key_string = k + ': ' + table_dict[k] + ', '\n", - " final_string += key_string\n", - " final_string += '|\\n'\n", - " return final_string\n", - "\n", - "def align_and_format_tables(text_dict):\n", - " pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL)\n", - " for page, text in text_dict.items():\n", - " tables = pattern.findall(text)\n", - " for table in tables:\n", - " pre_table = table.split('{')[0].strip() # Get the pretable text for alignment\n", - " text = text.replace(table, '') # Remove full table\n", - " table_only = table.replace(pre_table, '')\n", - " try:\n", - " formatted_table = format_table(table_only)\n", - " except:\n", - " formatted_table = table\n", - " text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table\n", - " text = text.replace('-------Table Start--------', '')\n", - " text = text.replace('-------Table End--------', '') \n", - " text_dict[page] = text\n", - " return text_dict\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "table_edit = align_and_format_tables(input_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Document Index\n", - "CALIFORNIA 1PROVIDER PARTICIPATION AGREEMENT 1FEE-FOR-SERVICE DIRECT NETWORK TEMPLATE 1\n", - "Table of ContentsSection/Addendum: Section I, Title: Definitions, Page: 1, |\n", - "Section/Addendum: Section II, Title: Duties of Provider, Page: 4, |\n", - "Section/Addendum: Section III, Title: Duties of Health Net, Page: 7, |\n", - "Section/Addendum: Section IV, Title: Financial Obligations, Page: 8, |\n", - "Section/Addendum: Section V, Title: Term and Tennination, Page: 12, |\n", - "Section/Addendum: Section VI, Title: Records, Audits and Regulatory Requirements, Page: 13, |\n", - "Section/Addendum: Section VII, Title: General Provisions, Page: 14, |\n", - "Section/Addendum: Exhibit I, Title: List of Facilities, Page: 19, |\n", - "Section/Addendum: Exhibit Il, Title: List of Professional Providers, Page: 20, |\n", - "Section/Addendum: Addendum A, Title: Commercial Benefit Programs; Provisions, Page: 21, |\n", - "Section/Addendum: Exhibit A-1, Title: Fee-For-Service Rate Exhibit - Commercial Benefit Programs, Page: 22, |\n", - "Section/Addendum: Addendum B, Title: Medicare Advantage Provisions, Page: 23, |\n", - "Section/Addendum: Exhibit B-1, Title: Medicare Advantage Fee-For-Service Rate Exhibit, Page: 27, |\n", - "Section/Addendum: Addendum C, Title: Medi-Cal Benefit Program; Provisions, Page: 28, |\n", - "Section/Addendum: Exhibit C-1, Title: Medi-Cal Fee-For-Service Rate Exhibit, Page: 35, |\n", - "Section/Addendum: Exhibit C-2, Title: Disclosure Form, Page: 36, |\n", - "Section/Addendum: Addendum D, Title: Healthy Families and Healthy Kids Benefit Program: Provisions, Page: 37, |\n", - "Section/Addendum: Exhibit D-1, Title: Healthy Families and Healthry Kids Benefit Program Rate Exhibit, Page: 39, |\n", - "Section/Addendum: Addendum E, Title: Direct Network Fee-For-Service Payment Conditions, Page: 40, |\n", - " 1\n", - "CALIFORNIA 2PROVIDER PARTICIPATION AGREEMENT 2FEF-FOR-SERVICE DIRECT NETWORK TEMPLATE 2\n", - "RECITALS 2\n", - "AGREEMENT 2\n", - "I. 2DEFINITIONS 2\n", - "II. 5DUTIES OF PROVIDER 5\n", - "III. 8DUTIES OF HEALTHNET 8\n", - "4.2 9Billing and Payment. 9\n", - "4.3 10Recompment of Overpayments; Right of Offset. 10\n", - "Net or a Payor. 12\n", - "V. 13TERM AND TERMINATION 13\n", - "VI. 14RECORDS, AUDITS AND REGULATORY REQUIREMENTS 14\n", - "VII. 15GENERAL PROVISIONS 15\n", - "7.9 18Indemnification. 18\n", - "THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED 19BY THE PARTIES, 19\n", - "EXHIBIT I 20LIST OF FACILITIES 20\n", - "Primary Location 20\n", - "Secondary Location 20\n", - "Remit Address 20\n", - "Baz Allergy, Asthma & Sinus Center 21\n", - "OFFICE LOCATIONS 21\n", - "Second Office Address: NPI# 1508885781 21\n", - "Third Office Address: NPI# 1023037108 21\n", - "Fourth Office Address: NPI # 1811916992 21\n", - "Fifth Office Address: NPI# 1215956396 21\n", - "Sixth Office Address NPI# 1336399013 21\n", - "EXHIBIT II 22LIST OF PROFESSIONAL PROVIDERS 22\n", - "INSERT ROSTER HERE 22\n", - "ADDENDUM A 23\n", - "COMMERCIAL BENEFIT PROGRAMS 23\n", - "NOT APPLICABLE 23\n", - "EXHIBIT A-1 24\n", - "DIRECT NETWORK FEE-FOR-SERVICE 24RATE EXHIBIT 24\n", - "COMMERCIAL BENEFIT PROGRAMS 24\n", - "ADDENDUMB 25\n", - "MEDICARE ADVANTAGE PROGRAM 25\n", - "EXHIBIT B-1 29\n", - "MEDICARE ADVANTAGE PROGRAM 29DIRECT NETWORK FEE-FOR-SERVICE 29RATE EXHIBIT 29\n", - "I. Payment Rates 29\n", - "ADDENDUM O 30\n", - "MEDI-CAL BENEFIT PROGRAM 30\n", - "A. 30COMPENSATION PROVISIONS. 30\n", - "B. 31GENERAL PROVISIONS 31\n", - "EXHIBIT C-2 38\n", - "DISCLOSURE FORM 38\n", - "ADDENDUMI 39\n", - "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS 39\n", - "A. 39COMPENSATION PROVISIONS 39\n", - "B. 39GENERAL PROVISIONS 39\n", - "EXHIBIT D-1 41\n", - "ADDENDUMI 42DIRECT NETWORK FEE-FOR-SERVICE PAYMENT CONDITIONS 42\n", - "\n", - "\n", - "\n", - "Start of Page No. = 1\n", - "CALIFORNIA\n", - "PROVIDER PARTICIPATION AGREEMENT\n", - "FEE-FOR-SERVICE DIRECT NETWORK TEMPLATE\n", - "Table of ContentsSection/Addendum: Section I, Title: Definitions, Page: 1, |\n", - "Section/Addendum: Section II, Title: Duties of Provider, Page: 4, |\n", - "Section/Addendum: Section III, Title: Duties of Health Net, Page: 7, |\n", - "Section/Addendum: Section IV, Title: Financial Obligations, Page: 8, |\n", - "Section/Addendum: Section V, Title: Term and Tennination, Page: 12, |\n", - "Section/Addendum: Section VI, Title: Records, Audits and Regulatory Requirements, Page: 13, |\n", - "Section/Addendum: Section VII, Title: General Provisions, Page: 14, |\n", - "Section/Addendum: Exhibit I, Title: List of Facilities, Page: 19, |\n", - "Section/Addendum: Exhibit Il, Title: List of Professional Providers, Page: 20, |\n", - "Section/Addendum: Addendum A, Title: Commercial Benefit Programs; Provisions, Page: 21, |\n", - "Section/Addendum: Exhibit A-1, Title: Fee-For-Service Rate Exhibit - Commercial Benefit Programs, Page: 22, |\n", - "Section/Addendum: Addendum B, Title: Medicare Advantage Provisions, Page: 23, |\n", - "Section/Addendum: Exhibit B-1, Title: Medicare Advantage Fee-For-Service Rate Exhibit, Page: 27, |\n", - "Section/Addendum: Addendum C, Title: Medi-Cal Benefit Program; Provisions, Page: 28, |\n", - "Section/Addendum: Exhibit C-1, Title: Medi-Cal Fee-For-Service Rate Exhibit, Page: 35, |\n", - "Section/Addendum: Exhibit C-2, Title: Disclosure Form, Page: 36, |\n", - "Section/Addendum: Addendum D, Title: Healthy Families and Healthy Kids Benefit Program: Provisions, Page: 37, |\n", - "Section/Addendum: Exhibit D-1, Title: Healthy Families and Healthry Kids Benefit Program Rate Exhibit, Page: 39, |\n", - "Section/Addendum: Addendum E, Title: Direct Network Fee-For-Service Payment Conditions, Page: 40, |\n", - "\n", - "California Provider Participation Agreement\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA- 04-08-11\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 2\n", - "CALIFORNIA\n", - "PROVIDER PARTICIPATION AGREEMENT\n", - "FEF-FOR-SERVICE DIRECT NETWORK TEMPLATE\n", - "This Provider Participation Agreement (\"Agreement\") is made and entered into by and between the\n", - "Provider identified on the signature page of this Agreement (\"Provider\"), and Health Net of California, Inc. on\n", - "behalf of itself and the subsidiaries and affiliates of Health Net, Inc. (collectively, \"Health Net\"). This Agreement is\n", - "effective as of the effective date shown on the signature page of this Agreement. subject to the completion of any\n", - "applicable credentialing requirements.\n", - "RECITALS\n", - "A.\n", - "Provider has the legal authority to enter into this Agreement, and to deliver or arrange for the\n", - "delivery of Contracted Services.\n", - "B.\n", - "Health Net has the legal authority to enter into this Agreement, and to perform the obligations of\n", - "Health Net hereunder with respect to the Benefit Programs.\n", - "C.\n", - "The parties desire to enter into this Agreement to arrange for Provider to participate in one or more\n", - "of Health Net's networks of Participating Providers that render Contracted Services to Beneficiaries of various\n", - "Benefit Programs.\n", - "D.\n", - "Provider's primary consideration shall be the quality of the health care services rendered to\n", - "Beneficiaries, pursuant to Title 10 CCR 2240.4\n", - "AGREEMENT\n", - "NOW, THEREFORE in consideration of the above recitals and the covenants contained herein, the parties hereby\n", - "agree as follows:\n", - "I.\n", - "DEFINITIONS\n", - "Many words and terms are capitalized throughout this Agreement to indicate that they are defined as set forth in this\n", - "Article I.\n", - "1.1\n", - "Beneficiary. A person who is properly enrolled in and eligible to receive Covered Services under\n", - "a Benefit Program at the time Covered Services are rendered. The parties acknowledge that the term 'member' may\n", - "be used by Health Net in certain related materials, such as Benefit Program documents covering various products,\n", - "marketing materials, and Health Net Policies. For reference purposes, the term Beneficiary includes the term\n", - "\"member\" wherever used.\n", - "1.2\n", - "Benefit Program. The group agreement, evidence of coverage. certificate of insurance, summary\n", - "plan description or similar documents in effect at the time Covered Services are rendered for lines of business\n", - "offered through Health Net. The Benefit Programs in which Provider participates and terms and conditions such as\n", - "payment rates relating to such Benefit Programs, are set forth in the Addenda to this Agreement.\n", - "1.3\n", - "Coinsurance. That portion of the cost of Covered Services that a Beneficiary is obligated to pay\n", - "under a particular Benefit Program which is calculated as a percentage of the contracted reimbursement rate for such\n", - "services. Coinsurance does not include Copayments or Deductibles.\n", - "California Provider Participation Agreement\n", - "I\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 3\n", - "1.4\n", - "Complete Claim. A Complete Claim means a claim or portion thereof, if separable, including\n", - "attachments and supplemental information or documentation, which provides reasonably relevant information as\n", - "defined by applicable State or federal statutes and regulations, and which is submitted to Health Net or a Payor by\n", - "Provider for payment of Contracted Services that may be processed by Health Net or a Payor without obtaining\n", - "additional information from Provider or from a third party.\n", - "1.5\n", - "Contracted Services. Covered Services that are (i) those services which Provider is licensed to\n", - "provide and which Provider customarily provides to its patients, and (ii) to be provided to a Beneficiary under the\n", - "terms of the applicable Benefit Program in effect at the time such services are rendered or as required by State or\n", - "federal law, and (iii) compensated in accordance with this Agreement except as otherwise may be required by State\n", - "or federal law.\n", - "1.6\n", - "Coordination of Benefits. The allocation of financial responsibility between two or more payors\n", - "of health care services, each with a legal duty to pay for or provide Covered Services to a Beneficiary at the same\n", - "time.\n", - "1.7\n", - "Copayment. That portion of the cost of Covered Services that a Beneficiary is obligated to pay\n", - "under a particular Benefit Program, which generally is a fixed dollar amount and is paid at the time Covered\n", - "Services are rendered. Copayments do not include Coinsurance or Deductibles.\n", - "1.8\n", - "Covered Services. The health care services, equipment and supplies that are covered as\n", - "determined by the Benefit Program and by applicable State and federal law and regulations, including without\n", - "limitation decisions issued as a result of independent medical review conducted under applicable State or federal\n", - "law.\n", - "1.9\n", - "Deductible. The amount of money that a Beneficiary must pay before the Benefit Program pays\n", - "certain benefits for Covered Services. Deductibles do not include Coinsurance or Copayments.\n", - "1.10\n", - "Dispute. The term \"Dispute\". as used in this Agreement, including Sections 7.5 and 7.6, shall\n", - "mean any controversy or disagreement that may arise our of or relate to this Agreement, or the breach thereof,\n", - "whether involving a claim in tort, contract or other applicable area of law.\n", - "1.11\n", - "Emergency. The term \"Emergency\" shall mean a medical condition inanifesting itself by acute\n", - "symptoms of sufficient severity (including severe pain) such that a prudent layperson who passesses average\n", - "knowledge of health and medicine could reasonably expect the absence of immediate medical attention to result in:\n", - "(i) placing the health of the individual (and in the case of a pregnant woman, her health or that of her unborn child)\n", - "in serious jeopardy, or (ii) serious impairment to bodily functions, or (iii) serious dysfunction of any bodily organ\n", - "or\n", - "part.\n", - "1.12\n", - "Excluded Services. Those health care services, equipment and supplies that are determined by\n", - "Health Net or a Payor to be non-Covered Services under the applicable Benefit Program in effect at the time such\n", - "services are rendered and for which Provider may bill the Beneficiary.\n", - "1.13\n", - "Facility(ies). All service locations owned, operated, leased. or subcontracted by Provider at which\n", - "Contracted Services are provided under this Agreement. Provider's service locations as of the date this Agreement\n", - "is executed by the parties are listed on an exhibit to this Agreement.\n", - "1.14\n", - "Health Net. A network of managed health care delivery or indemnity companies, owned,\n", - "controlled, controlling, under common control with. managed or administered in whole or in part now or hereafter,\n", - "by Health Net. Inc., a Delaware Corporation, its successors and assigns.\n", - "1.15\n", - "Health Net Policies. The policies, procedures and programs established by Health Net and\n", - "applicable to Participating Providers in effect at the time Covered Services are rendered, including without\n", - "limitation Health Net's grievance and appeal procedures, provider dispute and/or appeal process, drug formulary\n", - "or\n", - "preferred drug list, fraud detection, recovery procedures, eligibility verification, billing and coding guidelines,\n", - "payment and review policies, anti-discrimination requirements, medical management programs, continuity of care\n", - "California Provider Participation Agreement\n", - "2\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 4\n", - "policies, provider manuals and/or operations manuals. The medical management program includes policies\n", - "regarding topics such as credentialing, ntilization management, quality improvement, catastrophic care management,\n", - "peer review, medical and other record reviews, outcome rate reviews, Prior Authorization, and Referral.\n", - "1.16\n", - "Medically\n", - "Necessary. A Medically Necessary service or supply is one that meets the following\n", - "criteria: it is an otherwise covered category of service, not specifically excluded and is recommended by the treating\n", - "physician and determined by Health Net's Medical Director or physician designee to be: (i) for the purpose of\n", - "treating a medical condition: (ii) the most appropriate supply or level of service, considering potential benefits and\n", - "harm to the Beneficiary; not furnished primarily for the convenience of the Beneficiary or Provider; not required\n", - "solely for custodial, comfort or maintenance reasons; consistent with Health Net Policies and furnished in the most\n", - "appropriate place of service consistent with nationally recognized review criteria and/or guidelines, such as, for\n", - "example, Milliman or Interqual criteria; and (iii) known to be effective and safe in improving health outcomes. For\n", - "new treatments. services or supplies, effectiveness is determined by scientific evidence. For existing treatments,\n", - "services or supplies, effectiveness is determined first by scientific evidence, then by professional standards, then by\n", - "expert opinion. The fact that a physician or other provider may prescribe, order. recommend or approve a service,\n", - "supply or hospitalization does not, in itself. make it Medically Necessary or make it a Covered Service.\n", - "1.17\n", - "Participating Provider. A facility, physician, physician organization, physician group,\n", - "independent practice association, health care provider, supplier, or other organization which has met applicable\n", - "credentialing requirements, if any, and has, or is governed by, an effective written agreement directly with Health\n", - "Net or indirectly through another entity, such as a PPG, to provide Covered Services.\n", - "1.18\n", - "Pavor. Any public or private entity contracted with Health Net which provides, administers,\n", - "funds, insures or is responsible for paying Participating Providers for Covered Services rendered to Beneficiaries\n", - "under a Benefit Program, including self-funded health plaus.\n", - "1.19\n", - "PPG. A participating physician group that has entered into an agreement with Health Net to\n", - "deliver or arrange for the delivery of certain Covered Services to Beneficiaries.\n", - "1.20\n", - "Primary Care Physician (PCP). A Doctor of Medicine (M.D.) or Doctor of Osteopathy (D.O.)\n", - "who: (i) is duly licensed and qualified under the laws of the relevant jurisdiction to render certain Covered Services;\n", - "(ii) is a Participating Provider; and (iii) meets the credentialing standards of Health Net for designation as a PCP;\n", - "and (iv) is responsible for coordinating the provisions of health care services and providing for continuity of care\n", - "and twenty-four (24) hours a day, seven (7) days a week availability to Beneficiaries.\n", - "1.21\n", - "Prior Authorization. The written or electronically issued prior approval by Health Net or its\n", - "designee for the rendition of Covered Services that may be required under a Benefit Program or a Health Net Policy.\n", - "1.22\n", - "Professional Provider. The physicians, allied health professionals and other health care providers\n", - "who contract with Provider, or are employed by Provider, and who have been accepted by Health Net to provide\n", - "Contracted Services to Beneficiaries under the terms and conditions of this Agreement, and billed through\n", - "Provider's federal tax identification number and/or national provider identifier. Professional Providers covered by\n", - "this Agreement as of the date this Agreement is executed by the parties are listed on an exhibit to this Agreement.\n", - "1.23\n", - "Records.\n", - "Books, documents, contracts, subcontracts, and records prepared and/or\n", - "maintained by a party that relate to this Agreement whether in written or electronic format, including without\n", - "limitation medical records, Beneficiary billing and payment records, financial records, policies and procedures, and\n", - "other books and records that may be required by applicable federal and State law\n", - "1.24\n", - "Referral. The written or electronically issued referral of a Beneficiary by a Participating Provider\n", - "to another health care provider that may be required under a Benefit Program or a Health Net Policy prior to the\n", - "rendition of Covered Services, usually for a specified number of visits or type or duration of treatment.\n", - "1.25\n", - "State. The State of California.\n", - "California Provider Participation Agreement\n", - "3\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 5\n", - "1.26\n", - "Surcharge. An additional fee which is charged to a Beneficiary for a Covered Service, but which\n", - "is not approved by the applicable State and federal regulatory authority, and is neither disclosed nor provided for in\n", - "a Benefit Program.\n", - "II.\n", - "DUTIES OF PROVIDER\n", - "2.1 General Obligations. Provider agrees on behalf of itself, and each of its Facilities and Professional\n", - "Providers, as applicable, that during the term of this Agreement and any renewal terms, each of them is:\n", - "2.1.1\n", - "licensed without restriction or limitation by the State to provide Contracted Services to the extent\n", - "required by the State;\n", - "2.1.2\n", - "operating and providing Contracted Services in compliance with applicable local, State, and\n", - "federal laws, rules, regulations and legal standards of care;\n", - "2.1.3\n", - "delivering Contracted Services to Beneficiaries in the same manner and with the same availability,\n", - "as services are delivered to other patients;\n", - "2.1.4\n", - "maintaining such physical plant, equipment, patient service personnel and allied health personnel\n", - "as may be necessary to provide Contracted Services:\n", - "2.1.5\n", - "available to Beneficiaries twenty-four (24) hours per day, seven (7) days per week on an\n", - "Emergency basis.\n", - "2.1.6\n", - "Provider shall notify Health Net in writing, thirty (30) days in advance, of any changes to federal\n", - "tax identification numbers and/or national provider identifier numbers.\n", - "2.2\n", - "Provision of Services. Provider agrees to render Contracted Services to Beneficiaries of Benefit\n", - "Programs under the terms and conditions of this Agreement. Notwithstanding the foregoing, Provider understands\n", - "and agrees that Health Net or a Payor does not have an obligation under this Agreement to assign or refer to\n", - "Provider any minimum amount of Beneficiaries. Health Net has not represented or guaranteed to Provider that any\n", - "Beneficiaries shall receive Covered Services from Provider or that Provider shall participate in all networks of\n", - "Participating Providers offered by or through Health Net.\n", - "Provider acknowledges that Health Net or a Payor shall not be liable for, nor will exercise control or\n", - "direction over, the manner or method by which Provider, Facilities, and/or Professional Providers reuder any\n", - "Covered Services to Beneficiaries under this Agreement.\n", - "2.3\n", - "Verification of Eligibility. Except in an Emergency, Provider shall verify the eligibility of\n", - "Beneficiaries using Health Net's telephonically or electronically available system before providing Contracted\n", - "Services, in compliance with the timeframes and procedures set forth in Health Net Policies.\n", - "2.4\n", - "Non-Discrimination. Provider shall not discriminate against any Beneficiary in the provision of\n", - "Contracted Services hereunder, whether on the basis of the Beneficiary's coverage under a Benefit Program, age,\n", - "sex. marital status, sexual orientation, race, color, religion, ancestry, national origin, disability, handicap, health\n", - "status, source of payment, utilization of medical or mental health services, equipment, pharmaceuticals or supplies,\n", - "or other unlawful basis including, without limitation. the filing by such Beneficiary of any complaint, grievance or\n", - "legal action against Provider, Health Net or Payor. Provider agrees to make reasonable accommodations for\n", - "Beneficiaries with disabilities or handicaps, including but not limited to, providing such auxiliary aides and services\n", - "to Beneficiaries as are reasonable, necessary and appropriate for the proper rendering of Contracted Services at the\n", - "Provider's expense.\n", - "California Provider Participation Agreement\n", - "4\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 6\n", - "2.5\n", - "Professional Providers and Facilities. The following provisions apply when Provider utilizes\n", - "Professional Providers or Facilities to deliver Contracted Services to Beneficiaries:\n", - "2.5.1\n", - "Provider binds its Facilities and Professional Providers, if any, covered by this Agreement, to the\n", - "terms and conditions of this Agreement, to the extent Contracted Services and/or contractual\n", - "provisions are performed by, or apply to, such Facilities and Professional Providers:\n", - "2.5.2\n", - "No new or satellite facility shall be added to this Agreement, or be allowed to deliver Covered\n", - "Services under this Agreement until Health Net has approved such Facility. Health Net reserves\n", - "the right to deny participation under this Agreement to any new or satellite facility without any\n", - "obligation to provide a right to appeal except as may be required by applicable State and federal\n", - "law.\n", - "2.5.3\n", - "In the event Provider desires to add a new Participating Provider, Provider shall notify Health Net\n", - "in writing as soon as possible but no later than sixty (60) days before such proposed addition is to\n", - "become effective with Health Net. Provider agrees that no new Professional Provider shall be\n", - "added to this Agreement, or be allowed to render Covered Services under this Agreement, unless\n", - "and until Health Net has approved the addition of such Professional Provider. Health Net reserves\n", - "the right to deny participation under this Agreement to any proposed new Participating Provider\n", - "without any obligation to provide a right to appeal except as may be required under State or\n", - "federal law.\n", - "Provider additionally shall comply with the terms of Section 2,6 hereof with respect to its Facilities and\n", - "Professional Providers. to the extent Facilities are not owned, and/or Professional Providers are not employed, by\n", - "Provider.\n", - "2.6 Subcontracting The following requirements shall survive termination of this Agreement with respect\n", - "to Contracted Services rendered during the term of the Agreement and apply when Contracted Services are provided\n", - "by a subcontractor, such as a reference laboratory:\n", - "2.6.1\n", - "Provider shall fornish Health Net with copies of its subcontracts within ten (10) days of Health\n", - "Net's written request\n", - "2.6.2\n", - "Every subcontract shall comply with all applicable local, State and federal laws, including\n", - "privacy/confidentiality and medical record accuracy laws, be consistent with the terms and\n", - "conditions of this Agreement, and shall not be used by Provider with respect to Beneficiaries,\n", - "Benefit Programs and/or Contracted Services upon the reasonable request of Health Net.\n", - "2.6.3\n", - "Provider shall not subcontract either directly or indirectly, with any provider that has been\n", - "excluded from participation in the Medicare Advantage Program under Section 1128 or 1128A.[42\n", - "U.S.C. 1320a-7] of the Social Security Act or in the State Medi-Cal program.\n", - "2.6.4\n", - "Each such subcontractor shall meet applicable Health Net credentialing requirements, if any, prior\n", - "to the subcontract becoming effective with respect to Contracted Services.\n", - "2.6.5\n", - "(i) Provider shall be solely responsible to pay the subcourtractor and (ii) Provider shall hold Health\n", - "Net, Payors and Beneficiaries harmless from and against any and all claims which may be made\n", - "by subcontractors in connection with Covered Services provided to Beneficiaries by the\n", - "subcontractor; and (iii) Provider shall require that the subcontractor hold Health Net. Payors, and\n", - "Beneficiaries harmless from and against any and all claims for payment for such services and shall\n", - "not attempt to collect any sums owed by Provider from Health Net or a Beneficiary.\n", - "2.6.6\n", - "Subcontracts shall not restrict the rights and obligations of a healthcare provider to communicate\n", - "freely with Beneficiaries regarding their medical condition and treatment alternatives including\n", - "medication treatment options, regardless of benefit coverage limitations.\n", - "California Provider Participation Agreement\n", - "5\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-03-11\n", - "\n", - "Start of Page No. = 7\n", - "2.6.7\n", - "In the event that any of Provider's subcontracts fail to comply with the requirements set forth\n", - "herein, Health Net or Payor shall not be required to recognize the existence or validity of the\n", - "subcontract with respect to Beneficiaries, Benefit Programs and/or Covered Services. Health Net\n", - "or a Payor shall further have the right, but not the obligation, to directly pay subcontractors\n", - "submitting claims for Contracted Services, and to recoup any compensation otherwise due by\n", - "Health Net or a Payor to Provider pursuant to the terms and conditions of this Agreement.\n", - "Provider shall indemnify and hold harmless Health Net or a Payor for all such payments and\n", - "related costs.\n", - "2.7\n", - "Participating Providers. Except in an Emergency, as otherwise permitted in the applicable\n", - "Benefit Program Requirements, or as otherwise required by applicable federal or State law, if Provider refers a\n", - "Beneficiary for Covered Services, Provider shall refer Beneficiary only to Participating Providers and Provider shall\n", - "coordinate such referrals with Health Net or its designee to facilitate the utilization of the most appropriate\n", - "Participating Provider based upon the Medical Necessary level of care required for the Beneficiary.\n", - "2.8\n", - "Health Net Policies. Provider shall participate in and comply with all Health Net Policies in\n", - "effect on the effective date of this Agreement, and as modified periodically by Health Net in accordance with\n", - "Section 3.2 of this Agreement. Provider hereby acknowledges that it has had the opportunity to review Health Net\n", - "Policies regarding quality improvement and utilization management that pertain in Health Net and Provider's rights\n", - "and obligations under this Agreement at least fifteen (15) business days prior to the date Provider has executed this\n", - "Agreement.\n", - "2.9\n", - "Prior Authorization and Referrals. When either Prior Authorization and/or a Referral is\n", - "required for the rendition of a health care service, the receipt of the required Prior Authorization and/or the required\n", - "Referral, each being separate and distinct requirements, is a prerequisite to payment of Complete Claims for\n", - "Covered Services in addition to confirming eligibility prior to delivering service as required by this Agreement and\n", - "Health Net Policies. Health Net (or its desiguee as applicable) may rescind or modify its Prior Authorization, in a\n", - "manner consistent with Health Net Policies, based on variety of factors, including but not limited to the eligibility of\n", - "the Beneficiary and whether the rendered service is a Covered Service. However, when Health Net or its designee\n", - "issues a Prior Authorization for a specific service under a Benefit Program regulated by the California Department\n", - "of Managed Health Care or the California Department of Insurance, Health Net (or its designee as applicable) shall\n", - "not rescind or modify its Prior Authorization after Provider has rendered the specified and authorized service in\n", - "good faith and pursuant to the terms of the Prior Authorization for any reason, including, but not limited to, Health\n", - "Net's subsequent rescission, cancellation. or modification of the Beneficiary's contract or Health Net's subsequent\n", - "determination that it did not make an accurate determination of the Beneficiary's eligibility; provided, however that\n", - "this section shall not be construed to expand or alter benefits available to a Beneficiary under such Benefit Program.\n", - "2.10\n", - "Notification. Provider shall notify Health Net or 8 Payor and the appropriate PCP or PPG as\n", - "applicable, as soon as possible, but 110 later than 24 hours or by the next business day after a Beneficiary is admitted\n", - "to a Facility.\n", - "2.11\n", - "Credentialing Program. Provider shall submit to Health Net or its designee any applicable\n", - "Credentials Application, which meets minimum requirements of Health Net. Provider or any Professional Provider\n", - "or subcontractor shall not begin performing Provider's obligations under this Agreement, until Provider and/or\n", - "Professional Provider and/or Facility has satisfied applicable credentialing or re-credentialing requirements, if any,\n", - "2.12\n", - "Insurance. Provider shall maintain insurance in amounts and types as required by Health Net\n", - "Policies. Provider agrees to provide Health Net with a Certificate of Insurance from Provider's insurance carrier or\n", - "other mutually agreeable written evidence of such insurance coverage within three (3) days of such request by\n", - "Health Net. Provider also agrees to notify Health Net in writing at least thirty (30) days prior to any termination.\n", - "cancellation or material modification of any policy for all or any portion of the coverage required herein.\n", - "2.13\n", - "Trade names, Trademarks. Directories. Provider shall not use or display the trade names,\n", - "trademarks. or other identifying information of Health Net without Health Net's prior written approval of both form\n", - "and content, which approval shall not be unreasonably withheld. However, this provision shall not prohibit Provider\n", - "from posting a reasonable notice on its website or in its Facilities listing by name those insurance carriers that are\n", - "accepted by Provider so long as the notice lists each name in substantially similar format. Provider shall supply all\n", - "California Provider Participation Agreement\n", - "6\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 8\n", - "printed materials and other information requested by Health Net in connection with the production of provider\n", - "directories within seven (7) days of Health Net's request. Provider agrees that Health Net may list the name, address,\n", - "telephone number and other factual information of Provider, each Facility and Professional Provider. and of\n", - "Provider's subcontractors and their facilities in its provider directories, marketing and informational materials, and\n", - "electronic media.\n", - "2.14\n", - "Non-Solicitation. Neither Provider nor any employee, agent or subcontractor of Provider shall\n", - "solicit or attempt to convince or otherwise persuade any Beneficiary to discontinue participation in any Benefit\n", - "Program or in any other manner interfere with Health Net's contract and/or property rights; provided, however, that\n", - "this provision does not prohibit Provider from posting a reasonable notice on its website or in its Facilities listing by\n", - "name those insurance carriers that are accepted by Provider so long as the notice lists each name in substantially\n", - "similar format. Notwithstanding the foregoing, Health Net in no way restricts Provider from discussing medical\n", - "treatment options with Beneficiaries regardless of Benefit Program coverage options.\n", - "2.15\n", - "Language Assistance Program. Effective January 1, 2009, Provider shall comply with Health\n", - "Net's ongoing language assistance program to ensure Limited English Proficient (\"LEP\") Beneficiaries have\n", - "appropriate access to language assistance while accessing Provider services, pursuant to Health and Safety Code §§\n", - "1367(e)(3), 1367.04 and 1367.07 and Insurance Code §§ 10133.8 and 10133.9.\n", - "2.16\n", - "Additional Rights and Obligations. Any additional rights or obligations of Provider or Health\n", - "Net shall be set forth in the Addenda to this Agreement.\n", - "III.\n", - "DUTIES OF HEALTHNET\n", - "3.1\n", - "Payment. Health Net shall, or Health Net shall require Payor to, make payment to Provider for\n", - "Contracted Services in accordance with Article IV and the applicable addenda, schedules and exhibits of this\n", - "Agreement.\n", - "3.2\n", - "Health Net Policies. Health Net Policies are set forth in references and forms available to\n", - "Provider through the Provider Section, California Region, of Health Net's website at \"www.healthnet.com\" or by\n", - "other means which Health Net will communicate to Provider periodically. Health Net Policies in existence as of the\n", - "effective date of this Agreement are hereby incorporated into this Agreement by reference. Notwithstanding the\n", - "foregoing and/or any other provision of this Agreement, the parties agree that a formal amendment to this\n", - "Agreement shall not be required to effectuate modifications to Health Net Polices. Modifications to Health Net\n", - "Policies may be made periodically as determined by Health Net in accordance with the procedures set forth in\n", - "applicable State law (including without limitation the California Health Care Providers' Bill of Rights). Such\n", - "modifications shall be deemed incorporated in this Agreement as of the effective date of such modification unless\n", - "otherwise mutually agreed by the parties in writing at the time of the modification in accordance with applicable\n", - "State law (including without limitation the California Health Care Providers' Bill of Rights).\n", - "3.3\n", - "Insurance. Health Net shall maintain appropriate insurance programs or policies including bodily\n", - "injury and personal injury coverage, which includes persons serving on Health Net committees as insured by\n", - "definition. In the event that a policy or program is terminated or the coverage of committee persons is materially\n", - "changed, Health Net shall so notify Provider.\n", - "3.4\n", - "Reporting to Regulators. Health Net and/or Payor shall accept sole responsibility for filing\n", - "reports, obtaining approvais and complying with applicable laws and regulations of State, federal and other\n", - "regulatory agencies having jurisdiction over Health Net and/or Payor; provided, however, that Provider agrees to\n", - "cooperate in providing Health Net and/or Payor with any information and assistance reasonably required in\n", - "connection therewith, including without limitation, permitting the regulatory agencies to conduct periodic site\n", - "evaluations of Provider, Facilities, Professional Providers and any of their equipment, operations, and billing and\n", - "medical records of Beneficiaries. Such records shall be located in the State.\n", - "California Provider Participation Agreement\n", - "7\n", - "Fee-For-Service Direct Network Templare\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 9\n", - "3.5\n", - "Access To This Agreement.\n", - "3.5.1\n", - "Access by Health Net. As of the effective date of this Agreement, the following Health Net\n", - "companies may at their option access this Agreement: Health Net of California, Inc., Health Net\n", - "Life Insurance Company, Health Net Community Solutions, Inc., Health Net of Arizona, Inc.,\n", - "Health Net Health Plan of Oregon, Inc., Health Net of Pennsylvania, LLC, Health Net of the\n", - "Northeast, Inc., Health Net of Connecticut, Inc., Health Net of New York, Inc., Health Net of New\n", - "Jersey, Inc., Health Net One Payment Services, Inc., Health Net Insurance of New York, Inc.,\n", - "Health Net Insurance Services, Inc., Health Net Federal Services, LLC, and Managed Health\n", - "Network. Inc. Health Net may periodically modify the Health Net companies which may access\n", - "this Agreement. To the extent Health Net allows a Health Net company to access this Agreement,\n", - "Health Net binds such company to the terms and conditions of this Agreement.\n", - "3.5.2\n", - "Access by Payors. To the extent Health Net allows a self-funded Payor to access this Agreement,\n", - "Health Net has obligated such self-funded Payor to fund a claims payment account in a sufficient\n", - "and timely manner to pay claims for services provided by health care providers like Provider. In\n", - "the event a self-funded Payor accessing this Agreement fails to sufficiently and timely fund a\n", - "claims payment account to the material detriment of Provider. Provider may terminate this\n", - "Agreement as to such self-funded Payor in accordance with Section 5.3 hereof. and,\n", - "norwithstanding the provisions of Section 4.6 of this Agreement, take legal action against the self-\n", - "funded Payor and/or Beneficiary as may be permitted by law. Additional information regarding\n", - "Payors and conditions for accessing this Agreement is set forth in Addendum A of this\n", - "Agreement.\n", - "3.6\n", - "Notification to Beneficiaries: Termination of a Professional Provider. Health Net shall notify\n", - "Beneficiaries who are affected by the termination of a specialist. Professional Provider in writing, immediately upon\n", - "notification of such termination but no later than thirty (30) calendar days prior to the effective date of such\n", - "specialist's termination. Applicable to Commercial HMO Benefit Programs only: For Beneficiaries covered by an\n", - "HMO Benefit Program, Health Net shall be required to issue a notice regarding the termination of a specialist's\n", - "contract that contains the following language in not less than eight-point type: \"If you have been receiving care from\n", - "a health care Provider, you may have a right to keep your Provider for a designated time period. Please contact\n", - "Health Net's customer service department, and if you have further questions, you are encouraged to\n", - "contact the Department of Managed Health Care, which protects HMO customers. by telephone at its\n", - "toll-free number, 1-888-HMO-2219, or at a TDD number for the hearing impaired at 1-877-688-9891. or online at\n", - "www.hmohelp.ca.gov.\"\n", - "IV.\n", - "FINANCIAL OBLIGATIONS. The terms of this Article IV shall survive termination of this Agreement\n", - "with respect to Covered Services rendered during the term of this Agreement:\n", - "4.1\n", - "Payment Rates. Health Net shall pay (or shall require Payor to pay). and Provider shall accept as\n", - "payment in full for Contracted Services, the rates payable by Health Net or Payor under the terms and conditions of\n", - "this Agreement (including the payment conditions, chargemaster and other provisions set forth in the applicable\n", - "addenda, schedules and exhibits to this Agreement), less Copayments, Coinsurance and Deductibles payable by\n", - "Beneficiaries in accordance with the applicable Benefit Program or as otherwise permitted by the section of this\n", - "Agreement covering Third Party Lien Recoveries. Any overpayment, inaccurate payment or other payment error\n", - "made by Health Net or Payor shall not be deemed or construed or otherwise operate to change the payment terms or\n", - "rates provided for under this Agreement.\n", - "4.2\n", - "Billing and Payment.\n", - "4.21\n", - "Billing. Provider shall submit to Health Net/Payor, via Health Net's/Payor's electronic claims\n", - "submission program or hardcopy as determined by Health Net/Payor, Complete Claims within one\n", - "hundred twenty (120) days after Provider renders Contracted Services unless Provider\n", - "demonstrates good cause pursuant to applicable State law. Where Health Net and/or Payor is the\n", - "California Provider Participation Agreement\n", - "8\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 10\n", - "secondary payor under Coordination of Benefits, Provider shall submit Complete Claims for\n", - "Covered Services accompanied by the explanation of benefits (EOB) or explanation of payment\n", - "(EOP) from the primary payor to Health Net or a Payor within one hundred twenty (120) days of\n", - "the date of the EOB/EOP. If Provider fails to comply with the timely claims submission/filing\n", - "requirements set forth herein, Health Net shall have no obligation to pay for such claims, and\n", - "Provider shall be prohibited from billing the Beneficiary as set forth in Section 4.6 hereof.\n", - "Provider agrees that Health Net or a Payor shall have the right to determine the accuracy of all\n", - "Complete Claims submitted to it prior to payment. including verification of diagnostic codes,\n", - "DRG assignment, and whether Provider has delivered the Covered Service in good faith and\n", - "pursuant to the terms of an applicable Prior Authorization.\n", - "4.2.2\n", - "Payment. Health Net or Payor, as applicable, shall make payment on each of Provider's timely-\n", - "submitted Complete Claims in accordance with this Agreement and pursuant to the timeframes,\n", - "procedures and other requirements of applicable State and federal law, including without\n", - "limitation the calculation and payment of interest on overdue payments. Payment of interest plus\n", - "the amount of any Complete Claim payment deficiency shall be Provider's sole measure of\n", - "damages (i.e., claims for consequential or incidental damages do not apply) for failure of Health\n", - "Net or Payor to make timely and accurate payments. In no event shall Health Net be under any\n", - "obligation to pay Provider for any claim or expense, which is the responsibility of a self-funded\n", - "Payor.\n", - "4.2.3\n", - "Appeals. In addition to the dispute resolution and arbitration rights described in Section 7.5 and\n", - "Section 7.6 herein, Provider may dispute any Health Net action that adjusts, denies. or contests a\n", - "claim. billing practice, or other contractual provision so long as Provider submits a written dispute\n", - "to the Health Net Provider Appeals Unit. Unless Provider demonstrates good cause pursuant to\n", - "applicable State or federal law, Health Net or Payor shall not grant Provider reconsideration or\n", - "appeal of a claims payment for Covered Services that exceed three hundred sixty five (365) days\n", - "of Health Net's action or in the case of inaction, within three hundred sixty five (365) days after\n", - "the time for contesting or denying claims (as defined in applicable State or federal law) has\n", - "expired. Appeals shall be submitted by Provider in accordance with the procedures, and to the\n", - "address for Health Net's Provider Appeals Unit, listed in Health Net Policies. If Provider fails to\n", - "comply with the timely appeals submission/filing requirements set forth herein, Health Net shall\n", - "have no obligation to pay for such claims except as otherwise required by applicable State and\n", - "federal law, and Provider shall be prohibited from billing the Beneficiary as set forth in Section\n", - "4.6 hereof. Provider and Health Net agree to comply with all timeliness and procedural\n", - "requirements for submitting and responding to disputes submitted to Health Net's Provider\n", - "Appeals Unit as set forth in Health Net Policies.\n", - "4.3\n", - "Recompment of Overpayments; Right of Offset.\n", - "4.3.1\n", - "Provider shall inform Health Net of any overpayment made to Provider and shall return any such\n", - "overpayment to Health Net within thirty (30) business days from the date Provider first becomes\n", - "aware of any such overpayment.\n", - "4.3.2\n", - "In the event Health Net determines that it has overpaid a claim, either in connection with an audit\n", - "or otherwise, Health Net shall notify Provider in writing through a separate overpayment notice\n", - "clearly identifying the claim, the name of the Beneficiary, the date of service and explanation of\n", - "the basis upon which Health Net or Payor believes the amount paid on the claim was in excess of\n", - "the amount due, including any interest and penalties that may be due on the claim. Such\n", - "overpayment notice shall be issued within (i) three hundred sixty-five (365) days of the date of\n", - "payment on the overpaid amount for claims arising from Benefit Programs regulated by the\n", - "California Department of Managed Health Care or the California Department of Insurance, or\n", - "within (ii) three (3) years from the date of payment on the overpaid amount for claims arising from\n", - "other types of Benefit Programs that are not regulated by the California Department of Managed\n", - "Health Care or the California Department of Insurance, or (iii) at any time. in the event of fraud\n", - "California Provider Participation Agreement\n", - "9\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 11\n", - "and/or misrepresentation. Such notice shall be sent to Provider's address of record with Health\n", - "Net for the receipt of claim related correspondence and payments unless Provider informs Health\n", - "Net in writing of an alternative address to which such notices are to be sent at least thirty (30) days\n", - "in advance of the address change.\n", - "4.3.3\n", - "If Provider does not contest Health Net's overpayment notice, Provider shall reimburse Health Net\n", - "or Payor within thirty (30) business days from the date Provider receives the overpayment notice.\n", - "If Provider fails to reimburse Health Net or Payor within those thirty (30) business days, then,\n", - "beginning on the first calendar day after the expiration of this thirty (30) business day time period,\n", - "Health Net shall commence offsetting, as set forth herein and interest shall accrue on any and all\n", - "unpaid amounts at the rate of ten percent (10%) per annum.\n", - "4.3.4\n", - "In the event, Provider wishes to contest the overpayment notice, it must do so within thirty (30)\n", - "business days from the date Provider receives the overpayment notice, by sending to Health Net's\n", - "Provider Appeals Unit (at the address listed in Health Net Policies) a written appeal clearly stating\n", - "the basis upon which Provider believes that the claim was not overpaid. Health Net shall review\n", - "and make a decision with respect to Provider's appeal, and shall notify Provider of its decision in\n", - "writing within forty-five (45) business days from the date Health Net receives Provider's written\n", - "appeal. In the event Health Net denies Provider's appeal and upholds Health Net's determination\n", - "that an overpayment has been made, Provider shall reimburse Health Net or Payor for the\n", - "overpayment within thirty (30) business days from the date it receives the written notice of Health\n", - "Net's deuial of Provider's written appeal. If Provider fails to reimburse Health Net or Payor within\n", - "those thirty (30) business days, then beginning on the first calendar day after the expiration of this\n", - "thirty (30) business day time period, Health Net may commence offsetting as set forth herein, and\n", - "interest shall accrue on any and all unpaid amounts at the rate of ten percent (10%) per annum\n", - "4.3.5\n", - "If Health Net or Payor exercises offset rights hereunder against Provider's current claims\n", - "payments, Health Net or Payor shall give Provider a detailed written explanation identifying the\n", - "specific overpayments that have been offset against the specific current claims payments.\n", - "4.3.6\n", - "If Provider desires to continue to contest the overpayment, it shall do so by following the dispute\n", - "resolution process set forth in Sections 7.5 and 7.6 of this Agreement.\n", - "4.4\n", - "Eligibility.\n", - "The parties acknowledge that verification of eligibility by Health Net is based on\n", - "information available to Health Net from its customers on the date Provider seeks verification. Health Net shall use\n", - "reasonable efforts to discourage its customers from retroactively canceling or adding Beneficiaries to a Benefit\n", - "Program and encourage its customers to timely and accurately provide eligibility information.\n", - "Tu the event Contracted Services are provided to an individual who is not a Beneficiary, based on an\n", - "erroneous or delayed enrollment/eligibility list the following shall apply: (i) when the individual is enrolled in a\n", - "substitute or replacement health care service or insurance plan which is obligated under applicable law to make\n", - "payment to Provider for services delivered to the individual, Provider shall seek payment from the substimute or\n", - "replacement carrier; and (ii) when the individual does not have substitute or replacement coverage, Health Net shall\n", - "pay Provider for Contracted Services delivered to the individual by Provider prior to the time Provider received\n", - "notice of that individual's ineligibility pursuant to the terms and conditions of this Agreement, provided, however,\n", - "for those Benefit Programs that are not regulated by the California Department of Managed Health Care or the\n", - "California Department of Insurance, as an additional prerequisite for payment pursuant to this Section 4.4(ii),\n", - "Provider shall submit to Health Net evidence that Provider has unsuccessfully sought payment through two billing\n", - "cycles for all or a portion of such charges from the patient or the person having legal responsibility for the patient, or\n", - "from the entity having financial responsibility for such payment. In the event Health Net pays Provider pursuant to\n", - "this Section 4.4, Provider shall have no further right and shall not attempt to collect any additional payment from the\n", - "individual for said services (except for applicable Copayments, Coinsurance and Deductibles) and Provider hereby\n", - "assigns and transfers all legal rights of collection and Coordination of Benefits for services to Health Net.\n", - "4.5\n", - "Collection of Copayments, Coinsurance and Deductibles. Provider shall collect all\n", - "Copayments, Coinsurance and Deductibles due from Beneficiaries, and shall not waive or fail to pursue such\n", - "California Provider Participation Agreement\n", - "10\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 12\n", - "collection except when otherwise permitted through Provider's established patient financial assistance program.\n", - "Provider shall not charge Beneficiary any fees or Surcharges for Contracted Services rendered pursuant to this\n", - "Agreement (except for Copayments, Coinsurance and Deductibles). In addition, Provider shall not collect a sales,\n", - "use or other applicable tax from Beneficiaries for the sale or delivery of Contracted Services unless required by\n", - "applicable State or federal law. If Health Net or any Payor receives notice of any attempt to collect or the receipt of\n", - "any inappropriate additional charges. including without limitation Surcharges, Health Net or Payor shall take\n", - "appropriate action. Provider shall cooperate with Health Net or such Payor to investigate such allegations, and shall\n", - "promptly refund to the party who made the payment, any payment reasonably determined to be improper by Health\n", - "Net or a Payor.\n", - "4.6\n", - "Beneficiary Held Harmless. Provider agrees that in no event, including, but not limited to. non-\n", - "payment by Health Net or a Payor, insolvency of Health Net or a Payor, or breach of this Agreement, shall Provider\n", - "bill, charge, collect a deposit from, seek compensation, remuneration. or reimbursement from. or have any recourse\n", - "against Beneficiaries or persons acting on their behalf other than Health Net or a. Payor for Contracted Services\n", - "provided pursuant to this Agreement except for Copayments, Coinsurance, Deductibles, Excluded Services or\n", - "permitted third party liens under this Agreement and as permitted under Section 3.5.2 hereof. This provision shall\n", - "not prohibit collection of Copayments, Coinsurance or Deductibles or Excluded Services or permitted third party\n", - "liens under this Agreement made in accordance with applicable Benefit Program Requirements. Provider agrees\n", - "that: (i) this provision shall survive the termination of this Agreement regardless of the cause giving rise to\n", - "termination and shall be construed to be for the benefit of Beneficiaries; and (ii) this provision supersedes any oral\n", - "or written contrary agreement now existing or hereafter entered into between Provider and Beneficiaries or persons\n", - "acting on their behalf. Provider agrees to (iii) address any and all concerns it has with claims payment through\n", - "Health Net's provider appeal process pursuant to Health Net Policies and (iv) give the Beneficiary and Health Net\n", - "confirmation that Provider has rescinded the collection notice and taken any other actions necessary to clear the\n", - "Beneficiary's credit record of the collection matter.\n", - "4.7\n", - "Conditions for Compensation for Excluded Services. Provider may bill a Beneficiary for\n", - "Excluded Services rendered by Provider to such Beneficiary only if the Beneficiary is notified in advance that the\n", - "services to be provided are not Covered Services under the Beneficiary's Benefit Program, and the Beneficiary\n", - "requests in writing that Provider reuder the Excluded Services, prior to Provider's rendition of such services.\n", - "4.8\n", - "Coordination of Benefits. Provider agrees to conduct Coordination of Benefits in accordance\n", - "with federal and State laws and regulations and Health Net Policies (\"Coordination of Benefit Rules\"). including but\n", - "not limited to, the prompt notification to Health Net or a Payor of any third party entity who may be responsible for\n", - "payment and collection of Copayments. Provider shall not bill Beneficiaries for any portion of Contracted Services\n", - "not paid by the primary carrier when Health Net or a. Payor is the secondary carrier, but shall seek payment from\n", - "Health Net/Payor. When Health Net or a Payor is secondary under the Coordination of Benefits Rules, Health Net or\n", - "a Payor shall pay Provider an amount up to Beneficiary's primary plan's copayment, coinsurance or deductibles as\n", - "applicable, where that payment does not exceed Health Net's contracted rate under this Agreement. In the event that\n", - "Medicare is the primary carrier and Health Net is secondary, Health Net shall pay Provider only up to Medicare's\n", - "allowable amount and/or the Beneficiary's Copayment, Coinsurance or Deductibles as applicable. Such recoveries\n", - "shall be performed in accordance with the applicable Benefit Program Requirements and Health Net Policies.\n", - "4.9\n", - "Third Party Recoveries: Workers Compensation. In the event Provider provides Covered\n", - "Services to Health Net Beneficiaries for injuries resulting from the acts of third parties, or resulting from work\n", - "related injuries, Provider shall have the right to recover from any settlement, award, or recovery from any\n", - "responsible third-party the reasonable and necessary charges for such Covered Services to the extent permitted by\n", - "applicable law. Provider shall notify Health Net of any such recovery and shall provide Health Net with an\n", - "accounting of all such sums recovered. In the event Provider has recovered sums from a third party, Provider agrees\n", - "to pay such recovered sums to Health Net up to the fee-for-service amounts that Health Net paid to Provider, to the\n", - "extent that Health Net has not recovered such amounts from its own third party recovery efforts. Provider shall pay\n", - "these amounts to Health Net within sixty (60) days of Health Net informing Provider of the amounts Health Net\n", - "recovered from its own third party recovery efforts, if any This section does not obligate, nor does it prohibit, either\n", - "Health Net or Provider to undertake such third party recovery efforts.\n", - "California Provider Participation Agreement\n", - "11\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 13\n", - "4.10\n", - "Reciprocity.\n", - "Provider agrees that Health Net may allow the payment rates set forth in this\n", - "Agreement to be used by Participating Providers and PPGs who may periodically be responsible for compensating\n", - "Provider for Covered Services reudered by Provider to a Beneficiary.\n", - "V.\n", - "TERM AND TERMINATION\n", - "5.1\n", - "Term. The term of this Agreement shall commence on the Effective Date and shall continue for a\n", - "period of one (T) year thereafter (the \"Ininal Term\"). Either party may terminate this Agreement effective as of the\n", - "end of the Initial Term by providing at least one hundred twenty (120) days prior written notice to the other party.\n", - "This Agreement shall automatically renew for successive one (1) year periods (the \"Renewal Terins\").\n", - "5.2\n", - "Immediate Termination. Either party may terminate this Agreement immediately upon notice to\n", - "the other party, in the event of: (i) a party's violation of material law, rule or regulation; (ii) a party's failure to\n", - "maintain the insurance coverage specified hereunder; or (iii) a felony conviction related to the medical and/or\n", - "financial practices of a party. Health Net may terminate this Agreement immediately upon notice to Provider in the\n", - "event of (iv) action taken by a State or federal regulator that results in a material restriction upon Provider's ability\n", - "to perform Covered Services, including if applicable, operate a Facility or reportable discipline against Provider's\n", - "license, accreditation, or certification: (v) Health Net's determination that the health, safety or welfare of any\n", - "Beneficiary may be in jeopardy if this Agreement is not terminated; (vi) any material adverse finding as a result of a\n", - "lawsuit or claim, related to the medical and/or financial practices of Provider.\n", - "5.3\n", - "Termination Due to Material Breach. In the event either party believes the other party has\n", - "committed a material breach of this Agreement, the non-breaching party shall send the other party a written Notice\n", - "Of Breach and Demand to Cure (\"Notice\"). Without limiting either party's other termination rights under this\n", - "Article V. in the event that either party fails to cure a material breach of this Agreement within thirty (30) days of\n", - "receipt of the Notice from the other party (the \"Cure Period\"). the non-defaulting party may terminate this\n", - "Agreement by providing the defaulting party thirty (30) days prior written notice of termination. The non-defaulting\n", - "party may exercise this termination option, if at all; within thirty (30) days of the date the Cure Period expires. If the\n", - "breach is cured within the Cure Period, or if the breach is one, which cannot reasonably be corrected within the Cure\n", - "Period, and the defaulting party is making substantial and diligent progress toward correction during the Cure Period\n", - "to the reasonable satisfaction of the non-defaulting party, this Agreement shall remain in full force and effect. The\n", - "provisions of this Section 5.3 shall not apply to Health Net claims payment timeliness issues which are governed by\n", - "Article IV of this Agreement, unless and until the parties have completed the dispute resolution process set forth in\n", - "Sections 7.5 and 7.6 of this Agreement, and the dispute relates to habitual, chronic and material claims payment\n", - "timeliness issues. In the event a Payor fails in its obligations under the terms of this Agreement to niake Complete\n", - "Claim payments to Provider when due. Provider may terminate the specific delinquent Payor without terminating\n", - "this Agreement in its entirety, but only after all of the following conditions have been met: (i) Payor has failed to\n", - "make a payment to Provider within the applicable time frame set forth in this Agreement; (ii) Provider provides\n", - "written notice to Payor that such payment has not been made: (iii) Payor fails to remit payment to Provider within\n", - "ten (10) days following Payor's receipt of Provider's written notice: (iv) Provider has made a good-faith attempt to\n", - "meet with Health Net and Payor to resolve the payment issue(s).\n", - "5.4\n", - "Termination Upon Notice. Either party may terminate this Agreement during a Renewal Term\n", - "for any reason or no reason upon one hundred twenty (120) days prior written notice to the other party. In the event\n", - "that either party provides the other party with such notice, and following Health Net's completion of any applicable\n", - "regulatory filing requirements, Health Net may, at its option, begin to transition Beneficiaries under this Agreement\n", - "to another Participating Provider.\n", - "5.5\n", - "Information to Beneficiaries. The parties each agree not to disparage the other in any information\n", - "supplied by either party to Beneficiaries or other third parties in connection with any expiration, termination or non-\n", - "renewal of this Agreement. Health Net shall assume sole responsibility for notifying Beneficiaries, and Health Net\n", - "may commence transferring Beneficiaries to alternate providers, prior to the effective date of any expiration,\n", - "termination or non-renewal of this Agreement in accordance with State and federal law. If Beneficiaries seek\n", - "services or Participating Providers order tests or seek services from Provider after the effective date of any\n", - "expiration, termination or non-renewal, Provider shall inform such Beneficiaries and Participating Providers only\n", - "California Provider Participation Agreement\n", - "12\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 14\n", - "that Provider no longer has an agreement with Health Net to render Covered Services and shall direct them to Health\n", - "Net's customer service department. Provider shall not otherwise initiate communications with Beneficiaries or other\n", - "third parties, verbally or in writing, concerning the expiration, termination or non-renewal of this Agreement and\n", - "Provider's participation in Health Net's Participating Provider network, unless the parties have agreed in writing to\n", - "the content of such communications in the context of 21 mutually agreed communication plan. Nothing in this\n", - "provision is intended nor shall it be construed to prohibit or restrict Provider, Professional Provider, or other\n", - "Participating Providers from (i) disclosing to any Beneficiary information regarding treatment options available, the\n", - "risks, benefits and alternatives thereto, or (ii) disclosing to any Beneficiary the decision or process of Health Net or a\n", - "Payor to Prior Authorize or deny benefits under a Benefit Program. or (iii) posting a reasonable notice on Provider\n", - "website or in Provider's Facilities listing by name those insurance carriers that are accepted by Provider, provided\n", - "that the notice lists each name in substantially similar format. The terms of this Section 5.5 shall survive termination\n", - "of this Agreement.\n", - "5.6\n", - "Effect of Termination. In the event that a Beneficiary is receiving Contracted Services on the date\n", - "this Agreement expires, non-renews, and/or terminates, upon the request of Beneficiary and Health Net, Provider shall\n", - "continue to provide Contracted Services to the Beneficiary until the later of: (i) treatment is completed; (ii) the\n", - "Beneficiary is discharged if Provider is an inpatient facility: (iii) the Beneficiary is assigned to another Participating\n", - "Provider; or (iv) the anniversary date of the Beneficiary's Benefit Program. Provider's compensation for such\n", - "Contracted Services shall be at the rates contained in the applicable Addendum hereto. If Provider's services are\n", - "continued beyond the expiration, non-renewal. and/or termination of this Agreement, Provider shall be subject to the\n", - "same contractual terms and conditions that were imposed on Provider prior to the expiration/non-\n", - "renewal/ternination, including, but not limited to, credentialing, hospital privileging, utilization review, peer review,\n", - "and quality assurance requirements.\n", - "VI.\n", - "RECORDS, AUDITS AND REGULATORY REQUIREMENTS\n", - "6.1\n", - "Medical and Other Records. Provider shall prepare and maintain Records in accordance with the\n", - "general standards applicable to such Record-keeping and in compliance with all applicable federal and State\n", - "confidentiality and privacy laws. Provider shall maintain such Records for at least ten (10) years after the reudition\n", - "of Contracted Services, and Records of a minor child shall be kept for at least three (3) years after the minor has\n", - "reached the age of cighteen (18), but in no event less than ten (10) years after the rendition of Contracted Services.\n", - "Additionally, Provider shall maintain such Records as may be necessary and reasonably requested by Health Net to\n", - "comply with applicable federal and State law, and accrediting agency reporting requirements, rules and regulations.\n", - "Provider shall comply with and require Professional Providers to comply with all confidentiality and Beneficiary\n", - "records accuracy requirements. Provider's Records shall be and remain the property of Provider.\n", - "6.2\n", - "Access to Records and Audits by Regulatory Agencies and Accreditation Agencies. Subject\n", - "only to applicable State and federal confidentiality or privacy laws, Provider shall permit designated representatives\n", - "of local, State, and federal regulatory agencies having jurisdiction over Health Net or Payor (\"Regulatory\n", - "Agencies\") and designated representatives of accreditation agencies having jurisdiction over Health Net or Payor\n", - "(\"Accreditation Agencies\"), access to Provider's Records, at Provider's place of business in this State during normal\n", - "business hours, in order to audit, inspect and review and make copies of such Records. Such Regulatory Agencies\n", - "shall include. but not be limited to, the State Department of Health Care Services, the State Department of\n", - "Insurance, the State Department of Managed Healthcare, the United States Justice Department, CMS and the United\n", - "States Department of Health and Human Services and any of their representatives. Such Accreditation Agencies\n", - "shall include, but not be limited to, the National Committee on Quality Assurance (NCQA). When requested by\n", - "Regulatory Agencies aud/or Accreditation Agencies, Provider shall produce copies of any such Records at no\n", - "charge. Additionally, Provider agrees to permit Regulatory Agencies and Accreditation Agencies or their\n", - "representatives, to conduct audits, site evaluations and inspections of Provider's Records, offices and service\n", - "locations. Provider shall make available the access, audits, evaluations, inspections, records, and/or copies of\n", - "Records required by this Section, at no cost to Health Net, Payor, Regulatory Agencies and/or Accreditation\n", - "Agencies, and within a reasonable time period, but not more than five (5) days after the request is submitted to\n", - "Provider.\n", - "California Provider Participation Agreement\n", - "13\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 15\n", - "6.3\n", - "Access to Records and Audits by Health Net. Subject only to applicable State and federal\n", - "confidentiality or privacy laws, Provider shall permit Health Net or its designated representative access to\n", - "Provider's Records, at Provider's place of business in this State during normal business hours, in order to audit.\n", - "inspect, review and duplicate such Records. Access to Records for the purpose of an audit shall be scheduled at\n", - "mutually agreed upon times, npon at least thirty (30) business days prior written notice by Health Net or its\n", - "designated representative, but not more than sixty (60) days following such written notice. Provider shall attend an\n", - "exit interview upon completion of the audit for the purpose of obtaining a mutually agreed upon reconciliation of\n", - "the initial audit findings. Such exit interview shall be conducted al a mutually agreeable time at Provider's place of\n", - "business in this State during normal business hours upon at least ten (10) days prior written notice by Health Net or\n", - "its designated representative, but not more than thirty (30) days following such written notice. In the event\n", - "Provider fails to attend the scheduled exit interview. Provider shall be deemed to have accepted the andit findings.\n", - "Provider may be reimbursed reasonable fees associated with the retrieval of Provider's Records and or duplication\n", - "and preparation of requested Provider Records pursuant to applicable State law, including California Health and\n", - "Safety code Section 123110. Audit findings relating to any audit of claims shall include adjustment for late\n", - "charges, overcharges and undercharges. Any such adjustments shall be the net amounts as reflected in the audit\n", - "findings. Any payments owed by one party to the other as the result of an audit shall be paid within thirty (30) days\n", - "of the exit interview for such audit.\n", - "6.4\n", - "Continuing Obligation. The obligations of Provider under this Article VI shall not be terminated\n", - "upon termination of this Agreement, whether by rescission, non-renewal or otherwise. After such termination of this\n", - "Agreement, Health Net, Payors and Regulatory Agencies shall continue to have access to Provider's Records as\n", - "necessary to fulfill the requirements of this Agreement and to comply with all applicable laws, rules and regulations.\n", - "6.5\n", - "Regulatory Compliance. Each party agrees to comply with all applicable local, State, and federal\n", - "laws, rules and regulations. now or hereafter in effect, regarding the performance of the party's obligations\n", - "hereunder, including without limitation, laws or regulations governing Beneficiary confidentiality. privacy, appeal\n", - "and dispute resolution procedures to the extent that they directly or indirectly affect Provider, a Beneficiary, Health\n", - "Net, or Payor, and bear upon the subject matter of this Agreement. If Health Net is sanctioned by any Regulatory\n", - "Agency for non-compliance that is caused by Provider, Provider shall compensate Health Net for amounts tied to\n", - "this sanction incurred by Health Net including Health Net's costs of defense and fees.\n", - "VII.\n", - "GENERAL PROVISIONS\n", - "7.1\n", - "Amendments. This Agreement may be amended by mutual written agreement of the parties.\n", - "Notwithstanding the foregoing, amendments required to comply with State or federal laws or regulations,\n", - "requirements of Regulatory Agencies, or requirements of Accreditation Agencies, shall not require the consent of\n", - "Provider or Health Net and shall be effective immediately OIL the effective date of the requirement. The parties\n", - "acknowledge that changes to Health Net Policies that may affect a party's rights or obligations under this Agreement\n", - "are addressed in Section 3.2 hereof.\n", - "7.2\n", - "Separate Obligations. The rights and obligations of Health Net under this Agreement shall\n", - "apply to each Health Net company and/or Payor accessing this Agreement only to the extent such Health Net\n", - "company and/or Payor has accessed this Agreement with respect to the Benefit Programs of such Health Net\n", - "company or Payor. Health Net or Payor shall not be responsible for the obligations of any other Health Net\n", - "company or Payor under this Agreement with respect to the other's Benefit Programs. The terms of this Section 7.2\n", - "shall survive termination of this Agreement.\n", - "7.3\n", - "Assignment. Neither this Agreement, nor any of Provider's rights or obligations hereunder, is\n", - "assignable by Provider without the prior written consent of Health Net which consent shall not be unreasonably\n", - "withheld. Health Nel expressly reserves the right to assign, delegate or transfer any or all of its rights, obligations or\n", - "privileges under this Agreement to an entity controlling, controlled by, or under common control with Health Net,\n", - "Inc.,\n", - "California Provider Participation Agreement\n", - "14\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 16\n", - "7.4\n", - "Confidentiality. Health Net. Payors and Provider agree to hold Beneficiary health information\n", - "and records, the terms of this Agreement, and all confidential or proprietary information or trade secrets of each\n", - "other, in trust and confidence. Health Net. Payors and Provider each agree to keep strictly confidential all terms,\n", - "including without limitation compensation rates, set forth in this Agreement and its Addenda. except that this\n", - "provision does not preclude disclosure by Health Net to potential customers, Beneficiaries, Regulatory Agencies and\n", - "Accreditation Agencies of the method of compensation used by Health Net with respect to Health Net's\n", - "Participating Provider networks, e.g., fee-for-service, capitation, shared risk pool, DRG or per diem. Health Net,\n", - "Payors and Provider agree that such information shall be used only for the purposes contemplated herein, and not for\n", - "any other purpose. Health Net. Payors and Provider Agree that nothing in this Agreement shall be construed as a\n", - "limitation of (i) Provider's rights or obligations to discuss with the Beneficiaries matters pertaining to the\n", - "Beneficiaries health regardless of Benefit Program coverage options or (ii) Health Net's rights or obligations with\n", - "respect to subcontractors, including without limitation delegated providers, or (iii) disclosures to counsel or a\n", - "consultant of a party for the purpose of monitoring regulatory compliance or rendering legal advice pertaining only\n", - "to this Agreement or disclosures to internal or independent auditors of a party for audit purposes pertaining to this\n", - "Agreement, provided that in either case the counsel or consultant agrees in writing to comply with the provisions of\n", - "this Section 7.4 and agrees that the terms of this Agreement may not be disclosed to any other person or entity or\n", - "used in any manner whatsoever in connection with any other agreement involving Health Net. The terms of this\n", - "Section 7,4 shall survive termination of this Agreement.\n", - "7.5\n", - "Provider Dispute Resolution Procedure. The parties agree to use the dispute resolution process\n", - "set forth in this Section 7.5, and binding arbitration as described in Section 7.6, as the final steps in resolving any\n", - "Dispute. The parties each understand and agree that any and all Health Net internal appeals processes (including\n", - "without limitation as set forth in Section 4.2.3 hereof) must be properly pursued and exhausted before engaging in\n", - "the dispute resolution process set forth in this Section 7.5.\n", - "(i) Meet and Confer Process:\n", - "Initiation: If the parties are unable to resolve any Dispute through applicable Health Net internal appeal\n", - "processes, if any, the parties agree to meet and confer within thirty (30) days of a written request by either\n", - "party in a good faith effort to informally settle any Dispute. The parties each agree and understand that the\n", - "meet and confer requirements set forth herein may be satisfied only by meeting each of the following\n", - "requirements: (a) an actual meeting must occur between executive level employees of the parties who have\n", - "authority to resolve the Dispute and are each prepared to discuss in good faith the Dispute and proposed\n", - "resolution(s) to the Dispute, and (b) such meeting may take place either in person or on the telephone at a\n", - "mutually agreeable time, and (c) unless otherwise murually agreed by the parties, neither party is allowed\n", - "to have legal counsel present at the meeting or to substitute legal counsel for the executive level employee,\n", - "and (d) such meeting and all related discussions between the parties shall be treated in the same\n", - "manner as confidential protected settlement discussions under the State Rules of Civil Procedure.\n", - "Confidentiality: All documents created for the purpose of, and exchanged during, the meet and confer process\n", - "and all meet and confer discussions. negotiations and proceedings shall be treated as compromise and\n", - "settlement negotiations subject to applicable State law. To the extent the parties produce or exchange any\n", - "documents, the parties agree that such production or exchange shall not waive the protected nature of those\n", - "documents and shall not otherwise affect their inadmissibility as evidence in any subsequent proceedings.\n", - "(ii) Voluntary Mediation:\n", - "if the parties are unable to resolve any Dispute through the meet and confer process set forth above, and\n", - "desire to utilize other impartial dispute settlement techniques such as mediation or fact-finding, a joint\n", - "request for such services may be made to the American Arbitration Association (\"AAA\"), or the Judicial\n", - "Arbitration and Mediation Services (\"JAMS\") prior to submitting a Dispute to arbitration, or the parties\n", - "may initiate such other procedures as they may mutually agree upon.\n", - "7.6\n", - "Binding Arbitration. If the parties are unable to resolve a Dispute through the dispute resolution\n", - "process set forth in Section 7.5. the parties agree that such Dispute shall be settled by final and binding arbitration,\n", - "upon the motion of either party, under the appropriate rules of the AAA or JAMS, as agreed by the parties. Any\n", - "California Provider Participation Agreement\n", - "15\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 17\n", - "Arbitrator nust be either a judge, or an attorney licensed to practice law in the State of California, who is in good\n", - "standing with the State Bar, and has at least ten (10) years of experience with health care matters and the arbitration\n", - "of managed care disputes. The parties each understand and agree that the exhaustion of any Health Net internal\n", - "appeals processes and the Meet and Confer Process set forth in Section 7.5 (i) hereof are conditions precedent to\n", - "binding arbitration under this Section 7.6. Notwithstanding the foregoing, nothing contained herein is intended to\n", - "require binding arbitration of disputes alleging medical malpractice between a Beneficiary and Provider or to\n", - "Disputes between the parties alleging breaches of confidentiality of Beneficiary information, trade secret or\n", - "intellectual property obligations. The arbitration shall be conducted in San Francisco, California or Los Angeles,\n", - "California, or another location murnally agreeable to the parties that is not more than one hundred miles from\n", - "Provider's principal office. The written demand shall contain a detailed statement of the matter and facts and\n", - "include copies of all material documents supporting the demand. Except as provided below, arbitration must be\n", - "initiated within one year after the date the Dispute arose by submitting a written notice to the other party.\n", - "For the purposes of filing for arbitration regarding a Dispute over Health Net's alleged non-payment or\n", - "underpayment of Complete Claims under this Agreement, the parties agree that an arbitration shall be filed within\n", - "one (1) year after the date of Health Net's notice of its final determination on Provider's internal appeal if any, on\n", - "such Complete Claim.\n", - "The parties expressly agree that the deadlines to file arbitration set forth above shall not be subject to waiver, tolling,\n", - "alteration or modification of any kind or for any reason except for fraud. The failure to initiate arbitration before\n", - "such deadlines shall mean the complaining party shall be barred forever from initiating such proceedings.\n", - "All such arbitration proceedings shall be administered by the AAA or JAMS, as agreed by the parties; however, the\n", - "arbitrator shall be bound by applicable State and federal law, and shall issue a written opinion setting forth findings\n", - "of fact and conclusions of law. The parties agree that the decision of the arbitrator shall be final and binding as to\n", - "each of them. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction.\n", - "The arbitrator shall have no authority to make material errors of law or to award punitive damages or to add to.\n", - "modify, or refuse to enforce any agreements between the parties. The arbitrator shall make findings of fact and\n", - "conclusions of law and shall have no authority to make any award, which could not have been made by a court of\n", - "law. The party against whom the award is rendered shall pay any monetary award and/or comply with any other\n", - "order of the arbitrator within sixty (60) days of the entry of judgment on the award. The parties waive their right to\n", - "a jury or court trial.\n", - "The parties recognize and agree that theirs is an ongoing business relationship, which may lead to sensitive issues\n", - "with respect to the exchange of information related to any Dispute. The parties agree, therefore, to enter into such\n", - "protective orders (including without limitation creating a category of discovery documents \"for attorney's eyes\n", - "only\" to the extent feasible given the nature of the evidence and the Dispute). All discovery information shall be\n", - "used solely and exclusively for arbitration of the Dispute between the parties and may not be used for any other\n", - "purpose. After the arbitration award becomes final, each party shall return or destroy all documents obtained from\n", - "the other party during the course of the arbitration that are subject to a protective order, and within thirty (30) days\n", - "of such date shall provide to the other party an officer's certificate signed under penalty of perjury indicating that all\n", - "such information has been returned or destroyed.\n", - "In all cases submitted to arbitration, the parties agree to share equally the administrative fee as well as the arbitrator's\n", - "fee, if any, unless otherwise assessed by the arbitrator. The administrative fees shall be advanced by the initiating\n", - "party subject to final apportionnent by the arbitrator in this award. The parties agree that the content and decision\n", - "of any arbitration proceeding shall be confidential unless disclosure is required by applicable State or federal statutes\n", - "or regulations. The terms of Section 7.5 and Section 7.6 shall survive termination of this Agreement.\n", - "7.7\n", - "Entire Agreement. This Agreement represents the entire agreement between the parties hereto\n", - "with respect to the subject maiter hereof and supersedes any and all other agreements, either oral or written, between\n", - "the parties with respect to the subject matter hereof. and no other agreement, statement or promise relating to the\n", - "subject matter of this Agreement shall be valid or binding.\n", - "California Provider Participation Agreement\n", - "16\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 18\n", - "7.8\n", - "Governing Law. This Agreement shall be governed by and construed and enforced in accordance\n", - "with the laws of the State, except to the extent such laws conflict with or are preempted by any federal law, in which\n", - "case such federal law shall govern. Health Net is subject to the requirements of various local, State, and federal\n", - "laws, rules and regulations including, but not limited to, the requirements of Chapter 2.2 of Division 2 of the\n", - "California Health & Safety Code (the Knox-Keene Health Care Service Plan Act) and of Chapters 1 and 2, of\n", - "Division 1 of Title 28 of the California Code of Regulations (\"C.C.R.\") and Title 10 of the C.C.R. Any provision\n", - "required to be in this Agreement by any of the above shall bind Provider and Health Net whether or not expressly set\n", - "forth herein.\n", - "7.9\n", - "Indemnification.\n", - "7.9.1\n", - "Responsibility for Own Acts. Each party shall be responsible for its own acts or omissions and\n", - "for any and all claims, liabilities, injuries, suits, demands and expenses of all kinds which may result or\n", - "arise out of any alleged malfeasance or neglect caused or alleged to have been caused by that party or its\n", - "employees or representatives in the performance or omission of any act or responsibility of that party under\n", - "this Agreement.\n", - "7.9.2\n", - "Provider agrees to indemnify, defend, and hold harmless Health Net, its agents, officers, and\n", - "employees from and against any and all liability expense including defense costs and legal fees incurred in\n", - "connection with claims for damages of any nature whatsoever, including but not limited to, bodily injury,\n", - "death, personal injury, or property damage arising from Provider's performance or failure to perform its\n", - "obligations hereunder.\n", - "7.9.3 Health Net agrees to indemnify, defend, and hold harmless Provider, its agents, officers, and\n", - "employees from and against any and all liability expense, including defense costs and legal fees incurred in\n", - "connection with claims for damages of any nature whatsoever, including but not limited to, bodily injury,\n", - "death, personal injury, or property damage arising from Health Net's performance or failure to perform its\n", - "obligations hereunder.\n", - "7.10\n", - "Non-Exclusive Contract. This Agreement is non-exclusive and shall not prohibit Provider or\n", - "Health Net or Payor from entering into agreements with other health care providers or purchasers of health care\n", - "services.\n", - "7.11\n", - "No Third Party Beneficiary. Nothing in this Agreement is intended to. or shall be deemed or\n", - "construed to, create any rights or remedies in any third party, including a Beneficiary. Nothing contained herein\n", - "shall operate (or be construed to operate) in any manner whatsoever to increase the rights of any such Beneficiary or\n", - "the duties or responsibilities of Provider or Health Net or Payor with respect to such Beneficiaries.\n", - "7.12\n", - "Notice. Notices regarding the breach, term, termination or renewal of this Agreement shall be\n", - "given in writing in accordance with this Section 7.12 and shall be deemed given five (5) days following depositi in\n", - "the U.S. mail. postage prepaid. If sent by hand delivery, overnight courier, or facsimile, notices shall be\n", - "deemed\n", - "given upon documentation of delivery. All notices shall be addressed as follows:\n", - "Health Net:\n", - "Direct Network Contracting\n", - "180 Grand Avenue\n", - "Oakland, CA 94612\n", - "Facsimile: (510) 891-6958\n", - "Provider:\n", - "See Provider Primary Location Address in Exhibit I. Listing of Facilities\n", - "The addresses to which notices are to be sent may be changed by written notice given in\n", - "accordance with this Section.\n", - "7.13\n", - "Severability. If any provision of this Agreement is rendered invalid or unenforceable by any\n", - "local, State, or federal law, rule or regulation, or declared null and void by any court of competent jurisdiction. the\n", - "remainder of this Agreement shall remain in full force and effect.\n", - "California Provider Participation Agreement\n", - "17\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 19\n", - "7.14\n", - "Status as Independent Entitles. NoneCategory of Service: Covered Services delivered or arranged by Provider, excluding Laboratory services, Compensation: 100% of CMS Allowable, |\n", - "Category of Service: Anesthesia Services when provided by an Anesthesiologist or Certified Registered Nurse Anesthetist (American Society of Anesthesiology (ASA) unit scale), Compensation: $39 / ASA unit, |\n", - "Category of Service: Medical/Surgical Services by an Anesthesiologist or Certified Registered Norse Anesthetist, Compensation: 100% of CMS Allowable, |\n", - "Category of Service: Laboratory Services performed in Provider or Professional Provider office, Compensation: 100% of CMS Allowable, |\n", - "Category of Service: Pharmaceuticals With an established Medicare Value Without an established Medicare Value, Compensation: 100% of CMS Allowable 90% of the Average Wholesale Price (AWP), |\n", - "Category of Service: OB Services CPT 59400: Global Obstetric care with vaginal delivery CPT 59510: Global Obstetric care with cesarean delivery CPT 59610: Vaginal Delivery after previous cesarean delivery CPT 59618: Attempted vaginal delivery, resulting in cesareau, Compensation: $1,700.00 $1,700.00 $1,700.00 $1,700.00, |\n", - "Category of Service: Immunizations With an established Medicare Value Without an established Medicare Value, Compensation: 100% of CMS Allowable 90% of the Average Wholesale Price (AWP), |\n", - "Category of Service: General Health Panel CPT 80050: General Health Panel CPT 80055: Obstetric Panel, Compensation: $20.00 $15.00, |\n", - "Category of Service: By Report (BR) Procedures, Procedures not Listed and Procedures with Relativities not Established, and Pharmaceuticals/Immumizations without an established Medicare or AWP value, Compensation: 75% of billed charges for Covered Services, |\n", - " of the provisions of this Agreement is intended to create,\n", - "nor shall be deemed or construed to create any relationship between Provider and Health Net or a Payor other than\n", - "that of independent entities contracting with each other solely for the purpose of effecting the provisions of this\n", - "Agreement. Neither Provider nor Health Net/Payor, nor any of their respective agents, employees or representatives\n", - "shall be construed to be the agent, employee or representative of the other.\n", - "7.15\n", - "Addenda. Each Addendum to this Agreement is made a part of this Agreement as though set\n", - "forth fully herein. Any provision of an Addendum that is in conflict with any provision of this Agreement shall take\n", - "precedence and superseda the conflicting provision of this Agreement with respect to the subject matter of the\n", - "Addendum.\n", - "7.16\n", - "Calculation of Time. The parties agree that for purposes of calculating time under this\n", - "Agreement, any time period of less than ten (10) days shall be deemed to refer to business days and any time period\n", - "of ten (10) days or more shall be deemed to refer to calendar days unless the term \"business\" precedes the term\n", - "\"days\".\n", - "7.17\n", - "Waiver of Breach. The waiver of any breach of this Agreement by either party shall not\n", - "constitute a continuing waiver of any subsequent breach of either the same or any other provision(s) of this\n", - "Agreement. Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict\n", - "compliance in the future and to exercise any right or remedy related thereto.\n", - "THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED\n", - "BY THE PARTIES,\n", - "IN WITNESS WHEREOF, the parties have executed this Agreement.\n", - "PROVIDER Love\n", - "HEALTHNE\n", - "Signature\n", - "Obly\n", - "Signature Hoens\n", - "Health\n", - "Malik N. BAZ , M.D.\n", - "Cathy Hoens\n", - "Print Name\n", - "Print Name\n", - "Chairman\n", - "Title\n", - "VP Provider Network Management & Strategy\n", - "Title\n", - "Baz Allergy Asthma & Sinus\n", - "Group Name (If Applicable)\n", - "LIR.\n", - "6/29/2011\n", - "Date\n", - "77-0508441\n", - "7/15/11\n", - "Tax Identification Number\n", - "10-6-2011\n", - "Effective Date\n", - "Date\n", - "California Provider Participation Agreement\n", - "Fee-For-Service Direct Network Template\n", - "18\n", - "HN-DIN-PPA-04-08-11\n", - "\n", - "This page has 2 signature.\n", - "\n", - "Start of Page No. = 20\n", - "EXHIBIT I\n", - "LIST OF FACILITIES\n", - "The information below is mandatory. Please complete all applicable fields.\n", - "\"Provider must submit all National Provider Identifiers (NPI number/information) and a sample billing form (CMS\n", - "1500 or successor form) for each as directed by Health Net In addition, as directed by Health Net, the\n", - "corresponding Tax Identification Number shall be indicated, with a completed W-9 for each TIN as directed by\n", - "Health Net. The Tax Identification Number on the billing form must be consistent with the TIN on the W-9 form.\"\n", - "Please enter individual information if sale provider.\n", - "Baz Allergy Asthma $ Sinus Center\n", - "Provider Name (use Group name if applicable)\n", - "770208441\n", - "Provider / Group Tax Identification Number\n", - "1710906995\n", - "Primary / Group NPI Number\n", - "Primary Location\n", - "STATES\n", - "Address: 7471 STREET N. Fresno Street\n", - "can\n", - "218 COSE:\n", - "Fresno\n", - "STATES CA\n", - "93720-2457\n", - "Telephone #: 559-4364500\n", - "Fax #: 559-261-1526\n", - "Secondary Location\n", - "Address:\n", - "STREET\n", - "SEE ATTached Officelocations\n", - "STATES\n", - "CODS\n", - "STATE\n", - "ZIPILIZE\n", - "Telephone #:\n", - "Fax #:\n", - "Remit Address\n", - "Address: 7471 STREET N. Fresno St\n", - "SUFIEX\n", - "CITY\n", - "STATE\n", - "ZIP XXE\n", - "Fresno\n", - "CA\n", - "93720-2457\n", - "Telephone #: 559-436-4500\n", - "Fax #: 059-261-1526\n", - "Note: Please attach a list of additional locations on a separate sheet of paper if required.\n", - "California Provider Participation Agreement\n", - "19\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 21\n", - "Baz Allergy, Asthma & Sinus Center\n", - "**Tax ID Number 77-0208441 for all locations**\n", - "OFFICE LOCATIONS\n", - "Main Office\n", - "Billing/Mailing/Correspondence FOR ALL LOCATIONS:\n", - "NPI # 1710906995\n", - "7471 N. Fresno Street\n", - "Fresno, CA 93720\n", - "Phone: 559-436-4500\n", - "Fax: 559-261-1526\n", - "Second Office Address: NPI# 1508885781\n", - "1420 Shaw Ave. #105\n", - "Clovis, CA 93611\n", - "Phone: 559-325-9990\n", - "Fax: 559-797-0004\n", - "Third Office Address: NPI# 1023037108\n", - "1560 W. Lacey #103\n", - "Hanford, CA 93230\n", - "Phone: 559-582-8500\n", - "Fax: 559-584-9133\n", - "Fourth Office Address: NPI # 1811916992\n", - "2311 W. Cleveland #1\n", - "Madera, CA 93637\n", - "Phone: 559-674-0075\n", - "Fax: 559-673-6058\n", - "Fifth Office Address: NPI# 1215956396\n", - "220 S. Akers Suite A\n", - "Visalia, CA 93291\n", - "Phone: 559-713-1600\n", - "Fax: 559-713-1602\n", - "Sixth Office Address NPI# 1336399013\n", - "2021 E. Herndon 2nd floor\n", - "Clovis, CA 93611\n", - "Phone: 559-292-8700\n", - "Fax: 559-324-8748\n", - "\n", - "Start of Page No. = 22\n", - "EXHIBIT II\n", - "LIST OF PROFESSIONAL PROVIDERS\n", - "INSERT ROSTER HERE\n", - "Complete the information below, or you may attach as an Exhibit a complete Physician Roster including\n", - "Physician Name, License number, Specialties, Individual NPI number, CAQH number (if applicable), and\n", - "provider's Medicare Certification status (yes/no).\n", - "Provider / Group Tax Identification #: 770208441\n", - "FIRST NAME\n", - "CAST NAME\n", - "LICENSE NUTURER\n", - "PRIMARY EFECIALIY\n", - "Malik N. Bag\n", - "M.D.\n", - "A35393\n", - "SECONDAR SPECIALS\n", - "Allergy Immunology\n", - "Pediatrics\n", - "CARRE\n", - "UNDUAL NPT 7\n", - "MEDICAIS ORIGINAL\n", - "0941870\n", - "1114923307\n", - "Yes\n", - "PIRSONAME\n", - "MI\n", - "LANI NAME\n", - "LICENSE NUMBER\n", - "Praveen\n", - "Baddiga M.D.\n", - "A90273\n", - "PRIMARY SERCIAL'S\n", - "SECURDARY SPECIALTY\n", - "CADER\n", - "Allergy / EVERVIDUAL # Internal Medicine\n", - "Immunology\n", - "MEDICARE CERTIFIED (YYOT)\n", - "11854817\n", - "1598703480\n", - "Yes\n", - "FIRST NAME\n", - "321\n", - "MANT NAME\n", - "LIVINSE B\n", - "Belle\n", - "FREDRY SPECIALIT\n", - "Peralejo SUNS. KADDARY M.D. SPECIALTY\n", - "A102716\n", - "Allergy/\n", - "CASHEL'S\n", - "immunology\n", - "Internal Medicine\n", - "INDIVERIAL\n", - "MEDICARE\n", - "11871444\n", - "1104031327\n", - "Yes\n", - "2001\n", - "LAST RATE\n", - "LICENSE NUMBER\n", - "Baloh\n", - "Diaz\n", - "M.D\n", - "A066901\n", - "PRIMARY SPECIAL\n", - "SECUNDARY SPECIALIY\n", - "Pediatrics\n", - "CARDH A\n", - "INDIVIDUAL NPIC\n", - "MEDICAES CERTIFIED (Y/N)\n", - "10947949\n", - "1083682629\n", - "Yes\n", - "PINKI NAME\n", - "Mail\n", - "LAST NAME\n", - "LICENSI. NURSER\n", - "Lauren\n", - "FRIMAR MEDICALLY\n", - "S. Hiyana M.D.\n", - "A84500\n", - "SCOMURRY SPECIALTY\n", - "CAURE\n", - "Allergy INDIVIDUAL NET\n", - "Immunology\n", - "Internal Medicine\n", - "MEDICARE ERINSIED (7/N)\n", - "11799303\n", - "1215025275\n", - "Yes\n", - "FUIS\n", - "MI\n", - "LICENSE NUMBER\n", - "Hawaed [NAME\n", - "LAST NAME\n", - "D.\n", - "FRMMARY SPECIALTY\n", - "Pettigrew SECURDARY M.D. SPECIALITY PhD\n", - "4104136\n", - "ALLERGY\n", - "IMMUNOLOGY\n", - "INTERNAL MEDICINE\n", - "CARD\n", - "INDIVIDUAL\n", - "CESTUFIES (VN)\n", - "12172296\n", - "1891961835\n", - "Yes\n", - "California Provider Participation Agreement\n", - "20\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 23\n", - "ADDENDUM A\n", - "COMMERCIAL BENEFIT PROGRAMS\n", - "I.\n", - "Applicability This Addendum A and accompanying exhibits apply to Covered Services\n", - "delivered to Beneficiaries covered by commercial Benefit Programs that include but are not limited, to HMO, PPO.\n", - "EPO, POS, AIM, and any leased networks. All Covered Services delivered to a Beneficiary covered by a\n", - "commercial Benefit Program shall be paid in accordance with this Addendum A regardless of product specific name\n", - "unless otherwise specifically agreed by the parties and set forth in a separate rate exhibit.\n", - "IL\n", - "Preferred Provider Organization (PPO), Exclusive Provider Organization (EPO), Point of Service\n", - "(POS), Leased PPO Benefit Programs. and Pavor Disclosures. Provider understands and agrees that Health Net\n", - "may sell, lease, transfer or convey a list, including Provider, to Payors.\n", - "Payors shall actively encourage subscribers to use the list of contracted providers when obtaining medical care.\n", - "Active encouragement includes offering subscribers direct financial incentives to use the list of contracted providers\n", - "when obtaining medical care (such as reduced Copayments, Coinsurance and Deductibles), or providing or causing\n", - "the provision of information to subscribers advising such subscribers of the existence of a list of contracted\n", - "providers through a variety of advertising or marketing approaches that supply the names, addresses and telephone\n", - "numbers of contracted providers to subscribers in advance of their selection of a health care provider. Nothing in\n", - "this Addendum A shall be construed to require a Payor to actively encourage such Payor's subscribers to use the list\n", - "of contracted providers, including Provider, when obtaining medical care in the event of an Emergency.\n", - "Health Net shall not permit Payors to access this Agreement and pay Provider's contracted rate for the Benefit\n", - "Programs covered by this Addendum unless Payor, or Health Net on Payor's behalf, has actively enconraged\n", - "Payor's subscribers to use the list of contracted providers in obtaining medical care.\n", - "Provider agrees that the following commercial Benefit Program Payors are eligible to pay Provider's contracted rate\n", - "under this Addendum A as of the effective date of this Agreement:\n", - "NOT APPLICABLE\n", - "Health Net may modify the above list periodically. Provider may request in writing, and Health Net shall have thirty\n", - "(30) days from the date of such request, to provide Provider with an updated listing of Payors.\n", - "Provider understands and agrees that any Health Net company, including, but not limited to, Health Net Life\n", - "Insurance Company, are not Payors under this Addendum A but shall access this Agreement as Health Net.\n", - "III.\n", - "Payment. As compensation for rendering Contracted Services to Beneficiaries covered by\n", - "commercial HMO, PPO, EPO, POS and Leased PPO Benefit Programs under this Addendum A, Health Net shall\n", - "pay and Provider shall accept as payment in full the rates set forth in Exhibit A-1, subject to the payment conditions\n", - "set forth in Addendum E, the terms of this Agreement and applicable State and federal law. Notwithstanding any\n", - "other provision in this Agreement, the parties acknowledge that each Payor is solely responsible for paying Provider\n", - "for Covered Services rendered to those individuals for whom Payor provides health care coverage. For self-insured\n", - "Payors, Health Net shall not be obligated to pay all or any portion of any Provider claim on a Payor's behalf unless\n", - "and until Health Net has received sufficient funds from the applicable Payor to cover such claim. In the event such\n", - "Payor fails to provide funds to Health Net. Provider may seek payment from Beneficiary up to the rates specified in\n", - "this Exhibit. unless prohibited by applicable law.\n", - "California Provider Participation Agreement\n", - "21\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 24\n", - "EXHIBIT A-1\n", - "COMMERCIAL BENEFIT PROGRAMS\n", - "DIRECT NETWORK FEE-FOR-SERVICE\n", - "RATE EXHIBIT\n", - "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", - "Health Net or Payor shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary\n", - "Covered Services delivered under commercial Benefit Programs pursuant to this Addendum, the lesser of: (i) the rates\n", - "listed below, or (ii) 75% of Provider's billed charges.\n", - "California Provider Participation Agreement\n", - "22\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 25\n", - "ADDENDUMB\n", - "MEDICARE ADVANTAGE PROGRAM\n", - "I.\n", - "Applicability. Provider and Health Net agree that this Addendum shall apply only to Contracted Services\n", - "delivered to Beneficiaries covered by the Medicare Advantage (MA) Program, and this Addendum B may be\n", - "accessed by those Health Net entities holding a valid Medicare Advantage agreement with CMS at the time\n", - "Contracted Services are delivered.\n", - "II.\n", - "Payment. Subject to the terms of this Agreement, including without limitation the Payment Conditions set\n", - "forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically\n", - "Necessary Covered Services delivered under Health Net's MA Program pursuant to this Addendum\n", - "III.\n", - "Definitions\n", - "For purposes of this Addendum, the definitions included herein shall have the meaning required by law\n", - "governing the Medicare Advantage (MA) Program.\n", - "(a)\n", - "Medicare Advantage (MA). The statutory name applicable to the federal program of prepaid\n", - "health care services for Medicare beneficiaries established by the Medicare Prescription Drug,\n", - "Improvement, and Modernization Act of 2003.\n", - "(b)\n", - "Medicare (MA) Member or Beneficiary. A Medicare Beneficiary who has elected to enroll in\n", - "and receive coverage through a Health Net MA or MAPD Benefit Program, under the terms and conditions\n", - "of the Benefit Program's MA or MAPD Evidence of Coverage, and whose enrollment has been confirmed\n", - "by the Centers for Medicare and Medicaid Services (\"CMS\"), an agency of the U.S. Department of Health\n", - "and Human Services (\"HHS\") that administers the Medicare Advantage program.\n", - "(c)\n", - "Contract Period. The term of the contract between Health Net and CMS.\n", - "(d)\n", - "Delegation. Delegation to Provider of a certain contractual obligation of Health Net under Health\n", - "Net's contract with CMS.\n", - "(e)\n", - "MA Benefit Program. A Health Net Medicare Advantage Benefit Program that provides\n", - "coverage for certain health care services.\n", - "(f)\n", - "MAPD Benefit Program. An MA Benefit Program that includes Medicare Part D prescription\n", - "drug coverage.\n", - "(g)\n", - "MA/MAPD Evidence of Coverage. A legally binding statement of coverage, revised annually,\n", - "between a Medicare Member and Health Net under which a Medicare Member is entitled to receive\n", - "coverage for certain hospital, medical and other associated health care services.\n", - "(h)\n", - "Downstream Providers. A Participating PROVIDER who or which is contracted with PROVIDER\n", - "to render services to Beneficiaries.\n", - "IV.\n", - "General Provisions.\n", - "4.1\n", - "Provider Obligations. Provider agrees to render Contracted Services to MA Members, in\n", - "accordance with the terms and conditions of Health Net's MA Programs. Health Net shall provide Provider\n", - "with the Benefit Program Requirements of such Benefit Programs not set forth in this Addendum. Such Benefit\n", - "Program Requirements include the provisions of the applicable MA or MAPD Evidence of Coverage. and\n", - "Health Net Policies. Provider acknowledges that the determination of Covered Services shall be governed by\n", - "coverage guidelines established by Health Net and the MA Program, with Health Net being solely responsible\n", - "for final coverage determinations, subject to applicable appeal procedures.\n", - "California Provider Participation Agreement\n", - "23\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 26\n", - "4.2\n", - "Member Services. Provider shall cooperate fully with Health Net in the investigation and\n", - "resolution of complaints by MA Members regarding Provider, or the services Provider provides or arranges. in\n", - "compliance with CMS and state regulatory requirements.\n", - "4.3\n", - "Reports and Administration. Health Net shall have sole responsibility for filing reports,\n", - "obtaining approval from, and complying with the applicable laws and regulations of federal, state and local\n", - "governmental agencies having jurisdiction over Health Net. Provider shall cooperate in providing Health Net\n", - "with such information and assistance regarding MA. Members and Provider's performance under this Agreement\n", - "and Addendum as Health Net may reasonably require in filing such reports and complying with applicable laws\n", - "and regulations.\n", - "4.4\n", - "Health Net Policies and Procedures. Health Net shall promptly communicate any material\n", - "change in Health Net's policies and procedures affecting Provider in accordance with Section 3.2 of this\n", - "Agreement.\n", - "4.5\n", - "Provider and Health Net agree to: (i) Process contracted Provider claims received at Health Net\n", - "within 60 calendar days of receipt, (ii) Process non-contracted provider clean claims received by Provider\n", - "within 30 calendar days and 60 calendar days for all other claims.\n", - "4.6.\n", - "Regulatory Obligations.\n", - "A.\n", - "Health Net and Provider agree and Provider will require its subcontractors to agree to:\n", - "(1)\n", - "For a period of ten (10) years from the final date of the Contract Period or the date of completion\n", - "of any audit, whichever is later, allow HHS, CMS, the Comptroller General, or their designees the right to audit,\n", - "evaluate, and inspect any books, contracts, records, including medical records and documentation of Provider and\n", - "any Provider subcontractor. involving transactions related to Health Net's contract with CMSi *See 42 CFR §§\n", - "422.504(i)(2)(i) and (ii); Ch. 11. 100.4, MMCM]\n", - "(2)\n", - "Cooperate in assist in, and provide information as requested for audits, evaluations and inspections\n", - "performed under Section A(1) above. [*See Ch. 11. $100.4, MMCM]\n", - "(3)\n", - "Safeguard MA Member's privacy and confidentiality. [*See Ch. 11. $100.4, Medicare Managed\n", - "Care Manual (\"MMCM\")]\n", - "(4)\n", - "Specify a prompt payment requirement in its written agreement with a Provider subcontractor.\n", - "[See Ch. 11, $100.4, MMCM]\n", - "(5)\n", - "Ensure the accuracy of MA Member's health and other records. [*See Ch. 11. $100.4, MMCM]\n", - "(6)\n", - "Hold MA Members harmless for payment of any fees that are the legal obligation of Health Net in\n", - "the event of but not limited to, insolvency of, breach by or billing of Provider by Health Net. [*See 42 CFR\n", - "422.504(g)(1)(i) and (h)(3)(i)]\n", - "(7)\n", - "Comply with all applicable Medicare laws, regulations, reporting requirements and CMS\n", - "instructions. [*See 42 CFR § 422.504(i)(4)(v)]\n", - "(S)\n", - "Perform each service or other activity under this Agreement in a manner consistent and in\n", - "compliance with Health Net's contractual obligations to CMS. (*See 42 CFR 422.504(i)(3)(iii)]\n", - "(9)\n", - "Maintain all records relating to this contract for a minimum of ten (10) years from the final date of\n", - "the Contract Period or the date of the completion date of any audit. whichever is later. [See Ch. 11. §100.4,\n", - "MMCM]\n", - "California Provider Participation Agreement\n", - "24\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 27\n", - "(10)\n", - "Health Net oversees and is ultimately responsible to CMS for any functions and responsibilities\n", - "described in the Medicare Advantage regulations. Provider understands that this accountability provision also\n", - "applies to this Addendum. [See CH. 11 $100.4, MMCM. See 42 CFR §§422.504 (i)(4)(iii)\n", - "(11)\n", - "Comply with Health Net Policies. [*See Ch. 11. $100.4, MMCM]\n", - "(12)\n", - "Effective January 1. 2010, not collect cost sharing from any Medicare Member that exceeds the\n", - "amount of cost sharing that would be permitted with respect to the Medicare Member under Title XIX of the Social\n", - "Security Act if the Medicare Member were not enrolled in the MA/MAPD Benefit Program. In no event shall\n", - "Provider or a Provider subcontractor hold a Medicare Member responsible for any cost sharing for Covered Services\n", - "when a State entity is responsible for paying such amount. Where the State is responsible for paying the cost share\n", - "amount. Provider shall either accept Health Net's contracted rate as payment in full or bill the appropriate State\n", - "source for the cost share amount. [See CMS 2010 Call Letter.]\n", - "B.\n", - "Provider and Health Net agree:\n", - "(1)\n", - "Provider shall bill and Health Net shall make payment for Covered Services in accordance with\n", - "the Agreement and Health Net Policies. [*See Ch. 11 $100.4, Medicare Managed Care Manual (\"MMCM\")]\n", - "C.\n", - "Provider and its subcontractors further agree to the following:\n", - "(a)\n", - "To pay for emergency and urgently needed services consistent with federal regulations, if such\n", - "services are Provider's liability.\n", - "(b)\n", - "To pay for renal dialysis services for Beneficiaries temporarily outside the service area, if such\n", - "services are Provider's responsibility.\n", - "(c)\n", - "To direct access to mammography screening and influenza vaccinations.\n", - "(d)\n", - "To direct access to in-network women's health specialist for women for routine and preventative\n", - "services.\n", - "(e)\n", - "To have approved procedures to identify, assess and establish a treatment plan for Beneficiaries with\n", - "complex or serious medical conditions.\n", - "(f)\n", - "To provide access to benefits in a manner described by CMS.\n", - "(g)\n", - "To protect Beneficiaries who are hospitalized from loss of benefits through the period of time CMS\n", - "premiums are paid.\n", - "(h)\n", - "To work with Health Net in conducting a health assessment of all new Beneficiaries within ninety\n", - "(90) days of the effective date of enrollment.\n", - "(i)\n", - "That Provider must notify any Professional Provider being terminated, in writing, of the reason(s) for\n", - "denial, suspension or termination determinations.\n", - "(j)\n", - "To not employ or contract with individuals excluded from participation in Medicare under Section\n", - "1128 or 1128A of the Social Security Act.\n", - "(k)\n", - "To adhere to Medicare's appeals, expedited appeals and expedited review procedures for Medicare\n", - "HMO Beneficiaries, including gathering and forwarding information on appeals to Health Net. as\n", - "necessary.\n", - "California Provider Participation Agreement\n", - "25\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 28\n", - "(I)\n", - "That Medicare Beneficiaries health services are being paid for with Federal funds, and as such,\n", - "payments for such services are subject to laws applicable to individuals or entities receiving Federal\n", - "funds.\n", - "(in)\n", - "To assume the financial responsibility for any failure to comply with Medicare Regulations,\n", - "including, but not limited to, failure to provide records when requested and failure to provide or\n", - "document Notice(s) of Non-Coverage to Beneficiaries.\n", - "(11)\n", - "That for purposes of Medicare Beneficiaries, retroactive eligibility changes shall be limited to\n", - "thirty six (36) months or as otherwise required by CMS\n", - "(o)\n", - "To comply with Medicare laws. regulations, reporting requirements and CMS instructions.\n", - "(p)\n", - "To not collect any co-payment or other cost sharing for influenza vaccine and pneumococcal vaccines.\n", - "(q)\n", - "To comply with and require that all Downstream Providers comply with applicable State and federal\n", - "laws and regulations, including Medicare laws and regulations and CMS instructions.\n", - "VI.\n", - "DELEGATION.\n", - "The parties acknowledge that none of Health Net's MA Program contract obligations to CMS have been\n", - "delegated by Health Net to Provider.\n", - "California Provider Participation Agreement\n", - "26\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 29\n", - "EXHIBIT B-1\n", - "MEDICARE ADVANTAGE PROGRAM\n", - "DIRECT NETWORK FEE-FOR-SERVICE\n", - "RATE EXHIBIT\n", - "I. Payment Rates\n", - "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", - "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", - "Services delivered under Medicare Advantage Benefit Programs pursuant to this Addendum. the lesser of: (i) the rates\n", - "listed below, or (ii) 75% of Provider's billed charges.\n", - "California Provider Participation Agreement\n", - "27\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "\n", - "\n", - "\n", - "Start of Page No. = 30\n", - "ADDENDUM O\n", - "MEDI-CAL BENEFIT PROGRAM\n", - "Provider understands and agrees that the obligations of Health Net. Inc. set forth in this Addendum shall be the\n", - "obligations of Health Net of California, Inc. (\"Health Net\"), an Affiliate of Health Net, Inc, and not the obligations\n", - "of Health Net, Inc. or any other Affiliate of Health Net, Inc. Health Net has entered into one or more Medi-Cal\n", - "prepaid health plan agreements with the California Department of Health Care Services (\"DHCS\"). For the\n", - "purposes of this Addendum, Health Net's Medi-Cal agreements with the DHCS and any subcontracts with Medi-Cal\n", - "prepaid health plans, are hereinafter collectively referred to as the Medi-Cal Agreement\". Health Net has agreed,\n", - "under the Medi-Cal Agreement. to provide medical services covered under California's Medi-Cal Program,\n", - "including Provider risk services, to Medi-Cal HMO Beneficiaries enrolled in or otherwise assigned to Health Net, on\n", - "a prepaid basis. The provisions of the Addendum are required to appear in all subcontracts under the Medi-Cal\n", - "Agreement by the terms of the Medi-Cal Agreement and by Medi-Cal law and may not be altered.\n", - "Provider understands and agrees that Health Net Community Solutions, Inc. is an Affiliate of Health Net and of\n", - "Health Net of California, Inc. and has entered into an agreement with the Fresno-Kings-Madera Regional Health\n", - "Authority (CalViva Health) to provide Covered Services to Medi-Cal beneficiaries in Fresno, Kings and Madera\n", - "Counties who enroll in CalViva Health. Provider understands and agrees that Health Net of California, Inc. is\n", - "providing Covered Services as a subcontractor to Health Net Community Solutions, Inc. for the CalViva Health\n", - "Medi-Cal membership. Provider further understands and agrees that it will provide Covered Services to CalViva\n", - "Health Medi-Cal members pursuant to the Agreement and that all terms and conditions of the Agreement, including\n", - "reimbursement, shall apply to the provision of Covered services to the CalViva Health Medi-Cal members.\n", - "Health Net has or may enter into contracts with certain Payors, including local initiatives such as Cal Viva Health in\n", - "Fresno, Kings and Madera Counties, to provide or arrange for Covered Services to Medi-Cal beneficiaries enrolled in\n", - "the Medi-Cal plans of such Payors. Provider understands and agrees that a Payor may have adopted policies and\n", - "procedures, including, but not limited to, quality assurance and quality improvement programs. Provider further\n", - "understands and agrees that Provider and its Participating Providers shall comply with all the policies and procedures\n", - "adopted by a Payor and shall participate in the Payor's quality assurance aud quality improvement programs.\n", - "A.\n", - "COMPENSATION PROVISIONS.\n", - "1.\n", - "Compensation. Provider shall arrange and provide Contracted Services to Medi-Cal HMO\n", - "Beneficiaries of Health Net's Benefit Programs covered under this Addendum on a fee-for-service basis. As\n", - "compensation for providing such Contracted Services, Provider shall be paid in accordance with the rates set forth\n", - "on Exhibit C-1. subject to the payment conditions set forth in Addendum E. Such compensation shall be paid within\n", - "forty-five (45) working days of receipt of a complete and accurate claim for Covered Services rendered to a Medi-\n", - "Cal HMO Beneficiary.\n", - "2.\n", - "Billing. Notwithstanding anything to the contrary to the Agreement, if Provider is compensated on\n", - "a fee-for-service basis, Provider shall submit to Health Net, via Health Net's electronic claims submission program\n", - "or hardcopy as determined by Health Net, Complete Claims within one hundred eighty (180) days after the month in\n", - "which the Covered Service is reudered unless Provider demonstrates good cause pursuant to applicable State law.\n", - "Where Health Net is the secondary payor under Coordination of Benefits, Provider shall submit Complete Claims\n", - "for Covered Services accompanied by the explanation of benefits (EOB) or explanation of payment (EOP) from the\n", - "primary payor to Health Net within one hundred eighty (180) days of the date of the EOB/EOP.\n", - "If the Provider fails to comply with the timely claims submission/filing requirements set forth above,\n", - "Health\n", - "Net shall reimburse the Provider at the following rates: 75% of usual allowance for claims submitted during\n", - "the seventh through ninth month after the month of service; 50% of usual allowance for claims submitted during the\n", - "tenth through the twelfth month after the month of service. Health Net shall not be liable for payment to Provider\n", - "for any Complete Claims received after the twelfth month after the month of service.\n", - "California Provider Participation Agreement\n", - "28\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 31\n", - "B.\n", - "GENERAL PROVISIONS\n", - "1.\n", - "Provider Certification. Provider is certified to participate in Medicaid/Medi-Cal under Title XIX\n", - "of the Social Security Act or other applicable State law pertaining to Title XIX of the Social Security Act.\n", - "2.\n", - "Provision\n", - "of\n", - "Covered Services. Provider shall arrange Covered Services for assigned\n", - "Beneficiaries. For the purposes of this Addendum, \"Covered Services\" means those health care services, supplies\n", - "and items that are specified as being covered under the Medi-Cal Agreement. Provider shall arrange Covered\n", - "Services for Beneficiaries. in accordance with the following, each of which is hereby incorporated by reference as if\n", - "set out in full herein:\n", - "a)\n", - "The terms and conditions of this Addendun and the Agreement.\n", - "b)\n", - "The terms and conditions of the Medi-Cal Agreement and the applicable Evidence of\n", - "Coverage.\n", - "e)\n", - "Health Net Medi-Cal policies and procedures and physician bulletins.\n", - "d)\n", - "DHCS Medi-Cal Managed Care Division (MMCD) Policy Letters.\n", - "e)\n", - "All laws applicable to Provider and Health Net.\n", - "1)\n", - "Health Net's Utilization Care Management Program and Quality Improvement Program.\n", - "g)\n", - "Standards requiring services to be provided in the same manner, and with the same\n", - "availability, as services are rendered to other patients.\n", - "b)\n", - "No less than the minimum clinical quality of care and performance standards that are\n", - "professionally recognized and/or adopted, accepted or established by Health Net.\n", - "3.\n", - "Preparation and Retention of Records; Access to Records: Audits. Provider shall prepare and\n", - "maintain medical and other books and records required by law in a form maintained in accordance with the general\n", - "standards applicable to such book or record keeping, Provider shall maintain such financial. administrative and\n", - "other records as may be necessary for compliance by Health Net with all applicable local, State and federal laws.\n", - "Provider shall retain such books and records and all encounter data for a term of at least five (5) years from the close\n", - "of the California State fiscal year in which the Agreement is in effect. Provider shall make Provider's books, records\n", - "and encounter data pertaining to the goods and services furnished under the terms of the Agreement, available for\n", - "inspection, examination or copying by Health Net, DHCS, the United States Department of Health and Human\n", - "Services (\"DHHS\"), the California Department of Managed Health Care (\"DMHC\"), the United States Department\n", - "of Justice (\"DOJ\"), and any other regulatory agency having jurisdiction over Health Net. The records shall be\n", - "available at Provider's place of business, or at such other innitually agreeable location in California. When such\n", - "entities request Provider's records, Provider shall produce copies of the requested records at no charge. Provider\n", - "shall permit Health Net, and its designated representatives, and designated representatives of local, State, and federal\n", - "regulatory agencies having jurisdiction over Health Net. to conduct site evaluations and inspections of Provider's\n", - "offices and service locations. [22 CCR § 53250(e)(1); W & § 14452(c); Medi-Cal Agreement ].\n", - "4.\n", - "Subcontracting Under the Agreement. Provider shall not subcontract for the performance of\n", - "services under the Agreement without the prior written consent of Health Net Every such subcontract shall provide\n", - "that it is terminable with respect to Medi-Cal HMO Beneficiaries by Provider upon Health Net's request. Provider\n", - "shall furnish Health Net with copies of such subcontracts. and amendments thereto, within ten (10) days of\n", - "execution. Each such subcontracting Provider shall meet Health Net's credentialing requirements, prior to the\n", - "subcontract becoming effective. Provider shall be solely responsible to pay any health care Provider pennitted\n", - "under the subcontract, and shall hold, and ensure that health care Providers hold, Health Net, Beneficiaries and the\n", - "State harmless from and against any and all claims which may be made by such subcontracting Providers in\n", - "connection with services rendered to Medi-Cal HMO Beneficiaries under the subcontract. Provider shall maintain\n", - "California Provider Participation Agreement\n", - "29\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 32\n", - "and make available to Health Net, DHCS, DHHS, DMHC, DOJ, and any other regulatory agency having jurisdiction\n", - "over Health Net, copies of all Provider's subcontracts under the Agreement and to ensure that all such subcontracts\n", - "are in writing and require that the subcontractor: (1) make all applicable books and records available for inspection,\n", - "examination or copying by said entities; (2) retain such books and records for a term of at least five (5) years from\n", - "the close of the fiscal year in which the subcontract is in effect: and (3) maintain such books and records in a form\n", - "maintained in accordance with the general standards applicable to such book or record keeping. [22 CCR §\n", - "53250(e)(3)].\n", - "5.\n", - "Federal Disclosure Form. Provider shall submit to Health Net a completed Disclosure Form,\n", - "attached to this Addendum for officers and other persons associated with Provider as required by California Welfare\n", - "and Institutions Code 14452(a).\n", - "6.\n", - "Medi-Cal HMO Beneficiary Education. Provider shall make health education materials and\n", - "programs available to Medi-Cal HMO Beneficiaries on the same basis that it makes such materials and programs\n", - "available to the general public, and shall use its best efforts to encourage Medi-Cal HMO Beneficiaries to participate\n", - "in such health education programs. [Medi-Cal Agreement].\n", - "7.\n", - "Medi-Cal HMO Beneficiaries and State Held Harmless. Provider agrees that in no event,\n", - "including, but not limited to, non-payment by Health Net. the insolvency of Health Net, or breach of the Agreement,\n", - "shall Provider or a subcontractor of Provider bill, charge, collect a deposit from, seek compensation, remomeration,\n", - "or reimbursement from, or have any recourse against Medi-Cal HMO Beneficiaries, the State of California, or\n", - "persons other than Health Net acting on their behalf for services provided pursuant to the Agreement. Provider\n", - "agrees: (1) this provision shall survive the termination of the Agreement regardless of the cause giving rise to\n", - "termination and shall be construed to be for the benefit of Medi-Cal HMO Beneficiaries: and (2) this provision\n", - "supersedes any oral or written contrary agreement now existing or hereafter entered into between Provider and\n", - "Medi-Cal HMO Beneficiaries or persons acting on their behalf. Any modification, addition, or deletion of or to the\n", - "provisions of this clause shall be effective on a date no earlier than fifteen (15) days after the DHCS has received\n", - "written notice of such proposed change and has approved such change. [22 CCR § 53250(e)(6)].\n", - "8.\n", - "No Surcharges and No Copayments. Provider shall not charge a Medi-Cal HMO Beneficiary\n", - "any fee, surcharge or Copayment for health care services rendered pursuant to the Agreement except when explicitly\n", - "allowed by the Medi-Cal Benefit Program, for covered services rendered pursuant to the Agreement. In addition,\n", - "Provider shall not collect a sales, use or other applicable tax from Medi-Cal HMO Beneficiaries for the sale or\n", - "delivery of medical services. If Health Net receives notice of any additional charge, Provider shall fully cooperate\n", - "with Health Net to investigate such allegations, and shall promptly refund any payment deemed improper by Health\n", - "Net to the party who made the payment. [Knox-Keene Act and Medi-Cal Agreement].\n", - "9.\n", - "Grievances and Appeals. Provider shall resolve all grievauces and appeals relating to the\n", - "provision of services to Medi-Cal HMO Beneficiaries in accordance with the Health Net Medi-Cal grievance and\n", - "appeal procedures.\n", - "10.\n", - "Provider Patient Relationship. Provider shall be solely responsible, without interference from\n", - "Health Net or its agent, for providing Hospital Services to Medi-Cal HMO Beneficiaries, and shall have the right to\n", - "object to treating any individual who makes enerous the relationship between Provider and Medi-Cal HMO\n", - "Beneficiary. In the event of a breakdown in such relationship, Health Net shall make reasonable efforts to assign the\n", - "Medi-Cal HMO Beneficiary to another Participating Provider. If reassignment is unsuccessful, a request may be\n", - "filed with the DHCS to permit termination of services to such Medi-Cal HMO Beneficiary. Approval from the\n", - "DHCS must be obtained before Provider terminates services to such Medi-Cal HMO Beneficiary.\n", - "11.\n", - "Fair Employment Requirements. During the term of this Agreement, Provider and its\n", - "subcontractors shall not unlawfully discriminate against any employee or applicant for employment because of race,\n", - "religious creed, color, national origin, ancestry, physical disability, mental disability, medical condition. marital\n", - "status, age (over 40) or sex. Provider and its subcontractors also shall ensure that the evaluation and treatment of\n", - "their employees and applicants for employment are free of such discrimination. Provider and its subcontractors shall\n", - "comply with the provisions of the Fair Employment & Housing Act (California Government Code, Section 12990 et\n", - "seg.) and the applicable regulations promulgated thereunder (California Code of Regulations, Title 2, Section 7285.0\n", - "California Provider Participation Agreement\n", - "30\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 33\n", - "et seq.). The applicable regulations of the Fair Employment & Housing Commission implementing Government\n", - "Code, Section 12990, set forth in Chapter 5 of Division 4 of Title 2 of the California Code of Regulations are\n", - "incorporated into this Agreement by reference and made a part hereof as if set forth in full Provider and its\n", - "subcontractors shall give written notice of their obligations under this clause to labor organizations with which they\n", - "have a collective bargaining or other agreements.\n", - "12.\n", - "Governing Law. The Agreement shall be governed by and construed and enforced in accordance\n", - "with all laws and contractual obligations incumbent upon Health Net. Provider shall comply with all applicable\n", - "local, State, and federal laws, now or hereafter in effect, to the extent that they directly or indirectly affect Provider\n", - "or Health Net. and bear upon the subject matter of the Agreement. Provider shall comply with the provisions of the\n", - "Medi-Cal Agreement, and Chapters 3 and 4 of Subdivision 1 of Division 3 of Title 22 of the California Code of\n", - "Regulations. In addition, Health Net is subject to the requirements of Chapter 2.2 of Division 2 of the California\n", - "Health and Safety Code and Subchapter 5.5 of Chapter 3 of Title 10 of the California Code of Regulations. Any\n", - "provision required to be in the Agreement by either of the above laws shall bind the parties whether or not provided\n", - "in the Agreement. [22 CCR § 53250(c)(2)]; § 14452(a): Knox-Keene Act].\n", - "13.\n", - "Notice. Provider shall notify DHCS in the event this Agreement is amended or terminated\n", - "Notice to DHCS is considered given when properly addressed and deposited with the United States Postal Service as\n", - "first class registered mail, postage attached. [Knox-Keene Act and Medi-Cal Agreement].\n", - "14.\n", - "Reports: Provider shall provide Health Net. within the time requested by Health Net, with all\n", - "such reports and information AS Health Net may require to allow to meet the reporting requirements under the Medi-\n", - "Cal Agreement or any applicable law, [22 CCR 53250(c)(5)].\n", - "15.\n", - "Confidentiality of Information. Names of persons receiving public social services are\n", - "confidential and are to be protected from unauthorized disclosure in accordance with Title 45, Code of Federal\n", - "Regulations, Section 205.50 and Section 14100.2 of the California Welfare and Institutions Code and the regulations\n", - "adopted thereunder. For the purposes of this Agreement, all information, records, data, and data elements collected\n", - "and maintained for or in connection with performance under this Agreement and pertaining to Medi-Cal HMO\n", - "Beneficiaries shall be protected by Provider from unauthorized disclosure, With respect to any identifiable\n", - "information concerning a Medi-Cal HMO Beneficiary under this Agreement that is obtained by Providers or its\n", - "subcontractors, Provider: (1) will not use any such information for any purpose other than carrying out the express\n", - "terms of this Agreement; (2) will promptly transmit to Health Net all requests for disclosure of such information; (3)\n", - "will not disclose, except as otherwise specifically permitted by this Agreement, any such information to any party\n", - "other than Health Net without Health Net's prior written authorization specifying that the information is releasable\n", - "under applicable law, and (4) will, at the expiration or termination of this Agreement, return all such information to\n", - "Health Net or maintain such information according to written procedures provided Provider by Health Net for this\n", - "purpose. Provider shall ensure that its subcontractors comply with the provisions of this paragraph.\n", - "16.\n", - "Third Party Tort Liability. Provider shall make no claim for recovery for health care services\n", - "reudered to a Medi-Cal HMO Beneficiary when such recovery would result from an action involving the tort\n", - "liability of a third party or casualty liability insurance, including workers' compensation awards and uninsured\n", - "motorist coverage. Within five (5) days of discovery, Provider shall notify Health Net of cases in which an action\n", - "by the Medi-Cal HMO Beneficiary involving the tort or workers' compensation liability of a third party could result\n", - "in a recovery by the Medi-Cal HMO Beneficiary. Provider shall promptly provide: (1) all information requested by\n", - "Health Net in connection with the provision of health care services to a Medi-Cal HMO Beneficiary who may have\n", - "an action for recovery from any such third party; (2) copies of all requests by subpoena from attorneys, insurers or\n", - "Medi-Cal HMO Beneficiaries for copies of bills, invoices or claims for health care services; and (3) copies of all\n", - "documents released as a result of such requests. Provider shall ensure that its subcontractors comply with the\n", - "requirements of this provision.\n", - "17.\n", - "Amendments.\n", - "17.1\n", - "When required under Medi-Cal law, Amendments to the Agreement shall be submitted by Health\n", - "Net to the DHCS for prior approval at least thirty (30) days before the effective date of any proposed changes\n", - "governing compensation, services or term. Proposed changes, which are neither approved nor disapproved by the\n", - "California Provider Participation Agreement\n", - "31\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 34\n", - "Department, shall become effective by operation of law thirty (30) days after the DHCS has acknowledged receipt.\n", - "or upon the date specified in the amendment, whichever is later. Subcontracts between a prepaid health plan and a\n", - "subcontractor shall be public records on file with the DHCS. [22 CCR § 53250(a), (c)(3), & (e)(4); W §\n", - "14452(a)j.\n", - "17.2\n", - "Notwithstanding the foregoing and any provisions to the contrary in this Agreement, the parties\n", - "understand and agree that an amendment to the material terms of this Agreement shall be permitted without the\n", - "consent of Provider if: (i) Provider is a non-institutional provider; (ii) the amendment applies to the Medi-Cal or\n", - "Healthy Fannlies products; (iii) Provider is compensated on a fee-for-service basis; (IV) Health Net gives the\n", - "Provider a minimum of ninety (90) business days' notice of its intent to amend the Agreement; (v) Provider has the\n", - "right to exercise its intent to negotiate and agree to the amendment within thirty (30) business days of Provider's\n", - "receipt of the notice of amendment; and (vi) Provider has the right to terminate the Agreement within ninety (90)\n", - "business days from the date of receipt of such notice if Provider does not exercise the right to negotiate the\n", - "amendment and no agreement is reached. In such event, the amendment becomes effective ninety (90) days from\n", - "the date of the notice set forth in this paragraph if Provider does not exercise its right to negotiate the amendment or\n", - "to terminate the Agreement as described in this paragraph.\n", - "18.\n", - "Notice of Change in Availability or Location of Covered Services. Health Net is obligated to\n", - "ensure Medi-Cal HMO Beneficiaries are notified in writing of any changes in the availability or location of Covered\n", - "Services at least thirty (30) days prior to the effective date of such changes, or within fourteen (14) days prior to the\n", - "change in cases of unforeseeable circumstances. Such notifications must be approved by DHCS prior to the release.\n", - "In order for Health Net to meet this requirement, Provider is obligated to notify Health Net in writing of any changes\n", - "in the availability or location of Covered Services at least forty (40) days prior to the effective date of such changes.\n", - "19.\n", - "Transfer of Care Upon Termination of the Agreement. Provider shall, pursuant to the\n", - "requirements of the Medi-Cal Agreement, assist in the orderly transfer of care of all Medi-Cal HMO Beneficiaries\n", - "under the care of Provider in the event of the termination of the Agreement.\n", - "20.\n", - "Assignment and Delegation. Assignment or delegation of the Agreement shall be void unless\n", - "prior written approval is obtained from the DHCS, in the instances where approval by the DHCS is required.\n", - "21.\n", - "Carve-out of California Children's Services (CCS) Program Services. The parties\n", - "acknowledge that health care services to treat CCS-eligible conditions are \"carved out\" of Health Net's coverage\n", - "obligations under the Medi-Cal Benefit Programs. Provider shall identify and timely refer Beneficiaries with\n", - "possible CCS-eligible conditions to the appropriate County CCS Program. Upon referral, Provider shall inform the\n", - "Beneficiary's parent or guardian and shall notify Health Net of any such referral. The CCS Program requires\n", - "eligible children to be treated at CCS-certified facilities by CCS-paneled providers. The CCS Program may require\n", - "transfer to CCS-certified facilities with CCS-paneled providers. The CCS Program is financially responsible for\n", - "payment of health care costs to treat a CCS-eligible condition. The parties understand and agree that Health Net is\n", - "not financially responsible for payment of services to treat CCS-eligible conditions. In the event Health Net\n", - "inadvertently pays a claim for such services, Health Net may recover the amount paid pursuant to Section 4.3 of the\n", - "Agreement.\n", - "22.\n", - "Cultural and Linguistic Services. Provider shall: (1) not require or encourage Beneficiaries to\n", - "utilize family Beneficiaries or friends as interpreters; (2) record the language needs of Beneficiaries in the medical\n", - "record; and (3) document Beneficiary requests or refusals of interpreter services in the Beneficiary's medical record.\n", - "Provider shall arrange interpreter services for Beneficiaries either through telephone language services or face-to-\n", - "face interpreters. Provider is encouraged to directly make these interpretive services available. However, upon\n", - "request. Health Net's Member Services Department is available to provide certain interpretive assistance to facilitate\n", - "communications.\n", - "23.\n", - "EPSDT Supplemental Services. Provider shall arrange for Early and Periodic Screening,\n", - "Diagnosis and Treatment Supplemental Services for Beneficiaries under the age of 21 in accordance with the\n", - "requirements of the Medi-Cal Agreement.\n", - "California Provider Participation Agreement\n", - "32\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-03-11\n", - "\n", - "Start of Page No. = 35\n", - "24.\n", - "CHDP (Children's Health and Disability Prevention) Program. Health Net requires that all\n", - "providers of CHDP services be certified by the CHDP Program and adhere to the CHDP Program requirements.\n", - "25.\n", - "CCS (California Children's Services). Provider is responsible for timely referral of children\n", - "with potential CCS eligible conditions. Failure to appropriately refer will result in Provider assuming financial\n", - "responsibility for any related charges.\n", - "26.\n", - "CPSP (Comprehensive Perinatal Services Program). Provider is required to refer members to\n", - "DHCS-certified CPSP providers to ensure that all pregnant women have access to care in accordance with DHCS\n", - "requirements.\n", - "27.\n", - "Sensitive Services. Sensitive Services are those health care services. which are covered by more\n", - "restrictive confidential treatment rules. In accordance with the Medi-Cal Agreement, Medi-Cal Members may self-\n", - "refer anywhere to obtain Sensitive Services and no referral or prior authorization shall be required.\n", - "Access\n", - "to\n", - "Sensitive Services shall not be limited in any way geographically or by provider network participation.\n", - "The Sensitive Services are as follows:\n", - "abortion (pregnancy termination) services,\n", - "family planning services, inclusive of all methods of birth control covered by the Department of\n", - "Health Care Services for the Medi-Cal Program.\n", - "sexually transmitted disease testing and treatment, and\n", - "Human Immunodeficiency Virus (HIV) testing and counseling\n", - "28.\n", - "Vaccines for Children Program (VFC)\n", - "Provider shall not seek reimbursement from Health\n", - "Net for immunizations covered by the State of California under the Vaccines for Children Program (VFC), where a\n", - "Provider, or Professional Provider who is a participant in the VFC program, rendered the immunization.\n", - "29.\n", - "Local Health Department Coordination. As more fully set out in the Medi-Cal Agreement,\n", - "Health Net or a contracting Medi-Cal plan has (or will) entered into agreements for specified public health services\n", - "with certain county health departments (Los Angeles, Fresno, Tulare. Riverside, San Bemardino, San Diego. and\n", - "Sacramento counties). The public health agreements specify the scope and responsibilities of the local health\n", - "departments and Health Net. billing and reimbursements, reporting responsibilities, and medical record management\n", - "to ensure coordinated health care services. The public health services specified under the agreements are as follows:\n", - "29.1\n", - "Family planning services;\n", - "29.2\n", - "Sexually transmitted disease (\"STD\") services diagnosis and treatment of disease episode of the\n", - "following STDs: syphilis, gouorrhea, chlamydia, herpes simplex, chancroid, trichomoniasis,\n", - "human papilloma virus, non-gonococcal urethritis, lynphogranuloma venereum and granuloma\n", - "inguinale;\n", - "29.3\n", - "Confidential HIV testing and counseling;\n", - "29.4\n", - "Immunizations:\n", - "29.5\n", - "Child Health and Disability Prevention Program;\n", - "29.6\n", - "California Children Services;\n", - "29.7\n", - "Maternal and Child Health:\n", - "29.8\n", - "Refugee assessments;\n", - "29.9\n", - "Tuberculosis Direct Observed Therapy:\n", - "California Provider Participation Agreement\n", - "33\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 36\n", - "29.10\n", - "Women, Infants, and Children Supplemental Food Program:\n", - "29.11 Population based Prevention Programs: collaborate in local health department community based\n", - "prevention programs.\n", - "Provider shall, in accordance with the terms and conditions of the public health agreements with the local\n", - "health departments and Health Net's related policies and procedures, be responsible for the coordination and\n", - "arrangement of the public health services for its assigned Beneficiaries. The services specified in Sections 29.1\n", - "through 29.5 above require reimbursement to the applicable local health department. The services specified in\n", - "Sections 29.6 through 29.11 above do not require reimbursement to the applicable local health department.\n", - "[Medi-Cal Agreement]\n", - "California Provider Participation Agreement\n", - "34\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 37\n", - "EXHIBIT C-1\n", - "MEDI-CAL BENEFIT PROGRAM\n", - "DIRECT NETWORK FEE-FOR-SERVICE\n", - "RATE EXHIBIT\n", - "Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E,\n", - "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", - "Services delivered pursuant to this Addendum, the lesser of: 100% of the State of California Medi-Cal Fee Schedule\n", - "rates in effect at the time of service, subject to any adjustments made by the State of California under the applicable\n", - "Medi-Cal Fee-For-Service Program; (ii) Fee-for-service rates for the commercial Benefit Program set forth in\n", - "Addendum A, Exhibit A-1; or (iii) Provider's billed charges.\n", - "California Provider Participation Agreement\n", - "35\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 38\n", - "EXHIBIT C-2\n", - "DISCLOSURE FORM\n", - "(Required by California Welfare and Institutions Code Section 14452)\n", - "(Name of Provider)\n", - "The undersigned hereby certifies that the following information regarding:\n", - "(The \"Organization\") is true aud correct as of the date set forth below:\n", - "BagAllergy Asthma & Center, Inc\n", - "Officers/Directors/General Partners:\n", - "Malik N. Bag , M.D.\n", - "CHairman\n", - "Praveen Buddiga, M.D CEO\n", - "Co-Owner(s):\n", - "N/A\n", - "Stockholders owning more than ten percent of the stock of the Organization:\n", - "N/A\n", - "Major creditors holding more than five percent of Organization's debt:\n", - "NA\n", - "Form of Organization (Corporation, Partnership, Sole Proprietorship, Individual, etc.):\n", - "N/A\n", - "If not already disclosed above, is Organization, either directly or indirectly related to or affiliated with the\n", - "Contracting Health Plan? Please explain:\n", - "Dated: 5-17-2011\n", - "Signature:\n", - "Jay\n", - "Name: Praveen Buddga,MD.\n", - "(Please type or print)\n", - "Title:\n", - "CEO\n", - "(Please type or print)\n", - "California Provider Participation Agreement\n", - "Fee-For-Service Direct Network Template\n", - "36\n", - "HN-DN-PPA.-04-08-11\n", - "\n", - "Start of Page No. = 39\n", - "ADDENDUMI\n", - "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS\n", - "This Addendum D applies to Covered Services delivered to Beneficiaries covered by a Healthy Families Healthy\n", - "Kids or related benefit program. The term \"Beneficiary\" as used in this Addendum D shall be deemed to apply only\n", - "to Beneficiaries who are covered by a Healthy Families or a Healthy Kids Benefit Program. All Covered Services\n", - "delivered to such a Beneficiary shall be paid in accordance with this Addendum D regardless of product specific\n", - "name unless otherwise specifically agreed by the parties and set forth in a separate rate exhibit.\n", - "A.\n", - "COMPENSATION PROVISIONS\n", - "1.\n", - "Compensation and Applicability. Provider shall arrange and provide Contracted Services to\n", - "Beneficiaries of the Healthy Families and Healthry Kids Benefit Program covered under this Addendum on a fee-for-\n", - "service basis. As compensation for providing such Contracted Services, Provider shall be paid in accordance with\n", - "the rates set forth on Exhibit D-1, subject to the payment conditions set forth in Addendum E. This Addendum shall\n", - "be applicable to only those Healthy Families and Healthy Kids listed on the applicable remittance summaries. Health\n", - "Net will modify this Addendum to reflect a new rate structure for adults, pending regulatory approval of expanding\n", - "this program to parents. Provider shall submit claims for such services in accordance with the terms of this\n", - "Agreement and applicable State and federal law.\n", - "B.\n", - "GENERAL PROVISIONS\n", - "1.\n", - "Healthy Families Benefit Program. Health Net entered into an agreement with the California\n", - "Managed Risk Medical Insurance Board (\"MRMIB\") to arrange for the provision of Covered Services to persons\n", - "who are eligible under the California Children's Health Insurance Program \"Healthy Families Benefit Program\")\n", - "and enrolled in Health Net's Healthy Families Plan (\"Beneficiaries\"). Health Net arranges the Healthy Families\n", - "Program in certain California counties as a Health Maintenance Organization (\"HMO\") or an Exclusive Provider\n", - "Organization (\"EPO\"), as applicable to each county.\n", - "2.\n", - "Healthy Kids Benefit Program. Health Net participates in the Healthy Kids Programs (\"Healthy\n", - "Kids Benefit Program\") offered in certain California counties to provide medical coverage to children who qualify\n", - "for and are enrolled in the Healthy Kids Benefit Program in such California counties (\"Beneficiaries\"). Such\n", - "California counties have arranged for the provision of Covered Services to children who are eligible under the\n", - "individual county's program, collectively referred to herein as the Children's Health Insurance Collaborative\n", - "(\"CHIC\"). Health Net arranges the Healthy Kids Program in certain California counties as a Health Maintenance\n", - "Organization (\"HMO\") or an Exclusive Provider Organization (\"EPO\"). as applicable to each county.\n", - "3.\n", - "Participation and Provision of Covered Services. Provider understands and agrees that il shall\n", - "arrange and/or provide Covered Services to Beneficiaries in accordance with the terms and conditions of this\n", - "Agreement, applicable law, and the requirements of the Healthy Families and Healthy Kids Benefit Programs.\n", - "Provider understands that Benefit Program evidence of coverage documents are subject to review and approval by\n", - "appropriate regulatory or oversight entities including MRMIB, the California Department of Managed Health Care\n", - "and/or the CHIC.\n", - "4.\n", - "Carve-ont of California Children's Services (CCS) Program Services. The parties\n", - "acknowledge that health care services rendered to eligible Beneficiaries to treat CCS-eligible conditions are \"carved\n", - "out\" of Health Net's coverage obligations under the Healthy Families and Healthy Kids Benefit Programs. Provider\n", - "shall identify and timely refer Beneficiaries with possible CCS-eligible conditions to the appropriate County CCS\n", - "Program. Provider may ask Health Net to assist with this referral. Health Net shall also make a referral to CCS when\n", - "a Primary Care Physician refers a beneficiary to a specialist or where there is an inpatient admission which appears\n", - "to involve care for a CCS eligible condition. The CCS program will determine if the beneficiary's condition is\n", - "California Provider Participation Agreement\n", - "37\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 40\n", - "eligible for CCS services. Upon referral, Provider shall inform the Beneficiary's parent or guardian and shall notify\n", - "Health Net of any such referral. The CCS Program requires eligible children to be treated at CCS-certified facilities\n", - "by CCS-paneled providers. The CCS Program may require transfer to CCS-certified facilities with CCS-paneled\n", - "providers. The CCS Program is financially responsible for payment of health care costs to treat a CCS-eligible\n", - "condition. The parties understand and agree that Health Net is not financially responsible for payment of services to\n", - "treat eligible beneficiaries for CCS-eligible conditions. In the event Health Net inadvertently pays a claim for such\n", - "services, Health Net may recover the amount paid pursuant to Section 4.3 of the Agreement.\n", - "5.\n", - "Referral of Beneficiaries having possible mental health conditions to Managed Health\n", - "Network. Provider is required to identify and timely refer Beneficiaries with possible mental health conditions\n", - "(other than Severely Emotionally Disturbed as set out in the following section) to Health Net's affiliate and\n", - "subcontractor, Managed Health Network. Inc. Managed Health Network. Inc., is financially responsible for\n", - "payment of treatment of covered mental health services.\n", - "6.\n", - "Services for Severely Emotionally Disturbed (SED) Beneficiaries. For beneficiaries who are SED.\n", - "diagnosis and treatment are provided by the County Mental Health Department Health Net and the county mental\n", - "health department will coordinate to arrange or provide services so that all medically necessary services and\n", - "treatment are provided to Beneficiaries who are SED. Health care services to treat Beneficiaries who are SED are\n", - "the financial responsibility of the County Mental Health Department with the exception of the first thirty (30) days\n", - "of inpatient mental health care. Provider shall identify and timely refer Beneficiaries with possible SED to the\n", - "County Mental Health Department. Upon referral, Provider shall also inform the Beneficiary's parent or guardian.\n", - "The County Mental Health Department is exclusively responsible for the provision and payment of health care costs\n", - "to treat Beneficiaries who have been diagnosed as SED, with the exception of the provision and payment of health\n", - "care costs for the first thirty (30) days of inpatient mental health care. The Healthy Families Benefit Program defines\n", - "a Beneficiary as Severely Emotionally Disturbed (SED), if that beneficiary meets the criteria set forth in California\n", - "Welfare and Institutions Code 5600.3.\n", - "7.\n", - "Reports and Information. Provider shall provide to Health Net. within the time requested by\n", - "Health Net, all such reports and information as Health Net may require so that Health Net can meet the reporting\n", - "requirements hunder the Healthy Families, Healthy Kids or related benefit programs, and applicable law. Such reporting\n", - "obligations include, but are not limited to, monthly reporting of Beneficiary referrals to the following programs:\n", - "California Children's Services, referrals to Managed Health Network, and referrals of Beneficiaries with a diagnosis\n", - "of SED to the County Mental Health Department.\n", - "8.\n", - "Cultural and Linguistic Services. Provider shall: (1) not require or encourage Beneficiaries to\n", - "utilize family Beneficiaries or friends as interpreters; (2) record the language needs of Beneficiaries in the medical\n", - "record; and (3) document Beneficiary requests or refusals of interpreter services in the Beneficiary's medical record.\n", - "Provider shall arrange interpreter services for Beneficiaries either through telephone language services or face-to-\n", - "face interpreters. Provider is encouraged to directly make these interpretive services available. However, upon\n", - "request, Health Net's Member Services Department is available to provide certain interpretive assistance to facilitate\n", - "communications.\n", - "10.\n", - "Eligibility. Eligibility and commencement of enrollment under Healthy Families Benefit Program\n", - "is determined by MRMIB. Commencement of coverage for Beneficiaries can occur at any day of a month.\n", - "Eligibility and commencement of enrollment under Healthy Kids Benefit Program is determined by the appropriate\n", - "Healthy Kids county Children's Health Collaborative CIII, or its successor.\n", - "11.\n", - "Copayments. Copayments are subject to an animal limitation. Provider is encouraged to make\n", - "extended payment arrangements available to Beneficiaries experiencing an inability to pay a required copayment.\n", - "California Provider Participation Agreement\n", - "38\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-03-11\n", - "\n", - "Start of Page No. = 41\n", - "EXHIBIT D-1\n", - "HEALTHY FAMILIES AND HEALTHY KIDS BENEFIT PROGRAMS\n", - "DIRECT NETWORK FEE-FOR-SERVICE\n", - "RATE EXHIBIT\n", - "Subject to the terms of this Agreement, including without limitation the Payment Conditions sei forth in Addendum E.\n", - "Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered\n", - "Services delivered to Beneficiaries under the Healthy Families and Healthy Kids Benefit Program pursuant to this\n", - "Addendum the lesser of 75% of Provider's Billed Charges or:\n", - "(a) 120% of the State of California Medi-Cal Fee Schedule rates in effect at the time of service, subject to\n", - "any adjustments made by the State of California under the applicable Medi-Cal Fee-For-Service Program;\n", - "or\n", - "(b) for those \"by report procedures, and/or unlisted procedures and relativities that are not established on\n", - "the State of California's Medi-Cal Fee Schedule:\n", - "80% of the CMS allowable.\n", - "For those \"By report\" procedures, and/or unlisted procedures and relativities that are not established on\n", - "both the State of California's Medi-Cal Fee Schedule or the CMS participating provider fee schedule for physician's\n", - "Locality, Health Net shall pay 60% of Provider's billed charges.\n", - "California Provider Participation Agreement\n", - "39\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n", - "\n", - "Start of Page No. = 42\n", - "ADDENDUMI\n", - "DIRECT NETWORK FEE-FOR-SERVICE PAYMENT CONDITIONS\n", - "The Payment Conditions set forth in this Addendum supplement Health Net Policies, and are applicable to\n", - "Commercial Benefit Programs, Medicare Advantage Benefit Program, Medi-Cal Benefit Program and the Healthy\n", - "Families and Healthy Kids Benefit Programs.\n", - "1.\n", - "Provider shall utilize valid CPT/HCPCS/ICD9 diagnosis_codes, or successor codes, when\n", - "submitting Complete Claims for Covered Services under this Agreement. The parties acknowledge that applicable\n", - "coding agencies periodically issue coding modifications. Such modifications may be implemented by Health Net\n", - "within sixty (60) days of the date Health Net receives the modification from the applicable coding agency. The\n", - "parties agree that in the event such coding modifications have the effect. of changing a payment amount in this\n", - "Agreement, the resulting payment amount change shall be effective on a prospective basis. The parties further agree\n", - "to reasonably and in good faith discuss any contract rate amendment that may be appropriate based on\n", - "comprehensive and substantive coding changes by the applicable coding agency within ninety (90) days of written\n", - "notification by either party. Any and all rate modifications that may result from such contract amendment shall be\n", - "effective on a prospective basis.\n", - "2.\n", - "Pharmaceuticals shall include inhaled/infused/injected medications provided by. and delivered in\n", - "PPG/Professional Provider office. Support drugs and injection supplies for patient self-administered injections shall\n", - "be supplied only through a Health Net preferred specialty pharmacy provider. AWP pricing shall be determined by\n", - "Health Net utilizing a nationally recognized source for pharmaceutical pricing to be updated at least quarterly.\n", - "Provider shall bill administered dosage in accordance with applicable HCPCS codes. Multi-dose vials shall be\n", - "reimbursed on a per dose basis.\n", - "3.\n", - "Immunizations shall include procedure codes for immune globulins, vaccines and toxoids that\n", - "identify the product only. The administration for immune globulins, vaccines and toxoids are separate from the\n", - "product itself and are not considered immunizations. AWP pricing shall be determined by Health Net utilizing a\n", - "nationally recognized source for pharmaceutical pricing to be updated at least quarterly. Provider shall bill\n", - "administered dosage in accordance with applicable HCPCS codes. Multi-dose vials shall be reimbursed on a per\n", - "dose basis.\n", - "1.\n", - "The rates set forth in this Agreement apply to all current and future locations billed under this and\n", - "future Tax Identification Numbers indicated by Provider through a signed W-9 form and subject to terms of this\n", - "Agreement.\n", - "California Provider Participation Agreement\n", - "40\n", - "Fee-For-Service Direct Network Template\n", - "HN-DN-PPA-04-08-11\n" - ] - } - ], - "source": [ - "print(table_edit['Baz Allergy Asthma & Sinus_MU.txt'])" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "def extract_all_tables(input_dict):\n", - " tables_dict = {}\n", - " for filename, filetext in input_dict.items():\n", - " tables_dict[filename] = extract_tables(filetext)\n", - " return tables_dict\n", - "\n", - "t = extract_all_tables(input_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Baz Allergy Asthma & Sinus_MU.txt': []}" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "t" + "print(text_dict['19'])" ] }, { diff --git a/src/utils.py b/src/utils.py index 2f78503..e05e140 100644 --- a/src/utils.py +++ b/src/utils.py @@ -96,13 +96,13 @@ def preprocess_text_file(file_path): return pages -def format_td_check(td_dicts): +def format_td_check(td_dicts, dont_include_list): final_str = "" dict_count = 1 for td_dict in td_dicts: final_str += str(dict_count) + '. ' for k in td_dict.keys(): - if k not in ['Filename', 'page_num']: + if k not in dont_include_list: final_str += k + ': ' + td_dict[k] + ', ' final_str += '\n' dict_count += 1 From 576ef45a35e7956bf5bb2659621bc0207eaf6292 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 22 May 2024 13:37:50 -0500 Subject: [PATCH 26/78] Modified output, modified LOB prompt --- src/config.py | 10 +- src/main.py | 2 +- src/prompts.py | 4 +- src/test_notebook.ipynb | 246 ++-------------------------------------- 4 files changed, 19 insertions(+), 243 deletions(-) diff --git a/src/config.py b/src/config.py index d7e76b7..42d0d3c 100644 --- a/src/config.py +++ b/src/config.py @@ -13,7 +13,7 @@ TODAY = datetime.now().strftime("%Y%m%d") # I/O Options WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written READ_MODE = '_LOCAL_' # OR '_S3_' -OUTPUT_MODE = '_INDIVIDUAL_' # or'_CONSOLIDATED_' +OUTPUT_MODE = '_INDIVIDUAL_' # or '_CONSOLIDATED_' # Multithread Settings MAX_WORKERS = 20 @@ -28,12 +28,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXC5Y4PKXJ" -AWS_SECRET_ACCESS_KEY="f/mOlRpK7+iSlm1P5ZK1LOdS2UjnsbAQ2ysL7Mms" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEM3//////////wEaCXVzLWVhc3QtMiJGMEQCIGqPr40CU3Fz4nvSQJ1jaPgkOGIWWyDXBVLUU+VmsfIjAiAgluQOwSGIYph5hLcKVVsiFQ7OpLOs3lLsumxzeyw6nyqDAwhGEAAaDDY2MDEzMTA2ODc4MiIM5odYPf2eQP8/f9NBKuAC7eMH0qjb6ASoeBhtKNFzMWXpR5snAahfBheyiUo1my8q8VFbZem8m0n5c0vWnKsPhm9+K+qpVqM6ohhjVMIXCta/ZlGNizatHJt7gMpDOGQNyBI4rpXF9/urQMgN8PkjMG2hkD+XefkPYGmvgz9ZVKFs/oAnP/buG21Ju54F/D0qqnxhR/90pVM6Z48sgzkQThHHjB7wNgfy1hIA7CL9yeM0k+sdzlYvriDogDi3lcP3UkXMUJhQl5xQb7t8iLR70L/0H74ynfldpbaBfktWPred4ehoRMv2Zd/Jh6JNBIAeDQyfIKQTYn2FF/RuPBmqa+iLS9lJh/SnhwXNuMYc9JOERwMHaijgqQjenISnysSEr10u1Y+XiUI7waXRO6e8CBS/LygUKZZf8+cSeXIE3VN2XrMW72Ix5sg16QHAMvR3oUDJdYdf8jnWET995kocK6UctJdVKEna/FUnIdNgwzDirrKyBjqnAZjoQpVVuEh8p9tuGrjdWsjzLsXRkNRyqm/6xxIGtqITzvAv+TKQLSgJun/rRTOncaMLDwt0PAKL1Y8cVnOcuSIfVFUtpeZ+p27MgByL/fBQGd0yfZGHy15ZMPw51viYhjQo3MeA4y0YLRW+12v2ilrMxGHYlwTUT19Zj+qQNobTrlcJWQ7737lXmVL2WOZzmtduH7wf/fyAn2ZhZT/lcnSSQsef9ORk" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXDI7HGA7Q" +AWS_SECRET_ACCESS_KEY="mXAJEp6Ar35G+VEwUfKq/3cu+1Hn721R+pNs+Fxh" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEOj//////////wEaCXVzLWVhc3QtMiJHMEUCIQDlpQWz84O6hWg0qtTjfoNx1nIt3Ip82gSG/jdYh05Z/AIgK3N9ViVt3ez4iwTbo7hwOgtnTbeK6Jn5fpCRfcHDv2QqgwMIYhAAGgw2NjAxMzEwNjg3ODIiDC3AfcU4Q5o7CB6tkCrgAjwqqjf8jXvnSMmnxvYmhqy4IwD/N4hyCIwYSHC1iGMOp9QsFyBVKixjAEl6m9dzt24XomQCY/GSoVNchUbqewuFIy5im3nyKlF/tGYj/xi261JTIr0szXk5UDw5twCXHSSdem1PII+C3EmsGl6fLjRFbPNlXi6vY3Z/X6nnhix/J6ZJQzPg5Plj6jGY9uBrtE/CF5DYtfGRPvFT13ZiI+UYJ/97viOM5JOWcT1nHoVY8DCLWicRCZO4aXbrOUs8QtRNns7oJye1sL1mU+0ocgkrXCYOnX54McYj4dViAHy3UUYo9w9u6a2LeW2MfQjvKam0W+q2iHuLs1ydn8ip7WjDzmBzLTAKZPWnPTjKZXlCGUsPXTj7U1rF0wNgBmIpOgMO7d49TsbSL0MWifCXuOF4sgu5ICKt5bcd7N0rvGgQR3I/kAY7abH60EAl37tbhP79tLmPt//hlR4xJ2lKuD0wrrq4sgY6pgEsIEApALn8eIykqKZPFf5jz1QWbhuScAy3Yjh+OjOvk29eUERS3xvIeOgYUF40qlkPboFkCRY3NNgvHLkbfg9I9m929zKk6Tugb6S30Aiqy9prxpPNvIA9/X877EzIb0xoIbxJHb8ucvW1xBkUNEwwZGlA6xbzKXqTHau7/j6odzLd/CPno+IQ9lCBSilCbg/1W1NAiDZC4IwJjpV8rHONSaXVdA44" # File Paths -LOCAL_PATH = 'docs/test/' # Replace with local +LOCAL_PATH = 'docs/priority_health/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/main.py b/src/main.py index 744f71c..e9c1f79 100644 --- a/src/main.py +++ b/src/main.py @@ -55,7 +55,7 @@ def main(): # Write Output if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': - consolidated_df = utils.consolidate_individual(folder='temp') # Consolidate temp results to one file, then write output + consolidated_df = utils.consolidate_individual(input_folder='temp') # Consolidate temp results to one file, then write output if __name__ == "__main__": diff --git a/src/prompts.py b/src/prompts.py index 14bce4c..86871dd 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -14,7 +14,7 @@ Return at least one json object for each combination of attributes seen. It is p Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. -'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is direct dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. +'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. @@ -130,7 +130,7 @@ Here are the attributes to be included in each dictionary, and instructions on h 'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. -If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return them in a comma-separated list. +If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return a separate dictionary for them such that each of the four attributes is associated with the other attributes in the dictionary. For all attributes, only write what is written on the page. Do not make up new phrases or words. diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb index 6e9343e..fada209 100644 --- a/src/test_notebook.ipynb +++ b/src/test_notebook.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -19,250 +19,26 @@ "import claude_funcs\n" ] }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# Upload \n", - "input_dict = utils.read_input()" - ] - }, { "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [], - "source": [ - "filename = 'Baz Allergy Asthma & Sinus_MU.txt'\n", - "contract_text = input_dict['Baz Allergy Asthma & Sinus_MU.txt']\n", - "\n", - "contract_text = preprocess.clean_newlines(contract_text)\n", - "text_dict = preprocess.split_text(contract_text)\n", - "text_dict = table_funcs.align_and_format_tables(text_dict)\n", - "text_dict = preprocess.highlight_rates(text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "[{\"CONTRACT_LOB\": \"Commercial, Medicare Advantage, Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service, Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[\n", - " {\n", - " \"CONTRACT_LOB\": \"N/A\",\n", - " \"CONTRACT_PROGRAM\": \"N/A\",\n", - " \"CONTRACT_NETWORK\": \"FFS\",\n", - " \"PRODUCT\": \"Health Net\"\n", - " }\n", - "]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[\n", - " {\n", - " \"CONTRACT_LOB\": \"Commercial\",\n", - " \"CONTRACT_PROGRAM\": \"N/A\",\n", - " \"CONTRACT_NETWORK\": \"HMO\",\n", - " \"PRODUCT\": \"Health Net\"\n", - " }\n", - "]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"N/A\"}]\n", - "[{\"CONTRACT_LOB\": \"Commercial\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO, PPO, EPO, POS\", \"PRODUCT\": \"AIM, leased networks\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"PPO, EPO, POS, Leased PPO\", \"PRODUCT\": \"N/A\"}]\n", - "[{\"CONTRACT_LOB\": \"Commercial\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Direct Network, Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicare Advantage\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Direct Network, Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Health Net, CalViva Health, Medi-Cal\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net Medi-Cal\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Health Net, Medi-Cal HMO\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Medi-Cal, Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"HMO\", \"PRODUCT\": \"Medi-Cal, Healthy Families\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"CHDP (Children's Health and Disability Prevention) Program, CCS (California Children's Services), CPSP (Comprehensive Perinatal Services Program), Vaccines for Children Program (VFC)\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service\", \"PRODUCT\": \"Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Medi-Cal Benefit Program, Health Net\"}]\n", - "[{\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"Fee-For-Service Direct Network\", \"PRODUCT\": \"N/A\"}]\n", - "[\n", - " {\n", - " \"CONTRACT_LOB\": \"Medicaid\",\n", - " \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids\",\n", - " \"CONTRACT_NETWORK\": \"HMO, EPO\",\n", - " \"PRODUCT\": \"Healthy Families Plan\"\n", - " }\n", - "]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Health Net, Managed Health Network\"}]\n", - "[{\"CONTRACT_LOB\": \"Medicaid\", \"CONTRACT_PROGRAM\": \"Healthy Families, Healthy Kids Benefit Program\", \"CONTRACT_NETWORK\": \"Direct Network, FFS\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"Medi-Cal Fee-For-Service Program, CMS\"}]\n", - "[{\"CONTRACT_LOB\": \"Commercial, Medicare Advantage, Medicaid, Healthy Families, Healthy Kids\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"FFS\", \"PRODUCT\": \"Health Net\"}, {\"CONTRACT_LOB\": \"N/A\", \"CONTRACT_PROGRAM\": \"N/A\", \"CONTRACT_NETWORK\": \"N/A\", \"PRODUCT\": \"N/A\"}]\n" + "ename": "FileNotFoundError", + "evalue": "[WinError 3] The system cannot find the path specified: 'results'", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconsolidate_individual\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[1;32mc:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\src\\utils.py:57\u001b[0m, in \u001b[0;36mconsolidate_individual\u001b[1;34m(input_folder, output_folder)\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconsolidate_individual\u001b[39m(input_folder\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresults\u001b[39m\u001b[38;5;124m'\u001b[39m, output_folder\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutput\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m 56\u001b[0m dfs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m---> 57\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m filename \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_folder\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[0;32m 58\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m filename\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.csv\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m 59\u001b[0m filepath \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(input_folder, filename)\n", + "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: 'results'" ] } ], "source": [ - "import prompts\n", - "import dict_operations\n", - "\n", - "def run_top_down(filename, text_dict):\n", - " all_results = []\n", - "\n", - " # Primary\n", - " for page_num, page_text in text_dict.items():\n", - " if not page_num.isdigit():\n", - " continue\n", - " prompt = prompts.TOP_DOWN_PRIMARY(page_text)\n", - " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", - " answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries\n", - " all_results.append(answer_dicts)\n", - " td_primary = [i for d in all_results for i in d] # Consolidate to one list\n", - "\n", - " # Secondaries\n", - " \n", - "\n", - " return td_primary # List of list of dictionaries" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [], - "source": [ - "def format_td_check(td_dicts, dont_include_list):\n", - " final_str = \"\"\n", - " dict_count = 1\n", - " for td_dict in td_dicts:\n", - " final_str += str(dict_count) + '. '\n", - " for k in td_dict.keys():\n", - " if k not in dont_include_list:\n", - " final_str += k + ': ' + td_dict[k] + ', '\n", - " final_str += '\\n'\n", - " dict_count += 1\n", - " return final_str\n", - "\n", - "def TOP_DOWN_METAL_LEVEL(LOB, page):\n", - " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", - " \n", - "The above page was identified as applying to a Commercial or Marketplace Line of Business. \n", - "\n", - "What is the metal level(s) of the line of business. It might be Bronze, Silver, Gold, or Platinum. If multiple are found, return them all in a comma-separated list. If no metal level is found, return 'N/A'.\n", - "\n", - "Only write what is written on the page. Do not make up new phrases or words. \n", - "\n", - "ONLY return the metal level, with no other commentary or explanation.\n", - "\"\"\"\n", - "\n", - "\n", - "def TOP_DOWN_DATE(type_, d, page):\n", - " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", - "\n", - "Above is a page of a contract. Identify the {type_} Date associated with the information below:\n", - "\n", - "{d}\n", - "\n", - "Only write what is written on the page. Do not make up new phrases or words. If no {type_} date is listed, return 'N/A'.\n", - "\n", - "Ensure that you do not mistake EFFECTIVE date for TERMINATION date, or vice versa. They are not the same thing. \n", - "The TERMINATION date will NOT be labelled as the EFFECTIVE date, and the EFFECTIVE date will NOT be labelled as the TERMINATION date. This is very important, so pay extra close attention.\n", - "\n", - "ONLY return the answer, with no other commentary or explanation.\n", - "\"\"\"\n", - "\n", - "\n", - "def run_top_down_metal_level(d, page):\n", - " if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper():\n", - " prompt = TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page)\n", - " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", - " else:\n", - " answer = 'N/A' \n", - " return answer\n", - "\n", - "def run_top_down_date(type_, d, page):\n", - " formatted_d = format_td_check([d], ['Filename', 'page_num'])\n", - " prompt = TOP_DOWN_DATE('EFFECTIVE', formatted_d, page)\n", - " answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", - " return answer\n", - "\n", - "\n", - "def top_down_secondary(td_results, text_dict):\n", - " updated_dicts = []\n", - " for d in td_results:\n", - " # Run Metal Level\n", - " d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']])\n", - " \n", - " # Dates\n", - " d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']])\n", - " d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']])\n", - "\n", - " updated_dicts.append(d)\n", - " return updated_dicts\n", - "\n", - "td_dicts = top_down_secondary(td_results, text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 53, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'CONTRACT_LOB': 'N/A',\n", - " 'CONTRACT_PROGRAM': 'N/A',\n", - " 'CONTRACT_NETWORK': 'Fee-For-Service Direct Network',\n", - " 'PRODUCT': 'Health Net',\n", - " 'page_num': '19',\n", - " 'Filename': 'Baz Allergy Asthma & Sinus_MU.txt',\n", - " 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A',\n", - " 'LOB_PRICING_TERMS_EFFECTIVE_DATE': '10-6-2011',\n", - " 'LOB_PRICING_TERMS_TERMINATION_DATE': '10-6-2011'}" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "td_dicts[18]" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "19 7.14 Status as Independent Entitles. None of the provisions of this Agreement is intended to create, nor shall be deemed or construed to create any relationship between Provider and Health Net or a Payor other than that of independent entities contracting with each other solely for the purpose of effecting the provisions of this Agreement. Neither Provider nor Health Net/Payor, nor any of their respective agents, employees or representatives shall be construed to be the agent, employee or representative of the other. 7.15 Addenda. Each Addendum to this Agreement is made a part of this Agreement as though set forth fully herein. Any provision of an Addendum that is in conflict with any provision of this Agreement shall take precedence and superseda the conflicting provision of this Agreement with respect to the subject matter of the Addendum. 7.16 Calculation of Time. The parties agree that for purposes of calculating time under this Agreement, any time period of less than ten (10) days shall be deemed to refer to business days and any time period of ten (10) days or more shall be deemed to refer to calendar days unless the term \"business\" precedes the term \"days\". 7.17 Waiver of Breach. The waiver of any breach of this Agreement by either party shall not constitute a continuing waiver of any subsequent breach of either the same or any other provision(s) of this Agreement. Further, any such waiver shall not be construed to be a waiver on the part of such party to enforce strict compliance in the future and to exercise any right or remedy related thereto. THIS CONTRACT CONTAINS A BINDING ARBITRATION CLAUSE, WHICH MAY BE ENFORCED BY THE PARTIES, IN WITNESS WHEREOF, the parties have executed this Agreement. PROVIDER Love HEALTHNE Signature Obly Signature Hoens Health Malik N. BAZ , M.D. Cathy Hoens Print Name Print Name Chairman Title VP Provider Network Management & Strategy Title Baz Allergy Asthma & Sinus Group Name (If Applicable) LIR. 6/29/2011 Date 77-0508441 7/15/11 Tax Identification Number 10-6-2011 Effective Date Date California Provider Participation Agreement Fee-For-Service Direct Network Template 18 HN-DIN-PPA-04-08-11 This page has 2 signature.\n" - ] - } - ], - "source": [ - "print(text_dict['19'])" + "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" ] }, { From 548c7d1400888f4246203f22c6c9de1cb4a2db45 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 23 May 2024 14:46:51 -0500 Subject: [PATCH 27/78] Updated table preprocessing --- src/config.py | 2 +- src/preprocess.py | 1 - src/table_funcs.py | 74 ++++++++-------- src/test_notebook.ipynb | 184 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 209 insertions(+), 52 deletions(-) diff --git a/src/config.py b/src/config.py index 42d0d3c..0c99fc7 100644 --- a/src/config.py +++ b/src/config.py @@ -33,7 +33,7 @@ AWS_SECRET_ACCESS_KEY="mXAJEp6Ar35G+VEwUfKq/3cu+1Hn721R+pNs+Fxh" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEOj//////////wEaCXVzLWVhc3QtMiJHMEUCIQDlpQWz84O6hWg0qtTjfoNx1nIt3Ip82gSG/jdYh05Z/AIgK3N9ViVt3ez4iwTbo7hwOgtnTbeK6Jn5fpCRfcHDv2QqgwMIYhAAGgw2NjAxMzEwNjg3ODIiDC3AfcU4Q5o7CB6tkCrgAjwqqjf8jXvnSMmnxvYmhqy4IwD/N4hyCIwYSHC1iGMOp9QsFyBVKixjAEl6m9dzt24XomQCY/GSoVNchUbqewuFIy5im3nyKlF/tGYj/xi261JTIr0szXk5UDw5twCXHSSdem1PII+C3EmsGl6fLjRFbPNlXi6vY3Z/X6nnhix/J6ZJQzPg5Plj6jGY9uBrtE/CF5DYtfGRPvFT13ZiI+UYJ/97viOM5JOWcT1nHoVY8DCLWicRCZO4aXbrOUs8QtRNns7oJye1sL1mU+0ocgkrXCYOnX54McYj4dViAHy3UUYo9w9u6a2LeW2MfQjvKam0W+q2iHuLs1ydn8ip7WjDzmBzLTAKZPWnPTjKZXlCGUsPXTj7U1rF0wNgBmIpOgMO7d49TsbSL0MWifCXuOF4sgu5ICKt5bcd7N0rvGgQR3I/kAY7abH60EAl37tbhP79tLmPt//hlR4xJ2lKuD0wrrq4sgY6pgEsIEApALn8eIykqKZPFf5jz1QWbhuScAy3Yjh+OjOvk29eUERS3xvIeOgYUF40qlkPboFkCRY3NNgvHLkbfg9I9m929zKk6Tugb6S30Aiqy9prxpPNvIA9/X877EzIb0xoIbxJHb8ucvW1xBkUNEwwZGlA6xbzKXqTHau7/j6odzLd/CPno+IQ9lCBSilCbg/1W1NAiDZC4IwJjpV8rHONSaXVdA44" # File Paths -LOCAL_PATH = 'docs/priority_health/' # Replace with local +LOCAL_PATH = 'docs/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/preprocess.py b/src/preprocess.py index b65e356..90c33f3 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -49,4 +49,3 @@ def clean_billed_charges(text): else: return '100% of billed charges' return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) - diff --git a/src/table_funcs.py b/src/table_funcs.py index 7ae937d..ce29301 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -19,43 +19,43 @@ def extract_all_tables(input_dict): tables_dict[filename] = extract_tables(filetext) return tables_dict -def format_table(table): - table = table.replace('>>>', '').replace('<<<', '') - table_dict = eval(table) - - keys = list(table_dict.keys()) - # If values are lists - if isinstance(table_dict[keys[0]], list): - final_string = "" - for i in range(len(table_dict[keys[0]])): - for k in keys: - key_string = k + ': ' + table_dict[k][i] + ', ' - final_string += key_string - final_string += '|\n' - else: - # If values are not lists - final_string = "" - for k in keys: - key_string = k + ': ' + table_dict[k] + ', ' - final_string += key_string - final_string += '|\n' - return final_string + +def format_table(table_json): + table_text = "" + for i in range(len(table_json[list(table_json.keys())[0]])): + for key in table_json.keys(): + if table_json[key][i]: + table_text += key + ': ' + table_json[key][i] + ', ' + table_text += '\n' + return table_text def align_and_format_tables(text_dict): - pattern = re.compile(r'-------Table Start--------(.*?)-------Table End--------', re.DOTALL) - for page, text in text_dict.items(): - tables = pattern.findall(text) - for table in tables: - pre_table = table.split('{')[0].strip() # Get the pretable text for alignment - text = text.replace(table, '') # Remove full table - table_only = table.replace(pre_table, '') - try: - formatted_table = format_table(table_only) - except: - formatted_table = table - text = text.replace(pre_table, pre_table + formatted_table) # replace remaining pretable text with full table - text = text.replace('-------Table Start--------', '') - text = text.replace('-------Table End--------', '') - text_dict[page] = text - return text_dict + aligned_text_dict = {} + for key, text in text_dict.items(): + if 'Table Start' in text: + print(key) + table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) + for table_text in table_texts: + # Extract pretable text + pretable = table_text.split('{')[0].strip() + + # Extract and format table text + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table = json.loads(table_only) + table_formatted = format_table(table) + + # Align + if text.count(pretable) == 2: + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: + aligned_text = text.replace(table_text, pretable + table_formatted) + aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") + + else: + aligned_text_dict[key] = text + + return aligned_text_dict diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb index fada209..5695392 100644 --- a/src/test_notebook.ipynb +++ b/src/test_notebook.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -16,29 +16,187 @@ "import utils\n", "import preprocess\n", "import table_funcs\n", - "import claude_funcs\n" + "import claude_funcs" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { - "ename": "FileNotFoundError", - "evalue": "[WinError 3] The system cannot find the path specified: 'results'", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[3], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconsolidate_individual\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[1;32mc:\\Users\\kminhas\\Documents\\Doczy\\doczy.ai\\src\\utils.py:57\u001b[0m, in \u001b[0;36mconsolidate_individual\u001b[1;34m(input_folder, output_folder)\u001b[0m\n\u001b[0;32m 55\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconsolidate_individual\u001b[39m(input_folder\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mresults\u001b[39m\u001b[38;5;124m'\u001b[39m, output_folder\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moutput\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m 56\u001b[0m dfs \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m---> 57\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m filename \u001b[38;5;129;01min\u001b[39;00m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlistdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_folder\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[0;32m 58\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m filename\u001b[38;5;241m.\u001b[39mendswith(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.csv\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m 59\u001b[0m filepath \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(input_folder, filename)\n", - "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: 'results'" + "data": { + "text/plain": [ + "dict_keys(['2015-04-01 Marina Del Rey Hospital PPA.txt', '2017-12-01 Prime Healthcare-West Anaheim Medical Center Enhanced PPO AMD.txt', '2021-01-01 PH - Paradise Valley Hospital 7th AMD.txt', '2021-01-01 PH-West Anaheim Medical Center AMD.txt', '2023-01-01 UCSD Medical Center Non B&G AMD.txt', '2023-06-01 DH-Mercy Medical Center - Redding CDM LTR.txt', '47-1928508 Advanced Anesthesia.txt', '71-0986832 Pacific Gynecological Specialists.txt', 'Baz Allergy Asthma & Sinus.txt'])" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input_dict = utils.read_input(path='../docs/healthnet/')\n", + "input_dict.keys()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "contract_text = input_dict['2021-01-01 PH - Paradise Valley Hospital 7th AMD.txt']" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "#### PREPROCESS ####\n", + "contract_text = preprocess.clean_newlines(contract_text)\n", + "text_dict = preprocess.split_text(contract_text)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6\n", + "7\n", + "8\n", + "9\n", + "13\n", + "19\n", + "20\n" + ] + }, + { + "data": { + "text/plain": [ + "[None, None, None, None, None, None, None]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[print(k) for k in text_dict.keys() if 'Table Start' in text_dict[k]]" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Document': 'Document Index SEVENTH AMENDMENT 1TO THE 1PROVIDER PARTICIPATION AGREEMENT 1BETWEEN 1PARADISE VALLEY HOSPITAL 1AND 1HEALTH NET OF CALIFORNIA, INC. 1 EXHIBIT A-1 6COMMERCIAL BENEFIT PROGRAMS 6FEE-FOR-SERVICE RATE EXHIBIT 6 EXHIBIT C-1 8MEDI-CAL BENEFIT PROGRAMS 8FEE-FOR-SERVICE RATE EXHIBIT 8 EXHIBIT D-1 9ENHANCEDCARE PPO BENEFIT PROGRAMS 9FEE-FOR-SERVICE RATE EXHIBIT 9 EXHIBIT D-2 10ENHANCED PPO BENEFIT PROGRAMS 10FEE-FOR-SERVICE PAYMENT CONDITIONS 10 Pro-ration Calculation: 13 ADDENDUM E 14FEE-FOR-SERVICE PAYMENT CONDITIONS 14 2.5.4 18Adjusting and Pro-rating Contract Rates Examples. 18 Pro-ration Calculation: 19 EXHIBIT F-1 20COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS 20FEE-FOR-SERVICE RATE EXHIBIT 20 EXHIBIT F-2 21COMMUNITYCARE BENEFIT PROGRAMS 21HOSPITAL QUALITY PERFORMANCE PROGRAM 21\\n\\n\\n\\n',\n", + " '1': '1 SEVENTH AMENDMENT TO THE PROVIDER PARTICIPATION AGREEMENT BETWEEN PARADISE VALLEY HOSPITAL AND HEALTH NET OF CALIFORNIA, INC. The Provider Participation Agreement (\"Agreement\") dated September 1, 2013, as subsequently amended, between Prime Healthcare Paradise Valley, LLC, dba Paradise Valley Hospital (\"Provider\") and Health Net of California, Inc. on behalf of itself and the subsidiaries and affiliates of Health Net, LLC (formerly known as Health Net, Inc.) (collectively \"Health Net\"), is hereby further amended effective January 1, 2021. Provider and Health Net hereby agree to amend the Agreement as follows: 1. A new Section 1.29, ALLOWABLE CHARGES, is added to read as follows: Allowable Charges is defined as Provider\\'s billed charges for Contracted Services. 2. Section 2.9, PRIOR AUTHORIZATION/REFERRAL, shall be deleted and replaced to read as follows: When either Prior Authorization and/or a Referral is required for the rendition of a health care service, the receipt of the required Prior Authorization and/or the required Referral, each being separate and distinct requirements, is a prerequisite to payment of Complete Claims for Covered Services in addition to confirming eligibility prior to delivering service as required by this Agreement and Health Net Policies. Health Net (or its designee as applicable) may rescind or modify its Prior Authorization, in a manner consistent with Health Net Policies, based on the eligibility of the Beneficiary and whether the rendered service is a Covered Service. However, when Health Net or its designee issues a Prior Authorization for a specific service under a Benefit Program regulated by the California Department of Managed Health Care or the California Department of Insurance, Health Net (or its designee as applicable) shall not rescind or modify its Prior Authorization after Provider has rendered the specified and authorized service in good faith and pursuant to the terms of the Prior Authorization for any reason, including, but not limited to, Health Net\\'s subsequent rescission, cancellation, or modification of the Beneficiary\\'s contract or Health Net\\'s subsequent determination that it did not make an accurate determination of the Beneficiary\\'s eligibility; provided, however that this section shall not be construed to expand or alter benefits available to a Beneficiary under such Benefit Program. 3. Subsection 3.5.1, ACCESS BY HEALTH NET, shall be deleted and replaced to read as follows: As of the effective date of this Agreement, the following Health Net subsidiaries and affiliates may at their option access this Agreement: Health Net of California, Inc., Health Net Life Insurance Company, Health Net Community Solutions, Inc., California Health and Wellness Plan, Wellcare of California, Inc., Health Net of Arizona, Inc., Health Net Administrative Services of Arizona, Inc., Arizona Complete Health, Health Net Health Plan of Oregon, Inc., Health Net Insurance Services, Inc., Health Net Federal Services, LLC, Managed Health Network, Inc., MHN Government Services, Inc., and Network Providers, LLC. Health Net may periodically modify the Health Net subsidiaries and/or affiliates which may access this Agreement. To the extent Health Net allows a Health Net subsidiary or affiliate to access this Agreement, Health Net binds such subsidiaries and/or affiliates to the terms and conditions of this Agreement. INTENTIONALLY LEFT BLANK PV 2020 amd Page 1 of 22 HealthNet\\n\\n',\n", + " '2': '2 4. Section 5.1, TERM, shall be deleted and replaced to read as follows: The term of this Agreement shall commence on the Effective Date and shall continue through December 31, 2023. Either party may terminate this Agreement effective as of the end of the Initial Term by providing at least one hundred twenty (120) days prior written notice to the other party. This Agreement shall automatically renew for successive one (1) year periods (\"Renewal Terms\"). 5. Section 6.3, ACCESS TO RECORDS AND AUDITS BY HEALTH NET, shall be deleted and replaced to read as follows: Subject only to applicable State and federal confidentiality or privacy laws, Provider shall permit Health Net or its designated representative access to Provider\\'s Records, at Provider\\'s place of business in this State during normal business hours, in order to audit, inspect, review, perform chart reviews, and duplicate such Records unless Provider agrees to a remote audit of such records. If performed on site, access to Records for the purpose of an audit shall be scheduled at mutually agreed upon times, upon at least thirty (30) business days prior written notice by Health Net or its designated representative, but not more than sixty (60) days following such written notice. Provider shall attend an exit interview upon completion of the audit for the purpose of obtaining a mutually agreed upon reconciliation of the initial audit findings. Such exit interview shall be conducted at a mutually agreeable time at Provider\\'s place of business in this State during normal business hours upon at least ten (10) days prior written notice by Health Net or its designated representative, but not more than thirty (30) days following such written notice. In the event Provider fails to attend the scheduled exit interview, Provider shall be deemed to have accepted the audit findings. If the audit was performed remotely, such exit interview shall consist of Health Net or its designated representative sharing its audit findings with Provider via written or electronic communications as mutually agreed upon by both parties. Provider shall be allowed thirty (30) days to contest the audit results. If not contested within thirty (30) days then Provider shall be deemed to have accepted the audit findings. Except when Health Net or its designated representative requests Provider Records that are related to care management or claims, Provider may be reimbursed reasonable fees associated with the retrieval of Provider\\'s Records and or duplication and preparation of requested Provider Records. Such fees shall not be more than those provided pursuant to applicable State law, including California Health and Safety code Section 123110. Actions taken as a result of Audit findings shall include adjustment for late charges, overcharges and undercharges. Any such adjustments shall be the net amounts as reflected in the audit findings. Any payments owed by one party to the other as the result of an audit shall be paid within thirty (30) days of the exit interview for such audit. 6. Section 7.6, BINDING ARBITRATION, is deleted and replaced to read as follows: If the parties are unable to resolve a Dispute through the dispute resolution process set forth in Section 7.5, the parties agree that such Dispute shall be settled by final and binding arbitration, upon the motion of either party, under the appropriate rules of the AAA or JAMS, as agreed by the parties. Any Arbitrator must be either a judge, or an attorney licensed to practice law in the State of California, who is in good standing with the State Bar, and has at least ten (10) years of experience with health care matters and the arbitration of managed care disputes. The parties each understand and agree that the exhaustion of any Health Net internal appeals processes and the Meet and Confer Process set forth in Section 7.5 (i) hereof are conditions precedent to binding arbitration under this Section 7.6. Notwithstanding the foregoing, nothing contained herein is intended to require binding arbitration of disputes alleging medical malpractice between a Beneficiary and Provider or to Disputes between the parties alleging breaches of confidentiality of Beneficiary information, trade secret or intellectual property obligations. The arbitration shall be conducted in San Francisco, California or Los Angeles, California. The written demand PV 2020 amd Page 2 of 22\\n\\n',\n", + " '3': '3 shall contain a detailed statement of the matter and facts and include copies of all material documents supporting the demand. Except as provided below, arbitration must be initiated within one (1) year after the date the Dispute arose by submitting a written notice to the other party. For the purposes of filing for arbitration regarding a Dispute over Health Net\\'s alleged non- payment or underpayment of Complete Claims under this Agreement, the parties agree that an arbitration shall be filed within one (1) year from the date of Health Net\\'s notice of its final determination on Provider\\'s appeal if any, on such Complete Claim. The parties expressly agree that the deadlines to file arbitration set forth above shall not be subject to waiver, tolling, alteration or modification of any kind or for any reason except for fraud. The failure to initiate arbitration before such deadlines shall mean the complaining party shall be barred forever from initiating such proceedings. All such arbitration proceedings shall be administered by the AAA or JAMS, as agreed by the parties; however, the arbitrator shall be bound by applicable State and federal law, and shall issue a written opinion setting forth findings of fact and conclusions of law. The parties agree that the decision of the arbitrator shall be final and binding as to each of them. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction. The arbitrator shall have no authority to make material errors of law or to award punitive damages or to add to, modify, or refuse to enforce any agreements between the parties. The arbitrator shall make findings of fact and conclusions of law and shall have no authority to make any award, which could not have been made by a court of law. The party against whom the award is rendered shall pay any monetary award and/or comply with any other order of the arbitrator within sixty (60) days of the entry of judgment on the award. The parties waive their right to a jury or court trial. The parties recognize and agree that theirs is an ongoing business relationship, which may lead to sensitive issues with respect to the exchange of information related to any Dispute. The parties agree, therefore, to enter into such protective orders (including without limitation creating a category of discovery documents \"for attorney\\'s eyes only\" to the extent feasible given the nature of the evidence and the Dispute). All discovery information shall be used solely and exclusively for arbitration of the Dispute between the parties and may not be used for any other purpose. After the arbitration award becomes final, each party shall return or destroy all documents obtained from the other party during the course of the arbitration that are subject to a protective order, and within thirty (30) days of such date shall provide to the other party an officer\\'s certificate signed under penalty of perjury indicating that all such information has been returned or destroyed. In all cases submitted to arbitration, the parties agree to share equally the administrative fee as well as the arbitrator\\'s fee, if any, unless otherwise assessed by the arbitrator. The administrative fees shall be advanced by the initiating party subject to final apportionment by the arbitrator in this award. The parties agree that the content and decision of any arbitration proceeding shall be confidential unless disclosure is required by applicable State or federal statutes or regulations. The terms of Section 7.5 and Section 7.6 shall survive termination of this Agreement. INTENTIONALLY LEFT BLANK PV 2020 and Page 3 of 22 Health Net\\n\\n',\n", + " '4': '4 7. Section 7.12, NOTICE, is deleted and replaced to read as follows: Notices regarding the breach, term, termination or renewal of this Agreement shall be given in writing in accordance with this Section 7.12 and shall be deemed given five (5) days following deposit in the U.S. mail, postage prepaid. If sent by hand delivery, overnight courier, or facsimile, notices shall be deemed given upon documentation of delivery. All notices shall be addressed as follows: Health Net: Regional Network Director Health Net of California, Inc. 3131 Camino del Rio North, Suite 600 San Diego, CA 92108 Vice President and Deputy General Counsel Health Net of California, Inc. 21281 Burbank Blvd Woodland Hills, CA 91367 Provider: Paradise Valley Hospital 2400 E 4th Street National City, CA 91950 Attn: Hospital CEO Prime Healthcare Management, Inc. 3480 E. Guasti Road Ontario, CA 91761 Attn: General Counsel and Health Plan Operations The addresses to which notices are to be sent may be changed by written notice given in accordance with this Section. 8. Exhibit A-1, COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit A-1. 9. Exhibit C-1, MEDI-CAL FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit C-1. 10. Exhibit D-1, ENHANCED PPO BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit D-1. 11. Exhibit D-2, ENHANCED PPO BENEFIT PROGRAMS, FEE-FOR-SERVICE PAYMENT CONDITIONS, is deleted and replaced with the attached Exhibit D-2. 12. Addendum E, FACILITY FEE-FOR-SERVICE PAYMENT CONDITIONS, is deleted and replaced with the attached Addendum E. 13. Exhibit F-1, COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit F-1. INTENTIONALLY LEFT BLANK PV 2020 and Page 4 of 22 Health Net\\n\\n',\n", + " '5': '5 14. A new Exhibit F-2, COMMUNITYCARE BENEFIT PROGRAMS HOSPITAL QUALITY PERFORMANCE PROGRAM, is added as the attached Exhibit F-2. THIS AMENDMENT shall be deemed to be part of the Agreement and, except as modified herein, the Agreement is hereby reaffirmed and declared in full force and effect. IN WITNESS WHEREOF, the parties hereto have executed this Amendment by their officers duly authorized to be effective on the date and year first written above. PRIME HEALTHCARE PARADISE VALLEY LLC HEALTH NET OF CALIFORNIA, INC. dba Paradise Valley Hospital Digitally signed by Yina Shabamia Valentina Shabanian Date: 2020.12.11 Edual dz 16:47:59 -08\\'00\" Edward Gong Valentina T. Shabanian Regional Vice President, Managed Care Regional Health Plan Officer 11/30/2020 Date Date Federal Tax Identification Number: 20-5837239 PV 2020 amd Page 5 of 22 Health Net\\n\\n',\n", + " '6': \"6 EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C-Section and Vaginal Delivery (Mom and Baby), Codes: MS-DRGs: 768, 783-788, 796-798, 805-807, 1/1/21 - 12/31/21: $12,600/3 day case rate. Additional days $4,000/day, 1/1/22 - 12/31/22: $13,104/3 day case rate. Additional days $4,160/day, 1/1/23 and thereafter: $13,628/3 day case rate. Additional days $4,326/day, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes: 0170, 0171, 0179, 1/1/21 - 12/31/21: $1,000/day, 1/1/22 - 12/31/22: $1,040/day, 1/1/23 and thereafter: $1,082/day, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services with a codeable Medicare DRG, 1/1/21 - 12/31/21: 167.5% of Medicare Allowable for the entire stay, 1/1/22 - 12/31/22: 174.2% of Medicare Allowable for the entire stay, 1/1/23 and thereafter: 181.2% of Medicare Allowable for the entire stay, \\nCategory of Service: All other Inpatient Services with a non- codeable Medicare DRG, 1/1/21 - 12/31/21: 60% of Allowable Charges, not to exceed $7,000/day, 1/1/22 - 12/31/22: 60.6% of Allowable Charges, not to exceed $7,280/day, 1/1/23 and thereafter: 61.2% of Allowable Charges, not to exceed $7,571/day, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Ambulatory Surgery, Codes: Revenue Codes: 0360, 0369, 0490, 0499, 0750, and applicable CPT Codes, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $6,000/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $6,240/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $6,490/visit, \\nCategory of Service: Emergency Room Visit, Codes: Revenue Codes: 0450, 0451, 0452, 0459, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $4,110/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $4,274/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,445/visit, \\nCategory of Service: Emergency Room - Urgent Care Services (Facility), Codes: Revenue Codes: 0456, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $3,820/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $3,973/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,132/visit, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $3,820/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $3,973/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,132/visit, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under Commercial Benefit Programs pursuant to Addendum A, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 6 of 22 G HealthNet'\\n\\n\\n\\n\\n\",\n", + " '7': '7 No multiple ambulatory surgery payment methodology applies. PV 2020 amd Page 7 of 22 HealthNet\\n\\n\\nNone: Exclusions, 50% of Allowable Charges, not to exceed $3,820/visit: Inpatient and Outpatient, \\nAll other Outpatient Services: Surgical Implants, Prosthetics, Orthotics, Pacemakers, : Inpatient: Revenue Codes: 0274, 0275, 0276, 0278 Outpatient: Revenue Codes: 0274, 0275, 0276, 0278 or applicable HCPC Codes, 50% of Allowable Charges, not to exceed $3,820/visit: Included in above rates when each Revenue Code listed is less than $3,100 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: Included in above rates when each Revenue Code listed is less than $3,193 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: Included in above rates when each Revenue Code listed is less than $3,289 in aggregate Allowable Charges, \\nAll other Outpatient Services: Surgical Implants, Prosthetics, Orthotics, Pacemakers, : Inpatient: Revenue Codes: 0274, 0275, 0276, 0278 Outpatient: Revenue Codes: 0274, 0275, 0276, 0278 or applicable HCPC Codes, 50% of Allowable Charges, not to exceed $3,820/visit: 40% of Allowable Charges is applicable to each Revenue Code if at or over $3,100 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: 40.4% of Allowable Charges is applicable to each Revenue Code if at or over $3,193 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: 40.8% of Allowable Charges is applicable to each Revenue Code if at or over $3,289 in aggregate Allowable Charges, \\nAll other Outpatient Services: High Cost Drugs, : Revenue Codes: 0630-0636, 50% of Allowable Charges, not to exceed $3,820/visit: Included in above rates when each Revenue Code listed is less than $6,000 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: Included in above rates when each Revenue Code listed is less than $6,180 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: Included in above rates when each Revenue Code listed is less than $6,365 in aggregate Allowable Charges, \\nAll other Outpatient Services: High Cost Drugs, : Revenue Codes: 0630-0636, 50% of Allowable Charges, not to exceed $3,820/visit: 39.6% of Allowable Charges is applicable to each Revenue Code if at or over $6,000 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: 40% of Allowable Charges is applicable to each Revenue Code if at or over $6,180 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: 40.4% of Allowable Charges is applicable to each Revenue Code if at or over $6,365 in aggregate Allowable Charges, \\n\\n\\n',\n", + " '8': \"8 EXHIBIT C-1 MEDI-CAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBITCodes: Inpatient Services, \\nCategory of Service: Acute Rehabilitation, Codes: Revenue Codes: 118, 128, 138, 148, 158, Compensation: 105% of the Medi-Cal Rehabilitation Rate, \\nCategory of Service: Skilled Nursing Facility, Codes: Revenue Codes: 110, 120, 130, Compensation: 105% of the facility-specific published Medi-Cal per diem rate, \\nCategory of Service: Sub-Acute, Codes: Revenue Codes: 190-194, 199, Compensation: 105% of the facility-specific published Medi-Cal per diem rate, \\nCategory of Service: All other Inpatient Services, Compensation: 105% of APR-DRG based on the current statewide wage-adjusted base rate in effect at the time of service, \\nCodes: Outpatient Services, \\nCategory of Service: Outpatient Services, Compensation: 105% of the current Medi-Cal Fee Schedule, subject to any changes made by the State of California, \\nCategory of Service: Pharmaceuticals without an established Medi-Cal Value, Compensation: Average Wholesale Price (AWP) - 18%, \\nCategory of Service: Unlisted Procedures Procedures with Medi-Cal Units not Established but with a Medicare Value Procedures without Medi- Cal Units and without a Medicare Established Value, Compensation: 80% of the Current Medicare Allowable 15% of Allowable Charges, \\nCategory of Service: Surgical Implants, Prosthetics, Orthotics, Pacemakers for Ambulatory Surgery only, Codes: Revenue Codes 274, 275, 276, 278, Compensation: Cost + 5% for Ambulatory Surgery Only, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Exhibit C-3, Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered Services delivered pursuant to this Addendum C, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 8 of 22 Health Net\\n\\n\\n\\n\\n\",\n", + " '9': \"9 EXHIBIT D-1 ENHANCEDCARE PPO BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C- Section/Vaginal Delivery (Mom and Baby), Codes: DRG Codes: 768, 783- 788, 796-798, 805-807, 1/1/21 - 12/31/21: $9,500/case, 1/1/22 - 12/31/22: $9,880/case, 1/1/23 and thereafter: $10,275/case, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes: 0170, 0171, 0179, 1/1/21 - 12/31/21: $800/case, 1/1/22 - 12/31/22: $832/case, 1/1/23 and thereafter: $865/case, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services, Codes: All inpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 130% of Medicare Allowable, 1/1/22 - 12/31/22: 135.2% of Medicare Allowable, 1/1/23 and thereafter: 140.6% of Medicare Allowable, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Outpatient Services, Codes: All outpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 130% of Medicare Allowable, 1/1/22 - 12/31/22: 135.2% of Medicare Allowable, 1/1/23 and thereafter: 140.6% of Medicare Allowable, \\nCategory of Service: Provider-Based Billing, Codes: Rev Codes: 0510-0519, 1/1/21 - 12/31/21: Not covered, 1/1/22 - 12/31/22: Not covered, 1/1/23 and thereafter: Not covered, \\nCategory of Service: Outpatient Services that are not priced or codeable per Medicare guidelines, 1/1/21 - 12/31/21: 23.8% of Allowable Charges, not to exceed $2,980 per visit, 1/1/22 - 12/31/22: 24% of Allowable Charges, not to exceed $3,099 per visit, 1/1/23 and thereafter: 24.2% of Allowable Charges, not to exceed $3,223 per visit, \\n Subject to the terms of this Addendum, including without limitation the Payment Conditions set forth in Exhibit D-2 of this Addendum, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Contracted Services delivered under Benefit Programs pursuant to this Addendum, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 9 of 22 Health Net\\n\\n\\n\\n\\n\",\n", + " '10': '10 EXHIBIT D-2 ENHANCED PPO BENEFIT PROGRAMS FEE-FOR-SERVICE PAYMENT CONDITIONS The Payment Conditions set forth in this Exhibit D-2 supplement Health Net Policies. 1. Application of 72-Hour Rule. Payments made to Provider for inpatient Covered Services shall constitute payment for all of Provider\\'s charges relating to a Covered Person\\'s pre-admission testing for the same condition occurring within seventy-two (72) hours prior to an admission, including, but not limited to, charges for laboratory services, pathology services, radiology services, and medical/surgical supplies. If Member is admitted directly from the Emergency room or from an Observation level of care, the inpatient reimbursement shall be inclusive of all such emergency and/or observation related facility charges. 2. Admissions for Same or Related Diagnoses. Inpatient admissions for the same or a related diagnoses occurring within forty-eight (48) hours following a discharge in connection with a previous admission shall be considered part of the previous admission and are not separately reimbursable. 3. Hospital-Acquired Conditions and Provider Preventable Conditions. Payment to Provider under this Addendum shall comply with state and federal laws requiring reduction of payment or non-payment to a Provider for \"Hospital-Acquired Conditions\" and for \"Provider Preventable Conditions\" as such terms (or the reasonable equivalents thereof) are defined under applicable state and federal laws. 4. Never Events. Each Provider shall use best efforts to comply with applicable state and federal reporting or other requirements relating to Never Events and/or Serious Adverse Events, as the applicable term is defined by the National Quality Forum or by state or federal law. Providers shall not bill, charge, collect a deposit from, seek compensation, remuneration or reimbursement from, or have any recourse against Health Net or Members for any charges associated with Never Events and/or Serious Adverse Events. To the extent Provider receives any payment in connection with a Never Event or Serious Adverse Event, Provider shall promptly refund such amount. 5. Level of Care. Reimbursement under this Addendum shall not exceed the Allowed Amount corresponding to the level of care authorized by Health Net. 6. Payment for Professional Services. Payment for inpatient Covered Services under this Addendum includes payment for professional Covered Services (including but not limited to services provided by hospital-based physicians, Certified Registered Nurse Anesthetists or other professionals) that are provided in connection with such inpatient Covered Services and billed under the Provider\\'s tax identification number and provider identification number. Such professional Covered Services will not be separately reimbursable. Professional Covered Services billed by independent physicians under their own tax identification number and provider identification number are reimbursable directly to such physicians. 7. Payment for Multiple Procedures. Where multiple outpatient surgical or scope procedures performed on a Member during a single occasion of surgery, reimbursement will be as follows: i) the procedure for which the Allowed Amount under this Addendum is greatest will be reimbursed at one hundred percent (100%) of such Allowed Amount; ii) the procedures with second and third greatest Allowed Amounts under this Addendum shall each be reimbursed at fifty percent (50%) of such Allowed Amounts; iii) any additional procedures will not be eligible for reimbursement. 8. Multiple Dates of Service on a Single Claim Form. Provider shall identify on the applicable claim form each date of service when submitting claims spanning multiple dates of service. 9. Provider-Based Billing. Provider-Based Billing (as defined herein) will not be reimbursed under this Compensation Schedule as they are included as part of the compensation for professional fees under this Agreement. Neither Health Net nor the Member shall be responsible for such Provider-Based Billing. \"Provider-Based Billing\" are amounts charged by a clinic or facility as a technical component, or for PV 2020 amd Page 10 of 22\\n\\n',\n", + " '11': '11 overhead, in connection with professional services rendered in a clinic or facility, and include but are not limited to services billed using Revenue Codes 0510-0519. 10. Therapy Services. Provider shall bill therapy services with the appropriate modifier designating the type of therapy provided, when applicable. 11. National Provider Identifier/Effect of Exclusion or Suspension. Notwithstanding anything to contrary contained herein, Health Net shall not pay, and shall have no obligation to pay, any claim submitted by Provider based on an order or referral that does not include the National Provider Identifier (\"NPI\") for the ordering or referring physician. Neither Health Net nor the Member shall have any obligation to pay any claim submitted by Provider if the Provider has been excluded or suspended from participation in Medicare, Medicaid, or any other federal or state health care program. 12. Code Change Updates. Health Net utilizes nationally recognized coding structures (including, without limitation, revenue codes, CPT codes, HCPCS codes, ICD codes, national drug codes, ASA relative values, etc., or their successors) for basic coding and descriptions of the services rendered. Updates to billing-related codes shall become effective on the date (\"Code Change Effective Date\") that is the later of: (i) the first day of the month following sixty (60) days after publication by the governmental agency having authority over the applicable Product of such governmental agency\\'s acceptance of such code updates, (ii) the effective date of such code updates as determined by such governmental agency or (iii) if a date is not established by such governmental agency or the applicable Product is not regulated by such governmental agency, the date that changes are made to nationally recognized codes. Such updates may include changes to service groupings. Claims processed prior to the Code Change Effective Date shall not be reprocessed to reflect any such code updates. 13. Fee Change Updates. Updates to the fee schedule shall become effective on the effective date of such fee schedule updates, as determined by Health Net (\"Fee Change Effective Date\"). The date of implementation of any fee schedule updates, i.e. the date on which such fee change is first used for reimbursement (\"Fee Change Implementation Date\"), shall be the later of: (i) the first date on which Health Net is reasonably able to implement the update in the claims payment system; or (ii) the Fee Change Effective Date. Claims processed prior to the Fee Change Implementation Date shall not be reprocessed to reflect any updates to such fee schedule, even if service was provided after the Fee Change Effective Date. 14. Reimbursement Service Grouping. Not applicable. 15. Payment under this Compensation Schedule. All payments under this Compensation Schedule are subject to the terms and conditions set forth in the Agreement and Health Net Policies. 16. \"Allowable Charges\" means Provider\\'s billed charges for Contracted Services. 17. Medicare/CMS Allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 18. Health Net may use a vendor for the benefit administration of certain Covered Services related to substance abuse and behavioral health benefits. In the event of a conflict between this Agreement and the vendor\\'s contract with Provider, the vendor\\'s contract shall prevail when responsible for benefit administration. 19. Adjustments to Compensation Based on CMS Changes. In order to neutralize the impact of CMS changes to the DRG fee schedule for current and future years for all Commercial products, the parties agree to adjust the Inpatient rates based on DRGs in the above fee schedule as follows: 19.1 Inpatient. Health Net will pull a claims report of Provider\\'s Inpatient claims (both paid and denied) from Health Net\\'s system for dates of service January 1, 2019 through December 31, 2019 for PV 2020 amd Page 11 of 22 HealthNet\\n\\n',\n", + " '12': '12 Provider\\'s review and approval. Once the data set is agreed to, Health Net will price said claims using the DRG fee schedule effective October 1, 2019, and using the DRG fee schedule effective October 1, 2020. If Health Net\\'s analysis shows that the resulting trend comparing the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2019 DRG fee schedule to the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2020 DRG fee schedule is greater than or less than one percent (1%) as verified by Provider, the parties will execute an amendment with pro-rated Inpatient rates that counteract the DRG change. The parties will repeat this process each year using each year\\'s October 1 DRG fee schedule (i.e., the Inpatient claims data set for the 10/1/2021 DRG calculation will be cy2020, etc.). All Commercial products and all Prime Health hospitals will be included in the analysis. The data set will only include Commercial Inpatient claims that are reimbursed at the % of Medicare rate. Inpatient claims that are reimbursed at case rates etc. (i.e. OB rates) will be excluded from the analysis. 19.2 Medicare/CMS Allowable. Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 19.3 Time Frame for Analysis. CMS implements the Inpatient DRG fee schedule changes effective every October 1st, the parties will target January 31st of each year to complete the analysis. 19.4 Adjusting and Pro-rating Contract Rates Examples. Adjusted IP Contract Rate = \"Current IP Rate/(1+DRG Trend)\". Example A1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 167.5%/1.03=162.6%. Adjusted IP Contract Rate Year 2= 174.2%/1.03=169.1% Adjusted IP Contract Rate Year 3= 181.2%/1.03=175.9% Example A2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is 3%. Current HMO IP rates - Year 2: 169.1%, Year 3: 175.9%. Adjusted IP Contract Rate Year 2= 169.1%/1.03=164.2%. Adjusted IP Contract Rate Year 3= 175.9%/1.03=170.8%. Example A3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is 3%. Current HMO IP rates - Year 3: 170.8%. Adjusted IP Contract Rate Year 3= 170.8%/1.03=165.8%. Example B1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is - 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/0.97=172.7. Adjusted IP Contract Rate Year 2= 174.2%/0.97=179.6%. Adjusted IP Contract Rate Year 3= 181.2%/0.97=186.8%. Example B2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is - 3%. Current HMO IP rates - Year 2: 179.6%, Year 3: 186.8%. Adjusted IP Contract Rate Year 2= 179.6%/0.97=185.1%. Adjusted IP Contract Rate Year 3= 186.8%/0.97=192.6%. Example B3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is - 3%. PV 2020 amd Page 12 of 22\\n\\n',\n", + " '13': '13 Current HMO IP rates - Year 3: 192.6%. Adjusted IP Contract Rate Year 3= 192.6%/0.97=198.5%. Pro-Rating Adjustment Due to Delay in Implementing Adjusted Contract Rates: Pro-Rated Rate = \"Adjusted Contract Rate+[(Adjusted Contract Rate-Current Rate)*(x/(12-x))]. X = Delay in months in implementing the Adjusted Contract Rates. Inpatient: Compared to October (i.e. if rates implemented eff 2/1, X=4). Example A) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 162.6%+[(162.6%-167.5%)*(4/(12-4))]=160.2%. Pro-rating only applies to current year rates. No pro-rating for future years. Example B) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 172.7%+[(172.7%-167.5%)*(4/(12-4))]=175.3%. Pro-rating only applies to current year rates. No pro-rating for future years. If there is a need to adjust the Year 3 rates due to the changes in Medicare fee schedules (in January 2023), the Year 3 pro-rated rates will expire at the end of Year 3 (December 2023 - This will depend on the actual effective date of this agreement. Currently assumes it will be effective 12/1/2020). The rates thereafter will be \"Adjusted Contract Year 3 Rates\" (Non Pro-Rated). Pro-ration Calculation: PV 2020 amd Page 13 of 22\\n\\n\\n Example: No adjustment for delay in implementation and no inflationary increase to the base rate {\\'Example:\\': [\\'Example\\', \\'\\', \\'100 % of Medicare (no trend)\\', \\'Rate (no adjustment)\\', \\'Payment (not pro- rated)\\', \\'\\', \\'Example:\\', \\'100 % of Medicare (3% trend)\\', \\'Rate (pro- rated 4 months delay)\\', \\'Payment (pro-rated 4 months delay)\\'], \\'No adjustment\\': [\\'\\', \\'1\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'Base rate\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'for\\': [\\'Delayed\\', \\'2\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'increased\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'delay in\\': [\\'Months 1-4\\', \\'3\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'for\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'implementation\\': [\\'\\', \\'4\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'inflation,\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'\\': [\\'\\', \\'Total\\', \\'\\', \\'\\', \\'201,000\\', \\'\\', \\'\\', \\'\\', \\'\\', \\'201,000\\'], \\'and no\\': [\\'\\', \\'6\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'for the\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'inflationary\\': [\\'New\\', \\'7\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'delayed\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'increase\\': [\\'Rate Updated\\', \\'8\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'period\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'to the\\': [\\'Months\\', \\'9\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'base rate\\': [\\'5-12\\', \\'10\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\']} Example: No adjustment for delay in implementation and no inflationary increase to the base rate Pro-ration Calculation:Current Contract rate: DRG trend, 167.5%: 3%, \\nCurrent Contract rate: Adjust contract rate, 167.5%: 162.6%, \\nCurrent Contract rate: Pro-rated rate (x=4), 167.5%: 160.2%, \\n\\n\\n',\n", + " '14': '14 ADDENDUM E FEE-FOR-SERVICE PAYMENT CONDITIONS The Payment Conditions set forth in this Addendum E supplement Health Net Policies. I. PAYMENT CONDITIONS APPLICABLE TO (1) PAYMENT RATES FOR COMMERCIAL BENEFIT PROGRAMS, INCLUDING SALUD CON HEALTH NET AND COMMUNITYCARE HMO AND PURECARE PPO (EXCLUDING MEDI-CAL BENEFIT PROGRAM, ENHANCEDCARE PPO, and CALMEDICONNECT BENEFIT PROGRAM), NOT BASED ON MEDICARE/CMS ALLOWABLE; AND (2) PAYMENT RATES, FOR MEDICARE BENEFIT PROGRAMS, BASED ON MEDICARE/CMS ALLOWABLE: 1.1 The codes listed in the applicable rate exhibit shall be used by the parties for the purpose of defining the services for which payment will be made under this Agreement. Provider shall utilize the specified codes listed in the applicable rate exhibit when submitting Complete Claims for Covered Services under this Agreement. 1.2 The parties acknowledge that applicable coding agencies periodically issue coding modifications. Such modifications may be implemented by Health Net within sixty (60) days of the date Health Net receives the modification from the applicable coding agency or notification from Provider. The parties agree that in the event such coding modifications have the effect of changing a payment amount in this Agreement, the resulting payment amount change shall be effective on a prospective basis. The parties further agree to reasonably and in good faith discuss any contract rate amendment that may be appropriate based on comprehensive and substantive coding changes by the applicable coding agency within ninety (90) days of written notification by either party. Any and all rate modifications that may result from such contract amendment shall be effective on a prospective basis. 1.3 Only one per diem or case rate shall be payable a) for each Beneficiary who is admitted prior to midnight and remains past midnight; b) if a Beneficiary is admitted and discharged during the same day, provided that such admission and discharge is not within twenty-four (24) hours of a prior discharge; c) for mother and newborn child (children) when both mother and newborn child (children) are in the facility on the same day, unless the child (children) is in the neonatal intensive care unit. Pre-admission testing completed within one (1) day of admission will be included in the per diem and will not be billed separately by Provider. A per diem is an all-inclusive daily payment, including but not limited to observation services, supplies, implants, prosthetics, durable medical equipment, and pharmaceuticals with the exception of specific carve-outs or exclusions identified on the applicable rate exhibit. Inpatient Professional services billed by the Provider on behalf of the hospital-based physician(s) are included in the per diems or case rates. Admission kits are included in the per diem rate. Any Outpatient Services delivered to a Beneficiary within twenty-four (24) hours prior to an admission for Inpatient Services for the same medical condition at the same facility are included in the inpatient rate. 1.4 The rates set forth in this Agreement apply only to inpatient admissions commencing on or after the Effective Date of this Agreement. The applicable rate of payment that is in effect on the date an inpatient admission commences shall apply to the entire length of stay regardless of any rate change that may occur during the length of stay. Notwithstanding the foregoing, inpatient case rates are applicable beginning the first day of admission if a case rate procedure is performed at any time during the inpatient stay. 1.5 All inpatient case rates shall include any readmission charges if the patient is readmitted within forty-eight (48) hours following discharge for complications related to the prior admission. Should non-standard conditions occur, the second admission within forty-eight (48) hours may be eligible for reimbursement, provided such admission is in accordance with Health Net Policies. INTENTIONALLY LEFT BLANK PV 2020 amd Page 14 of 22 HealthNet\\n\\n',\n", + " '15': \"15 1.6 If a Beneficiary is admitted to the hospital from the emergency room and/or observation room, separate reimbursement for the emergency room, observation room and other outpatient charges shall not be paid; however, the applicable inpatient reimbursement shall be reimbursed as all-inclusive reimbursement. The rate payable for Emergency Services is an all-inclusive case rate, including but not limited to supplies, implants, prosthetics, durable medical equipment, observation charges, diagnostic services, therapeutic services, pharmaceuticals, and other services incidental to the emergency room visit, with the exception of specific exclusions identified on the applicable rate exhibit. 1.7 The rate payable for outpatient surgery is an all-inclusive case rate, including but not limited to supplies, implants, prosthetics, durable medical equipment, emergency room charges, observation charges, diagnostic services, therapeutic services and pharmaceuticals with the exception of specific exclusions identified on the applicable rate exhibit. Only one (1) case rate shall be paid for each Beneficiary per date of service. 1.8 Other Outpatient case rates are an all-inclusive payment, including but not limited to supplies, diagnostic services, therapeutic services and pharmaceuticals with the exception of specific exclusions identified on the applicable rate exhibit. Only one (1) case rate shall be paid for each Beneficiary per date of service; provided, however, that if one or more unique case rate procedure is performed on different body parts, each applicable case rate shall be reimbursed. 1.9 Intentionally Left Blank 1.10 In the event that a Beneficiary is transported either within Provider's Facility or to another facility for a service that is within Provider's customary scope of practice, but due to malfunction or other cause cannot be delivered, Provider shall be solely financially responsible for claims incurred for such transportation. 1.11 For purposes of this Agreement, New Service/Technology is defined as a service, procedure, device, test, or other item that, as of the effective date of this Agreement: (i) is not performed by Provider, or (ii) is not Covered by Health Net under the applicable Benefit Program, or (iii) for which there is no assigned CPT or other relevant code defined. New Service/Technology does not include a new code that is assigned as a change to an existing service, procedure, device, test, or other item. If Provider offers a New Service/Technology after the effective date of this Agreement for which Provider desires reimbursement under this Agreement, Provider agrees to give Health Net sixty (60) days prior written notice that Provider requests that such New Service/Technology be included in Contracted Services, and that Provider receive a different reimbursement rate for the use of such New Service/Technology for Beneficiaries. Health Net agrees to determine whether such New Service/Technology qualifies as a Covered Service and whether Health Net desires to include such New Service/Technology in Contracted Services within thirty (30) days of the date of Provider's notice hereunder. Health Net's determination of whether a New Service/Technology qualifies as a Covered Service shall be made in Health Net's sole discretion, and shall be final and binding on Provider with respect to this Agreement. Only if Health Net agrees that such New Service/Technology is a Covered Service and desires to include such New Service/Technology in Contracted Services, the parties shall enter into good faith negotiations to modify Provider's rates to reflect the New Service/Technology. Any rate modification shall be made only prospectively. 1.12 The lowest room and board billed charge rate shall apply to all days billed within the same revenue code level of care. 1.13 All professional Covered Services billed by Participating Providers on a CMS 1500 form under Provider's federal tax identification number shall be reimbursed at one hundred percent (100%) of CMS allowable in effect on the date of service. PV 2020 amd Page 15 of 22 HealthNet\\n\\n\",\n", + " '16': '16 1.14 If any payment rates are based on Provider\\'s billed charges, upon Health Net\\'s written request, Provider shall send Health Net an electronic copy of Provider\\'s current Charge Master. Provider shall notify Health Net in writing via certified letter at least sixty (60) days prior to making any changes to Provider\\'s Charge Master (\"Notice\"). The rates in Provider\\'s Charge Master on September 1, 2013, for example the date Provider\\'s most recent fiscal year began, shall be the \"Baseline Charge Master.\" Changes to the Baseline Charge Master shall be measured during the twelve (12) month period immediately preceding the date of the Notice (the \"Measurement Period\"). The Notice shall be signed by Provider\\'s Chief Financial Officer (or equivalent) and shall include a statement of any and all changes made to the Baseline Charge Master during the Measurement Period together with the effective date of such change. All changes included in the Notice shall be calculated using the methodology set forth herein. Any change to Provider\\'s Charge Master during the term of this Agreement shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. The terms of this section shall survive termination and/or subsequent renegotiation of this Agreement with respect to Contracted Services rendered during the term of the Agreement and Health Net shall retain its right to pursue recovery of any overpayments and/or to adjust the rates under the terms and provisions of the Agreement and applicable law. Health Net reserves all of its rights under this Agreement, at law and in equity related to any Charge Master dispute that may exist or arise in the future. The methodology for calculating Changes to the Charge Master and rates pursuant to this section is as follows: (a) First Year: Provider agrees that any Notice delivered during the first year of this Agreement shall reflect no changes made to the Baseline Charge Master. In the event Provider has made changes to its Baseline Charge Master, any such change shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement, such that there will not be a change to Health Net\\'s reimbursement to Provider during the first year of this Agreement. (b) Subsequent Years: For each subsequent Measurement Period, Provider shall not change Provider\\'s Baseline Charge Master in a manner that results in an increase in reimbursement under this Agreement by more than three percent (3%) in the aggregate, unless expressly agreed upon in writing by the parties hereto. Upon request, Provider shall promptly provide Health Net with a Notice that completely and accurately sets forth any and all changes during the Measurement Period. Health Net shall have the right to reconcile and reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. In the event Health Net exercises its right to reconcile and adjust payment, Health Net shall give Provider at least thirty (30) days prior written notice, which notice shall be attached to the applicable rate exhibit in this Agreement. Any resulting change in payment rates shall be effective upon thirty (30) days prior written notice to Provider and shall be applicable to all claims adjudicated after the thirty (30) day notice period has expired regardless of the dates of service for such claims. (c) Both parties agree that Charge Master changes shall be calculated using Provider\\'s aggregate weighted average charge increase where weights equal volume for specific charges. Health Net may periodically review Provider\\'s Charge Master filed with the State or Health Net\\'s historical claims data. If Health Net determines as a result of such review that Provider has made changes to the Baseline Charge Master but has failed to give Health Net Notice, the parties agree that such changes shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. Any such change in rates shall be effective upon thirty (30) days prior written notice to Provider and shall be applicable to all claims adjudicated after the thirty (30) day notice period has expired regardless of the dates of service for such claims. Health Net shall use the above methodology for calculating changes to the Charge Master and rates pursuant to this section. PV 2020 amd Page 16 of 22 Health Net\\n\\n',\n", + " '17': \"17 II. PAYMENT CONDITIONS APPLICABLE TO PAYMENT RATES BASED ON MEDICARE/CMS ALLOWABLE. The following payment conditions are in addition to those set forth in Article I above, and apply to payment rates based upon a percentage of Medicare/CMS allowable rates, and methodology. Notwithstanding the foregoing, to the extent any payment condition set forth in this Article II conflicts with a payment condition in Article I, the payment condition set forth in this Article II shall control with respect to payment based upon a percentage of Medicare/CMS allowable rates, and methodology: 2.1 Only one DRG/case rate shall be payable for each Beneficiary who is admitted, provided that such admission is not within 48 hours of a prior discharge and for Benefit Program Beneficiaries other than Medicare Beneficiaries, for mother and newborn child (children) when both mother and newborn child (children) are in the facility on the same day, unless the child (children) is in the neonatal intensive care unit. Any outpatient services delivered to a Beneficiary within forty-eight (48) hours prior to an admission for Inpatient Services at the same facility are included in the inpatient rate. 2.2 Provider shall be reimbursed in accordance with CMS guidelines for all Outpatient Services. 2.3 Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME) and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 2.4 Provider agrees to adhere to CMS/Medicare billing and compensation guidelines for all services. 2.5 Adjustments to Compensation Based on CMS Changes. In order to neutralize the impact of CMS changes to the DRG fee schedule for current and future years for all Commercial products, the parties agree to adjust the Inpatient rates based on DRGs in the above fee schedule as follows: 2.5.1 Inpatient. Health Net will pull a claims report of Provider's Inpatient claims (both paid and denied) from Health Net's system for dates of service January 1, 2019 through December 31, 2019 for Provider's review and approval. Once the data set is agreed to, Health Net will price said claims using the DRG fee schedule effective October 1, 2019, and using the DRG fee schedule effective October 1, 2020. If Health Net's analysis shows that the resulting trend comparing the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2019 DRG fee schedule to the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2020 DRG fee schedule is greater than or less than one percent (1%) as verified by Provider, the parties will execute an amendment with pro-rated Inpatient rates that counteract the DRG change. The parties will repeat this process each year using each year's October 1 DRG fee schedule (i.e., the Inpatient claims data set for the 10/1/2021 DRG calculation will be cy2020, etc.). All Commercial products and all Prime Health hospitals will be included in the analysis. The data set will only include Commercial Inpatient claims that are reimbursed at the % of Medicare rate. Inpatient claims that are reimbursed at case rates etc. (i.e. OB rates) will be excluded from the analysis. 2.5.2 Medicare/CMS Allowable. Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass- throughs. 2.5.3 Time Frame for Analysis. CMS implements the Inpatient DRG fee schedule changes effective every October 1st, the parties will target January 31st of each year to complete the analysis. INTENTIONALLY LEFT BLANK PV 2020 amd Page 17 of 22 HealthNet\\n\\n\",\n", + " '18': '18 2.5.4 Adjusting and Pro-rating Contract Rates Examples. Adjusted IP Contract Rate = \"Current IP Rate/(1+DRG Trend)\". Example A1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/1.03=162.6%. Adjusted IP Contract Rate Year 2= 174.2%/1.03=169.1%. Adjusted IP Contract Rate Year 3= 181.2%/1.03=175.9%. Example A2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is 3%. Current HMO IP rates - Year 2: 169.1%, Year 3: 175.9%. Adjusted IP Contract Rate Year 2= 169.1%/1.03=164.2%. Adjusted IP Contract Rate Year 3= 175.9%/1.03=170.8%. Example A3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is 3%. Current HMO IP rates - Year 3: 170.8%. Adjusted IP Contract Rate Year 3= 170.8%/1.03=165.8%. Example B1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is -3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/0.97=172.7%. Adjusted IP Contract Rate Year 2= 174.2%/0.97=179.6%. Adjusted IP Contract Rate Year 3= 181.2%/0.97=186.8%. Example B2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is -3%. Current HMO IP rates - Year 2: 179.6%, Year 3: 186.8%. Adjusted IP Contract Rate Year 2= 179.6%/0.97=185.1%. Adjusted IP Contract Rate Year 3= 186.8%/0.97=192.6%. Example B3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is -3%. Current HMO IP rates - Year 3: 192.6%. Adjusted IP Contract Rate Year 3= 192.6%/0.97=198.5%. Pro-Rating Adjustment Due to Delay in Implementing Adjusted Contract Rates: Pro-Rated Rate = \"Adjusted Contract Rate+[(Adjusted Contract Rate-Current Rate)*(x/(12-x))]. X = Delay in months in implementing the Adjusted Contract Rates. Inpatient: Compared to October (i.e. if rates implemented eff 2/1, X=4). Example A) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 162.6%+[(162.6%-167.5%)*(4/(12-4))]=160.2% Pro-rating only applies to current year rates. No pro-rating for future years. Example B) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 172.7%+[(172.7%-167.5%)*(4/(12-4))]=175.3% Pro-rating only applies to current year rates. No pro-rating for future years. If there is a need to adjust the Year 3 rates due to the changes in Medicare fee schedules (in January 2023), the Year 3 pro-rated rates will expire at the end of Year 3 (December 2023 - This will depend on the actual effective date of this agreement. Currently assumes it will be effective 12/1/2020). The rates thereafter will be \"Adjusted Contract Year 3 Rates\" (Non Pro-Rated). PV 2020 amd Page 18 of 22\\n\\n',\n", + " '19': \"19 Pro-ration Calculation: PV 2020 amd Page 19 of 22 HealthNet\\n\\n\\n Example: No adjustment for delay in implementation and no inflationary increase to the base rate {'Example:': ['', '', '100 % of Medicare (no trend)', 'Rate (no adjustment)', 'Payment (not pro- rated)', '', 'Example:', '100 % of Medicare (3% trend)', 'Rate (pro- rated 4 months delay)', 'Payment (pro-rated 4 months delay)'], 'No adjustment': ['', '1', '10,000', '167.5%', '16,750', '', 'Base rate', '10,300', '167.5%', '17,253'], 'for': ['Delayed', '2', '10,000', '167.5%', '16,750', '', 'increased', '10,300', '167.5%', '17,253'], 'delay in': ['Months 1-4', '3', '10,000', '167.5%', '16,750', '', 'for', '10,300', '167.5%', '17,253'], 'implementation': ['', '4', '10,000', '167.5%', '16,750', '', 'inflation,', '10,300', '167.5%', '17,253'], '': ['', 'Total', '', '', '201,000', '', '', '', '', '201,000'], 'and no': ['', '6', '10,000', '167.5%', '16,750', '', 'for the', '10,300', '160.2%', '16,499'], 'inflationary': ['New', '7', '10,000', '167.5%', '16,750', '', 'delayed', '10,300', '160.2%', '16,499'], 'increase': ['Rate Updated', '8', '10,000', '167.5%', '16,750', '', 'period', '10,300', '160.2%', '16,499'], 'to the': ['Months', '9', '10,000', '167.5%', '16,750', '', '', '10,300', '160.2%', '16,499'], 'base rate': ['5-12', '10', '10,000', '167.5%', '16,750', '', '', '10,300', '160.2%', '16,499']} Example: No adjustment for delay in implementation and no inflationary increase to the base rate Pro-ration Calculation:Current Contract rate: DRG trend, 167.5%: 3%, \\nCurrent Contract rate: Adjust contract rate, 167.5%: 162.6%, \\nCurrent Contract rate: Pro-rated rate (x=4), 167.5%: 160.2%, \\n\\n\\n\",\n", + " '20': \"20 EXHIBIT F-1 COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C- Section/Vaginal Delivery (Mom and Baby), Codes: DRG Codes: 768, 783-788, 796-798, 805-807, 1/1/21 - 12/31/21: $9,500/case, 1/1/22 - 12/31/22: $9,880/case, 1/1/23 and thereafter: $10,275/case, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes:0170, 0171, 0179, 1/1/21 - 12/31/21: $800/case, 1/1/22 - 12/31/22: $832/case, 1/1/23 and thereafter: $865/case, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Outpatient Services, Codes: All outpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 23.8% of Allowable Charges, not to exceed $3,094 per visit, 1/1/22 - 12/31/22: 24% of Allowable Charges, not to exceed $3,218 per visit, 1/1/23 and thereafter: 24.2% of Allowable Charges, not to exceed $3,346 per visit, \\nCategory of Service: Ambulatory Surgery, Codes: Revenue Codes 360, 369, 490, 499, 750, and applicable CPT codes, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Emergency Room Visit, Codes: Revenue Codes 450, 451, 452, 459, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Emergency Room - Urgent Care Services (Facility), Codes: Revenue Code 456, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 140% of Medicare Allowable, 1/1/22 - 12/31/22: 145.6% of Medicare Allowable, 1/1/23 and thereafter: 151.4% of Medicare Allowable, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under CommunityCare Benefit Programs pursuant to Addendum F, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 20 of 22 CP HealthNet\\n\\n\\n\\n\\n\",\n", + " '21': '21 EXHIBIT F-2 COMMUNITYCARE BENEFIT PROGRAMS HOSPITAL QUALITY PERFORMANCE PROGRAM I. FFS COMPENSATION. Provider shall render Covered Services to Members and shall accept the compensation set forth in the applicable Exhibit as payment-in-full for Covered Services rendered to such Members. II. INCENTIVE PROGRAM. Effective for measurement/calendar year 2020, Provider shall qualify for participation in the quality incentive program if at least four (4) of the following requirements are met: 2.1 Electronic Health Records (EHR) Access. Provider must, upon request, grant Health Net access to all Health Net Member EHRs (i.e., medical records access and individual export of medical records for Health Net members via Health Information Exchange that Health Net participates with or via direct EHR access on Provider EHR portal). 2.2 Nurse Care Manager Access. Provider must, upon request, allow Health Net care management nurses access to Provider facilities in order to perform any needed case management duties. 2.3 Notification of Admission/Discharge/Transfer. Provider must provide Health Net with notification within twenty-four (24) hours, or the next business day if admitted/discharged on a weekend, for ninety percent (90%) of Provider\\'s admissions, discharges, transfers (two events per member per occurrence). Alternatively, Provider must provide Health Net with Admission/Discharge data via a Health Information Exchange that Health Net participates with or send it directly to Health Net so that Health Net receives admission and discharge notifications electronically (i.e. HL7/industry standard messages) in near real time. 2.4 Hospital Consumer Assessment of Healthcare Providers and Systems (HCAHPS) Score. Provider achieves both of the following: 1 A score of seventy-five percent (75%) or greater for the HCAHPS question \"Patients who reported YES they would definitely recommend the hospital\", as reported in the most recent survey period by CMS. 2 A composite score that meets the most recent Medicare Stars 4 star threshold for the HCAHPS Composite Measure \"Discharge Information\", as reported in the most recent survey period by CMS, based on the following questions: a) .did hospital staff talk with you about whether you would have the help you needed when you left the hospital?\" b) \" did you get information in writing about what symptoms or health problems to look out for after you left the hospital?\" 2.5 California Maternal Quality Care Collaborative (CMQCC) Maternal Data Center Reporting and Data Release. Provider with maternity service participates in the CMQCC Maternal Data Center Reporting initiative and signs an authorization release to share their hospital-specific results with Health Net. III. PROVIDER PARTICIPATION. If Provider satisfies the requirements above, then participation in the quality incentive program shall be granted. VI. METRICS. The quality incentive program is comprised of selected metrics, each carrying its own separate award value. The metrics are as follows: PV 2020 amd Page 21 of 22 HealthNet\\n\\n',\n", + " '22': \"22 4.1 Hospital Acquired Conditions (HACs). Provider Standardized Infection Ratio (SIR) for each of the five (5) HACs listed below is either less than 1.0 for inpatient admissions during the calendar year, or Provider must reduce its SIR by at least 0.25 for inpatient admissions in relation to its prior year experience. Each individual HAC is measured separately and is worth twenty percent (20%) of the overall HACs incentive. HACs measured are listed below: 1. Central line-associated bloodstream infection (CLABSI) 2. Catheter-associated urinary tract infections (CAUTI) 3. Methicillin-resistant staphylococcus aureus (MRSA) 4. Clostridium difficile (C.Diff) 5. Colorectal Surgical Site Infections (SSI-Colon) Source: California Department of Public Health (CDPH) portal and National Healthcare Safety Network (NHSN), if a measure is not reported on the CDPH portal, as posted on January 31 following the calendar year. Value: Twenty percent (20%) of the maximum incentive for the measurement year. 4.2 All-Cause Readmissions. Provider must have a risk-adjusted all-cause/any facility Member readmission rate that is ten percent (10%) lower for the Benefit Program for the measurement/calendar year compared to the provider year-end baseline performance from the prior calendar year. Readmission rates at or below four percent (4%) automatically qualify for this incentive regardless of improvement level from prior measurement years. Readmission rates that are higher than the original baseline performance will not qualify for this incentive. Source: Claims submitted to Health Net Value: Forty percent (40%) of the maximum incentive for the measurement year 4.3 Discharge Instructions. Upon review, Provider must demonstrate to Health Net that at least ninety-five percent (95%) of Members received discharge instructions including discharge information summarizing care and the following: name of hospitalist during inpatient stay, treatment/procedures provided, discharge diagnosis, current medication list and allergies, test results and a notation of pending/no pending test(s), and patient care instructions. Source: Provider EMR system via direct access or extract or other electronic notification system approved by Health Net Value: Twenty percent (20%) of the maximum incentive for the measurement year. 4.4 NTSV C-Sections. Provider NTSV C-Section rate provided by CMQCC is either twenty- three and nine tenths percent (23.9%) or less or ten percent (10%) lower for the CommunityCare Benefit Program for the measurement/calendar year compared to the provider year-end baseline performance from the prior calendar year. NTSV C-Section rate is defined as a live baby born at or beyond thirty-seven (37.0) weeks during a first pregnancy, who is a singleton, and is in the vertex position during cesarean birth. Source: California Maternal Quality Care Collaborative (CMQCC) Value: Twenty percent (20%) of the maximum incentive for the measurement year. V. PAYMENT. The maximum incentive for this program shall be equal to two percent (2%) of the inpatient fee-for-service (FFS) payments Health Net made to Provider during the measurement year related to providing Covered Services to Members in the measurement year for the applicable Benefit Program(s). Health Net shall confirm each of the requirements for the metrics above are met during the measurement year. If Health Net confirms one or more of the requirement are met, Provider shall receive a quality incentive payment equal to the combined percentage values of the maximum incentive. Payment shall be made within one-hundred and eighty (180) days after the end of the measurement year. Health Net reserves the right to discontinue or modify this incentive program with thirty (30) days' notice prior to the beginning of a measurement year. PV 2020 amd Page 22 of 22 Health Net\"}" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "\n", + "def format_table(table_json):\n", + " table_text = \"\"\n", + " for i in range(len(table_json[list(table_json.keys())[0]])):\n", + " for key in table_json.keys():\n", + " if table_json[key][i]:\n", + " table_text += key + ': ' + table_json[key][i] + ', '\n", + " table_text += '\\n'\n", + " return table_text\n", + "\n", + "\n", + "def align_and_format_tables(text_dict):\n", + " aligned_text_dict = {}\n", + " for key, text in text_dict.items():\n", + " if 'Table Start' in text:\n", + " print(key)\n", + " table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)\n", + " for table_text in table_texts:\n", + " # Extract pretable text\n", + " pretable = table_text.split('{')[0].strip()\n", + "\n", + " # Extract and format table text\n", + " table_only = \"{\" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace(\"'\", '\"') + \"}\"\n", + " table = json.loads(table_only)\n", + " table_formatted = format_table(table)\n", + "\n", + " # Align\n", + " if text.count(pretable) == 2:\n", + " # Remove original table\n", + " aligned_text = text.replace(table_text, \"\")\n", + " # Align new table\n", + " aligned_text = aligned_text.replace(pretable, pretable + table_formatted)\n", + " else:\n", + " aligned_text = text.replace(table_text, pretable + table_formatted)\n", + " aligned_text_dict[key] = aligned_text.replace(\"-------Table Start--------\", \"\").replace(\"-------Table End--------\", \"\")\n", + " \n", + " else:\n", + " aligned_text_dict[key] = text\n", + " \n", + " return aligned_text_dict\n", + "\n", + "align_and_format_tables(text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "6 EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under Commercial Benefit Programs pursuant to Addendum A, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 6 of 22 G HealthNet'\n", + "\n", + "\n", + "-------Table Start-------- EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT {'Category of Service': ['', 'OB/C-Section and Vaginal Delivery (Mom and Baby)', 'Boarder Baby', 'Rehabilitation', 'All other Inpatient Services with a codeable Medicare DRG', 'All other Inpatient Services with a non- codeable Medicare DRG', '', 'Ambulatory Surgery', 'Emergency Room Visit', 'Emergency Room - Urgent Care Services (Facility)', 'Observation'], 'Codes': ['', 'MS-DRGs: 768, 783-788, 796-798, 805-807', 'Revenue Codes: 0170, 0171, 0179', 'Revenue Code: 0128', '', '', '', 'Revenue Codes: 0360, 0369, 0490, 0499, 0750, and applicable CPT Codes', 'Revenue Codes: 0450, 0451, 0452, 0459', 'Revenue Codes: 0456', 'Revenue Codes: 0760, 0762'], '1/1/21 - 12/31/21': ['Inpatient Services', '$12,600/3 day case rate. Additional days $4,000/day', '$1,000/day', '$2,500/diem', '167.5% of Medicare Allowable for the entire stay', '60% of Allowable Charges, not to exceed $7,000/day', 'Outpatient Services', '50% of Allowable Charges, not to exceed $6,000/visit', '50% of Allowable Charges, not to exceed $4,110/visit', '50% of Allowable Charges, not to exceed $3,820/visit', '50% of Allowable Charges, not to exceed $3,820/visit'], '1/1/22 - 12/31/22': ['', '$13,104/3 day case rate. Additional days $4,160/day', '$1,040/day', '$2,600/diem', '174.2% of Medicare Allowable for the entire stay', '60.6% of Allowable Charges, not to exceed $7,280/day', '', '50.5% of Allowable Charges, not to exceed $6,240/visit', '50.5% of Allowable Charges, not to exceed $4,274/visit', '50.5% of Allowable Charges, not to exceed $3,973/visit', '50.5% of Allowable Charges, not to exceed $3,973/visit'], '1/1/23 and thereafter': ['', '$13,628/3 day case rate. Additional days $4,326/day', '$1,082/day', '$2,704/diem', '181.2% of Medicare Allowable for the entire stay', '61.2% of Allowable Charges, not to exceed $7,571/day', '', '51% of Allowable Charges, not to exceed $6,490/visit', '51% of Allowable Charges, not to exceed $4,445/visit', '51% of Allowable Charges, not to exceed $4,132/visit', '51% of Allowable Charges, not to exceed $4,132/visit']} -------Table End--------\n", + "\n", + "\n" ] } ], "source": [ - "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" + "print(text_dict['6'])" ] }, { From 0bbba55705c3b09cb1ca8e2cbc508f8d50603dd6 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 24 May 2024 15:28:33 -0500 Subject: [PATCH 28/78] TD BU alignment fix --- src/config.py | 8 +- src/prompt_funcs.py | 98 +++++++--- src/prompts.py | 20 +- src/table_funcs.py | 37 ++-- src/test_notebook.ipynb | 424 ++++++++++++++++++++++++++-------------- 5 files changed, 376 insertions(+), 211 deletions(-) diff --git a/src/config.py b/src/config.py index 0c99fc7..3c8e05f 100644 --- a/src/config.py +++ b/src/config.py @@ -28,12 +28,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXDI7HGA7Q" -AWS_SECRET_ACCESS_KEY="mXAJEp6Ar35G+VEwUfKq/3cu+1Hn721R+pNs+Fxh" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEOj//////////wEaCXVzLWVhc3QtMiJHMEUCIQDlpQWz84O6hWg0qtTjfoNx1nIt3Ip82gSG/jdYh05Z/AIgK3N9ViVt3ez4iwTbo7hwOgtnTbeK6Jn5fpCRfcHDv2QqgwMIYhAAGgw2NjAxMzEwNjg3ODIiDC3AfcU4Q5o7CB6tkCrgAjwqqjf8jXvnSMmnxvYmhqy4IwD/N4hyCIwYSHC1iGMOp9QsFyBVKixjAEl6m9dzt24XomQCY/GSoVNchUbqewuFIy5im3nyKlF/tGYj/xi261JTIr0szXk5UDw5twCXHSSdem1PII+C3EmsGl6fLjRFbPNlXi6vY3Z/X6nnhix/J6ZJQzPg5Plj6jGY9uBrtE/CF5DYtfGRPvFT13ZiI+UYJ/97viOM5JOWcT1nHoVY8DCLWicRCZO4aXbrOUs8QtRNns7oJye1sL1mU+0ocgkrXCYOnX54McYj4dViAHy3UUYo9w9u6a2LeW2MfQjvKam0W+q2iHuLs1ydn8ip7WjDzmBzLTAKZPWnPTjKZXlCGUsPXTj7U1rF0wNgBmIpOgMO7d49TsbSL0MWifCXuOF4sgu5ICKt5bcd7N0rvGgQR3I/kAY7abH60EAl37tbhP79tLmPt//hlR4xJ2lKuD0wrrq4sgY6pgEsIEApALn8eIykqKZPFf5jz1QWbhuScAy3Yjh+OjOvk29eUERS3xvIeOgYUF40qlkPboFkCRY3NNgvHLkbfg9I9m929zKk6Tugb6S30Aiqy9prxpPNvIA9/X877EzIb0xoIbxJHb8ucvW1xBkUNEwwZGlA6xbzKXqTHau7/j6odzLd/CPno+IQ9lCBSilCbg/1W1NAiDZC4IwJjpV8rHONSaXVdA44" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXNEUYUPUY" +AWS_SECRET_ACCESS_KEY="3HljGiz+dqWBtj+dkqs/syyxoguUoTahdLyceEeo" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEBUaCXVzLWVhc3QtMiJHMEUCIQCX9LcYh9A28cLNA/lDEHJ6ZqrulN0FampT1ptNyX+9JAIgIFrmDiZS5KfHbZWptuvveZt2s7x5r0xDwCU6t6EgBAEqjAMIjv//////////ARAAGgw2NjAxMzEwNjg3ODIiDOLRiVmy5kw46/5mzCrgAscMrf+1rEm7kpwpmd+BHjLNV4ggEy0oqQQ6z5Ny5BV6Ouo0tRmXqOqSaI5/YKXeKsRo8s4lrDNNq8qIK0GxpugvmbqguKYfEzKwyihGi5IVVb2JGL1nBkL495v0w5j1fuXqNwB7VCPkJyJCqxDHeehgtQ1AymuVAiRowkWmr7vVvFQyKKcGsBJlIzBZyguTtJhT2HPCp8IW9LhQMS2XBAa4xCnHqpPZSYIoKyp2zPPwi3pBKm+7eE3NGH7V7/Vrl6IL29zNkk5K/IeqAcyBzaFgaCPilr5N6lSMeKlkz8w0ls3guZeaEjDRf/Mz3NvMD9HeUVqGfi2M+ODxudgEfmfH3Op0EC2g5fsIBu/uvcjJf7PgUo9xT6CH5P5HY/e6TLaZH55ioz468uaTfoNw10TZ/6iymqYEXSpLfqHejC6qCx2pfcYf1UizgD2rn22GfcGs8u6oKFSThJ4fETq1mogwzafCsgY6pgHl1EzoiRwiXqCDl3GsA162dYrapmEPqcIVXTTuNuYXbl7B3GQfxTmjHjqSqKFhieWEvydD4SSQAhSXz9tPsN69Myyelwg3t6r2EWGL+JiZXEFpO1peWJgu3xfd3F1tPnp7Nfv7bVGmuIvft1JYTmx20W/vH9Q4J7jM4tPqEeQs5fRORTdXcm10Yysuq5FYo+9bahUJnPYblcJveO30vVirwzVWvmPm" # File Paths -LOCAL_PATH = 'docs/test/' # Replace with local +LOCAL_PATH = 'docs/priority_health/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 6bb3653..c8ff729 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -3,6 +3,7 @@ import os import re import pandas as pd +import json from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate from llama_index.core import SummaryIndex, Document @@ -95,17 +96,6 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts -def consolidate_lob(lob_dicts, results_dicts): - final_dicts = [] - for d in results_dicts: - try: - d.update(lob_dicts[d['page_num']]) - final_dicts.append(d) - except: - final_dicts.append(d) - return final_dicts - - def run_bottom_up(filename, text_dict): # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY @@ -158,33 +148,79 @@ def run_top_down(filename, text_dict): continue prompt = prompts.TOP_DOWN_PRIMARY(page_text) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to list of dictionaries - all_results.append(answer_dicts) - td_primary = [i for d in all_results for i in d] # Consolidate to one list + answer_dict = json.loads(answer) + answer_dict.update({'page_num' : page_num, 'Filename' : filename}) + #answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to dictionaries + all_results.append(answer_dict) + + # td_primary = [i for d in all_results for i in d] # Consolidate to one list + # print(td_primary) # Secondaries - td_final = top_down_secondary(td_primary, text_dict) + td_final = top_down_secondary(all_results, text_dict) return td_final # List of list of dictionaries +def clean_td(td): + td_clean = [] + for d in td: + new_d = {} + for k in d.keys(): + if k not in ['page_num', 'Filename']: + if ',' in d[k]: + if 'DATE' not in k: + new_d[k] = [v.strip() for v in d[k].split(',')] + elif d[k] == 'N/A': + new_d[k] = [] + else: + new_d[k] = [d[k]] + else: + new_d[k] = d[k] + td_clean.append(new_d) + return td_clean + +def get_unique_td(td): + unique_td_dict = {} + for d in td: + for key, value in d.items(): + if key not in ['page_num', 'Filename']: + for v in value: + if key not in unique_td_dict.keys(): + unique_td_dict[key] = [v] + else: + unique_td_dict[key].append(v) + final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()} + return final_dict + def td_bu_combine(td, bu, text_dict): + bu_dicts = bu.copy() + td_dicts = td.copy() + + td_dicts_clean = clean_td(td_dicts) + unique_td = get_unique_td(td_dicts_clean) + combined_list = [] - for bud in bu: - td_dicts = [d for d in td if d['page_num'] == bud['page_num']] - # Case 1: Only one option - if len(td_dicts) == 1: - td_dict = td_dicts[0] - bud.update(td_dict) - # Case 2: More than one option - elif len(td_dicts) > 1: - td_check_formatted = utils.format_td_check(td_dicts, ['Filename', 'page_num']) - prompt = prompts.LOB_CHECK(bud, td_check_formatted, text_dict[bud['page_num']]) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - answer_idx = ''.join(re.findall(r'\d', answer)) - bud.update(td_dicts[int(answer_idx)-1]) - # Case 3: No option for page - else: - continue + for bud_ in bu_dicts: + bud = bud_.copy() + for key, value in unique_td.items(): + if len(value) == 0: # No value in contract + bud.update({key : ""}) + elif len(value) == 1: # 1 value in contract + bud.update({key : value[0]}) + else: # More than one value in contract + # Check for value on the page + page_value = [] + for d in td_dicts_clean: + if (d['page_num'] == bud['page_num']) and (key in d.keys()): + for v in d[key]: + page_value.append(v) + + if len(page_value) == 1: # One value on page + bud.update({key : page_value[0]}) + elif len(page_value) > 1: # Multiple values on page + bud.update({key : ', '.join(page_value)}) + elif len(page_value) == 0: + bud.update({key : ""}) combined_list.append(bud) return combined_list diff --git a/src/prompts.py b/src/prompts.py index 86871dd..2506edd 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -122,34 +122,34 @@ Only return the dictionary, with no other commentary or explanation. Ensure you def TOP_DOWN_PRIMARY(page): return f"""### PAGE START ### {page} ### PAGE END -The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a list of dictionaries with the values below. +The preceding text is one page of a contract between a Payer and a provider in their network. Your job is to extract the names of certain entities listed in the contract and return a single dictionary with the values below: Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: -'CONTRACT_LOB' : List any Line of Business mentioned on the page. CONTRACT_LOB will ONLY be one of the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs +'CONTRACT_LOB' : List any Line of Business mentioned on the page. Choose ONLY from the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs. If none of these are found on the page, simply write 'N/A' for this answer. 'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS 'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. -If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return a separate dictionary for them such that each of the four attributes is associated with the other attributes in the dictionary. +If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list. For all attributes, only write what is written on the page. Do not make up new phrases or words. -Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ def LOB_CHECK(bu_dict, td_dicts, page): return f"""### PAGE START ### {page} ### PAGE END - Service: {bu_dict['SERVICE']} - Methodology: {bu_dict['FULL_METHODOLOGY']} +Service: {bu_dict['SERVICE']} +Methodology: {bu_dict['FULL_METHODOLOGY']} - Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects: +Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects: - {td_dicts} +{td_dicts} - Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. - """ +Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. +""" def TOP_DOWN_METAL_LEVEL(LOB, page): diff --git a/src/table_funcs.py b/src/table_funcs.py index ce29301..86b5db6 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -1,6 +1,7 @@ import re import pandas as pd +import json import ast def extract_tables(text): @@ -33,29 +34,29 @@ def align_and_format_tables(text_dict): aligned_text_dict = {} for key, text in text_dict.items(): if 'Table Start' in text: - print(key) table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) for table_text in table_texts: - # Extract pretable text - pretable = table_text.split('{')[0].strip() + try: + # Extract pretable text + pretable = table_text.split('{')[0].strip() - # Extract and format table text - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table = json.loads(table_only) - table_formatted = format_table(table) + # Extract and format table text + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table = json.loads(table_only) + table_formatted = format_table(table) - # Align - if text.count(pretable) == 2: - # Remove original table - aligned_text = text.replace(table_text, "") - # Align new table - aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - else: - aligned_text = text.replace(table_text, pretable + table_formatted) + # Align + if text.count(pretable) == 2: + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: + aligned_text = text.replace(table_text, pretable + table_formatted) + except: + aligned_text = text aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") - else: aligned_text_dict[key] = text - return aligned_text_dict - + return aligned_text_dict \ No newline at end of file diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb index 5695392..9361dbd 100644 --- a/src/test_notebook.ipynb +++ b/src/test_notebook.ipynb @@ -10,34 +10,48 @@ "import pandas as pd\n", "import re\n", "import ast\n", + "import json\n", "\n", "import prompt_funcs\n", "import config\n", "import utils\n", "import preprocess\n", "import table_funcs\n", - "import claude_funcs" + "import claude_funcs\n", + "import prompts" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 23, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keys(['2015-04-01 Marina Del Rey Hospital PPA.txt', '2017-12-01 Prime Healthcare-West Anaheim Medical Center Enhanced PPO AMD.txt', '2021-01-01 PH - Paradise Valley Hospital 7th AMD.txt', '2021-01-01 PH-West Anaheim Medical Center AMD.txt', '2023-01-01 UCSD Medical Center Non B&G AMD.txt', '2023-06-01 DH-Mercy Medical Center - Redding CDM LTR.txt', '47-1928508 Advanced Anesthesia.txt', '71-0986832 Pacific Gynecological Specialists.txt', 'Baz Allergy Asthma & Sinus.txt'])" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "input_dict = utils.read_input(path='../docs/healthnet/')\n", - "input_dict.keys()" + "input_dict = utils.read_input(path='../docs/test/')\n", + "input_dict.keys()\n", + "filename = '47-1928508 Advanced Anesthesia.txt'" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "contract_text = input_dict[filename]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "#### PREPROCESS ####\n", + "contract_text = preprocess.clean_newlines(contract_text)\n", + "text_dict = preprocess.split_text(contract_text)\n", + "#text_dict = table_funcs.align_and_format_tables(text_dict)\n", + "#text_dict = preprocess.highlight_rates(text_dict)" ] }, { @@ -46,157 +60,271 @@ "metadata": {}, "outputs": [], "source": [ - "contract_text = input_dict['2021-01-01 PH - Paradise Valley Hospital 7th AMD.txt']" + "#### TOP DOWN ####\n", + "td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page\n", + "\n", + "#### BOTTOM UP ####\n", + "bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", + "\n", + "#### COMBINE ####\n", + "# combined_results = prompt_funcs.td_bu_combine(td_results, bu_results, text_dict)\n", + "\n", + "# answer_df = pd.DataFrame(combined_results)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 92, "metadata": {}, "outputs": [], "source": [ - "#### PREPROCESS ####\n", - "contract_text = preprocess.clean_newlines(contract_text)\n", - "text_dict = preprocess.split_text(contract_text)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "6\n", - "7\n", - "8\n", - "9\n", - "13\n", - "19\n", - "20\n" - ] - }, - { - "data": { - "text/plain": [ - "[None, None, None, None, None, None, None]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "[print(k) for k in text_dict.keys() if 'Table Start' in text_dict[k]]" - ] - }, - { - "cell_type": "code", - "execution_count": 52, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'Document': 'Document Index SEVENTH AMENDMENT 1TO THE 1PROVIDER PARTICIPATION AGREEMENT 1BETWEEN 1PARADISE VALLEY HOSPITAL 1AND 1HEALTH NET OF CALIFORNIA, INC. 1 EXHIBIT A-1 6COMMERCIAL BENEFIT PROGRAMS 6FEE-FOR-SERVICE RATE EXHIBIT 6 EXHIBIT C-1 8MEDI-CAL BENEFIT PROGRAMS 8FEE-FOR-SERVICE RATE EXHIBIT 8 EXHIBIT D-1 9ENHANCEDCARE PPO BENEFIT PROGRAMS 9FEE-FOR-SERVICE RATE EXHIBIT 9 EXHIBIT D-2 10ENHANCED PPO BENEFIT PROGRAMS 10FEE-FOR-SERVICE PAYMENT CONDITIONS 10 Pro-ration Calculation: 13 ADDENDUM E 14FEE-FOR-SERVICE PAYMENT CONDITIONS 14 2.5.4 18Adjusting and Pro-rating Contract Rates Examples. 18 Pro-ration Calculation: 19 EXHIBIT F-1 20COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS 20FEE-FOR-SERVICE RATE EXHIBIT 20 EXHIBIT F-2 21COMMUNITYCARE BENEFIT PROGRAMS 21HOSPITAL QUALITY PERFORMANCE PROGRAM 21\\n\\n\\n\\n',\n", - " '1': '1 SEVENTH AMENDMENT TO THE PROVIDER PARTICIPATION AGREEMENT BETWEEN PARADISE VALLEY HOSPITAL AND HEALTH NET OF CALIFORNIA, INC. The Provider Participation Agreement (\"Agreement\") dated September 1, 2013, as subsequently amended, between Prime Healthcare Paradise Valley, LLC, dba Paradise Valley Hospital (\"Provider\") and Health Net of California, Inc. on behalf of itself and the subsidiaries and affiliates of Health Net, LLC (formerly known as Health Net, Inc.) (collectively \"Health Net\"), is hereby further amended effective January 1, 2021. Provider and Health Net hereby agree to amend the Agreement as follows: 1. A new Section 1.29, ALLOWABLE CHARGES, is added to read as follows: Allowable Charges is defined as Provider\\'s billed charges for Contracted Services. 2. Section 2.9, PRIOR AUTHORIZATION/REFERRAL, shall be deleted and replaced to read as follows: When either Prior Authorization and/or a Referral is required for the rendition of a health care service, the receipt of the required Prior Authorization and/or the required Referral, each being separate and distinct requirements, is a prerequisite to payment of Complete Claims for Covered Services in addition to confirming eligibility prior to delivering service as required by this Agreement and Health Net Policies. Health Net (or its designee as applicable) may rescind or modify its Prior Authorization, in a manner consistent with Health Net Policies, based on the eligibility of the Beneficiary and whether the rendered service is a Covered Service. However, when Health Net or its designee issues a Prior Authorization for a specific service under a Benefit Program regulated by the California Department of Managed Health Care or the California Department of Insurance, Health Net (or its designee as applicable) shall not rescind or modify its Prior Authorization after Provider has rendered the specified and authorized service in good faith and pursuant to the terms of the Prior Authorization for any reason, including, but not limited to, Health Net\\'s subsequent rescission, cancellation, or modification of the Beneficiary\\'s contract or Health Net\\'s subsequent determination that it did not make an accurate determination of the Beneficiary\\'s eligibility; provided, however that this section shall not be construed to expand or alter benefits available to a Beneficiary under such Benefit Program. 3. Subsection 3.5.1, ACCESS BY HEALTH NET, shall be deleted and replaced to read as follows: As of the effective date of this Agreement, the following Health Net subsidiaries and affiliates may at their option access this Agreement: Health Net of California, Inc., Health Net Life Insurance Company, Health Net Community Solutions, Inc., California Health and Wellness Plan, Wellcare of California, Inc., Health Net of Arizona, Inc., Health Net Administrative Services of Arizona, Inc., Arizona Complete Health, Health Net Health Plan of Oregon, Inc., Health Net Insurance Services, Inc., Health Net Federal Services, LLC, Managed Health Network, Inc., MHN Government Services, Inc., and Network Providers, LLC. Health Net may periodically modify the Health Net subsidiaries and/or affiliates which may access this Agreement. To the extent Health Net allows a Health Net subsidiary or affiliate to access this Agreement, Health Net binds such subsidiaries and/or affiliates to the terms and conditions of this Agreement. INTENTIONALLY LEFT BLANK PV 2020 amd Page 1 of 22 HealthNet\\n\\n',\n", - " '2': '2 4. Section 5.1, TERM, shall be deleted and replaced to read as follows: The term of this Agreement shall commence on the Effective Date and shall continue through December 31, 2023. Either party may terminate this Agreement effective as of the end of the Initial Term by providing at least one hundred twenty (120) days prior written notice to the other party. This Agreement shall automatically renew for successive one (1) year periods (\"Renewal Terms\"). 5. Section 6.3, ACCESS TO RECORDS AND AUDITS BY HEALTH NET, shall be deleted and replaced to read as follows: Subject only to applicable State and federal confidentiality or privacy laws, Provider shall permit Health Net or its designated representative access to Provider\\'s Records, at Provider\\'s place of business in this State during normal business hours, in order to audit, inspect, review, perform chart reviews, and duplicate such Records unless Provider agrees to a remote audit of such records. If performed on site, access to Records for the purpose of an audit shall be scheduled at mutually agreed upon times, upon at least thirty (30) business days prior written notice by Health Net or its designated representative, but not more than sixty (60) days following such written notice. Provider shall attend an exit interview upon completion of the audit for the purpose of obtaining a mutually agreed upon reconciliation of the initial audit findings. Such exit interview shall be conducted at a mutually agreeable time at Provider\\'s place of business in this State during normal business hours upon at least ten (10) days prior written notice by Health Net or its designated representative, but not more than thirty (30) days following such written notice. In the event Provider fails to attend the scheduled exit interview, Provider shall be deemed to have accepted the audit findings. If the audit was performed remotely, such exit interview shall consist of Health Net or its designated representative sharing its audit findings with Provider via written or electronic communications as mutually agreed upon by both parties. Provider shall be allowed thirty (30) days to contest the audit results. If not contested within thirty (30) days then Provider shall be deemed to have accepted the audit findings. Except when Health Net or its designated representative requests Provider Records that are related to care management or claims, Provider may be reimbursed reasonable fees associated with the retrieval of Provider\\'s Records and or duplication and preparation of requested Provider Records. Such fees shall not be more than those provided pursuant to applicable State law, including California Health and Safety code Section 123110. Actions taken as a result of Audit findings shall include adjustment for late charges, overcharges and undercharges. Any such adjustments shall be the net amounts as reflected in the audit findings. Any payments owed by one party to the other as the result of an audit shall be paid within thirty (30) days of the exit interview for such audit. 6. Section 7.6, BINDING ARBITRATION, is deleted and replaced to read as follows: If the parties are unable to resolve a Dispute through the dispute resolution process set forth in Section 7.5, the parties agree that such Dispute shall be settled by final and binding arbitration, upon the motion of either party, under the appropriate rules of the AAA or JAMS, as agreed by the parties. Any Arbitrator must be either a judge, or an attorney licensed to practice law in the State of California, who is in good standing with the State Bar, and has at least ten (10) years of experience with health care matters and the arbitration of managed care disputes. The parties each understand and agree that the exhaustion of any Health Net internal appeals processes and the Meet and Confer Process set forth in Section 7.5 (i) hereof are conditions precedent to binding arbitration under this Section 7.6. Notwithstanding the foregoing, nothing contained herein is intended to require binding arbitration of disputes alleging medical malpractice between a Beneficiary and Provider or to Disputes between the parties alleging breaches of confidentiality of Beneficiary information, trade secret or intellectual property obligations. The arbitration shall be conducted in San Francisco, California or Los Angeles, California. The written demand PV 2020 amd Page 2 of 22\\n\\n',\n", - " '3': '3 shall contain a detailed statement of the matter and facts and include copies of all material documents supporting the demand. Except as provided below, arbitration must be initiated within one (1) year after the date the Dispute arose by submitting a written notice to the other party. For the purposes of filing for arbitration regarding a Dispute over Health Net\\'s alleged non- payment or underpayment of Complete Claims under this Agreement, the parties agree that an arbitration shall be filed within one (1) year from the date of Health Net\\'s notice of its final determination on Provider\\'s appeal if any, on such Complete Claim. The parties expressly agree that the deadlines to file arbitration set forth above shall not be subject to waiver, tolling, alteration or modification of any kind or for any reason except for fraud. The failure to initiate arbitration before such deadlines shall mean the complaining party shall be barred forever from initiating such proceedings. All such arbitration proceedings shall be administered by the AAA or JAMS, as agreed by the parties; however, the arbitrator shall be bound by applicable State and federal law, and shall issue a written opinion setting forth findings of fact and conclusions of law. The parties agree that the decision of the arbitrator shall be final and binding as to each of them. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction. The arbitrator shall have no authority to make material errors of law or to award punitive damages or to add to, modify, or refuse to enforce any agreements between the parties. The arbitrator shall make findings of fact and conclusions of law and shall have no authority to make any award, which could not have been made by a court of law. The party against whom the award is rendered shall pay any monetary award and/or comply with any other order of the arbitrator within sixty (60) days of the entry of judgment on the award. The parties waive their right to a jury or court trial. The parties recognize and agree that theirs is an ongoing business relationship, which may lead to sensitive issues with respect to the exchange of information related to any Dispute. The parties agree, therefore, to enter into such protective orders (including without limitation creating a category of discovery documents \"for attorney\\'s eyes only\" to the extent feasible given the nature of the evidence and the Dispute). All discovery information shall be used solely and exclusively for arbitration of the Dispute between the parties and may not be used for any other purpose. After the arbitration award becomes final, each party shall return or destroy all documents obtained from the other party during the course of the arbitration that are subject to a protective order, and within thirty (30) days of such date shall provide to the other party an officer\\'s certificate signed under penalty of perjury indicating that all such information has been returned or destroyed. In all cases submitted to arbitration, the parties agree to share equally the administrative fee as well as the arbitrator\\'s fee, if any, unless otherwise assessed by the arbitrator. The administrative fees shall be advanced by the initiating party subject to final apportionment by the arbitrator in this award. The parties agree that the content and decision of any arbitration proceeding shall be confidential unless disclosure is required by applicable State or federal statutes or regulations. The terms of Section 7.5 and Section 7.6 shall survive termination of this Agreement. INTENTIONALLY LEFT BLANK PV 2020 and Page 3 of 22 Health Net\\n\\n',\n", - " '4': '4 7. Section 7.12, NOTICE, is deleted and replaced to read as follows: Notices regarding the breach, term, termination or renewal of this Agreement shall be given in writing in accordance with this Section 7.12 and shall be deemed given five (5) days following deposit in the U.S. mail, postage prepaid. If sent by hand delivery, overnight courier, or facsimile, notices shall be deemed given upon documentation of delivery. All notices shall be addressed as follows: Health Net: Regional Network Director Health Net of California, Inc. 3131 Camino del Rio North, Suite 600 San Diego, CA 92108 Vice President and Deputy General Counsel Health Net of California, Inc. 21281 Burbank Blvd Woodland Hills, CA 91367 Provider: Paradise Valley Hospital 2400 E 4th Street National City, CA 91950 Attn: Hospital CEO Prime Healthcare Management, Inc. 3480 E. Guasti Road Ontario, CA 91761 Attn: General Counsel and Health Plan Operations The addresses to which notices are to be sent may be changed by written notice given in accordance with this Section. 8. Exhibit A-1, COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit A-1. 9. Exhibit C-1, MEDI-CAL FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit C-1. 10. Exhibit D-1, ENHANCED PPO BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit D-1. 11. Exhibit D-2, ENHANCED PPO BENEFIT PROGRAMS, FEE-FOR-SERVICE PAYMENT CONDITIONS, is deleted and replaced with the attached Exhibit D-2. 12. Addendum E, FACILITY FEE-FOR-SERVICE PAYMENT CONDITIONS, is deleted and replaced with the attached Addendum E. 13. Exhibit F-1, COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT, is deleted and replaced with the attached Exhibit F-1. INTENTIONALLY LEFT BLANK PV 2020 and Page 4 of 22 Health Net\\n\\n',\n", - " '5': '5 14. A new Exhibit F-2, COMMUNITYCARE BENEFIT PROGRAMS HOSPITAL QUALITY PERFORMANCE PROGRAM, is added as the attached Exhibit F-2. THIS AMENDMENT shall be deemed to be part of the Agreement and, except as modified herein, the Agreement is hereby reaffirmed and declared in full force and effect. IN WITNESS WHEREOF, the parties hereto have executed this Amendment by their officers duly authorized to be effective on the date and year first written above. PRIME HEALTHCARE PARADISE VALLEY LLC HEALTH NET OF CALIFORNIA, INC. dba Paradise Valley Hospital Digitally signed by Yina Shabamia Valentina Shabanian Date: 2020.12.11 Edual dz 16:47:59 -08\\'00\" Edward Gong Valentina T. Shabanian Regional Vice President, Managed Care Regional Health Plan Officer 11/30/2020 Date Date Federal Tax Identification Number: 20-5837239 PV 2020 amd Page 5 of 22 Health Net\\n\\n',\n", - " '6': \"6 EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C-Section and Vaginal Delivery (Mom and Baby), Codes: MS-DRGs: 768, 783-788, 796-798, 805-807, 1/1/21 - 12/31/21: $12,600/3 day case rate. Additional days $4,000/day, 1/1/22 - 12/31/22: $13,104/3 day case rate. Additional days $4,160/day, 1/1/23 and thereafter: $13,628/3 day case rate. Additional days $4,326/day, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes: 0170, 0171, 0179, 1/1/21 - 12/31/21: $1,000/day, 1/1/22 - 12/31/22: $1,040/day, 1/1/23 and thereafter: $1,082/day, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services with a codeable Medicare DRG, 1/1/21 - 12/31/21: 167.5% of Medicare Allowable for the entire stay, 1/1/22 - 12/31/22: 174.2% of Medicare Allowable for the entire stay, 1/1/23 and thereafter: 181.2% of Medicare Allowable for the entire stay, \\nCategory of Service: All other Inpatient Services with a non- codeable Medicare DRG, 1/1/21 - 12/31/21: 60% of Allowable Charges, not to exceed $7,000/day, 1/1/22 - 12/31/22: 60.6% of Allowable Charges, not to exceed $7,280/day, 1/1/23 and thereafter: 61.2% of Allowable Charges, not to exceed $7,571/day, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Ambulatory Surgery, Codes: Revenue Codes: 0360, 0369, 0490, 0499, 0750, and applicable CPT Codes, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $6,000/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $6,240/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $6,490/visit, \\nCategory of Service: Emergency Room Visit, Codes: Revenue Codes: 0450, 0451, 0452, 0459, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $4,110/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $4,274/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,445/visit, \\nCategory of Service: Emergency Room - Urgent Care Services (Facility), Codes: Revenue Codes: 0456, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $3,820/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $3,973/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,132/visit, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 50% of Allowable Charges, not to exceed $3,820/visit, 1/1/22 - 12/31/22: 50.5% of Allowable Charges, not to exceed $3,973/visit, 1/1/23 and thereafter: 51% of Allowable Charges, not to exceed $4,132/visit, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under Commercial Benefit Programs pursuant to Addendum A, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 6 of 22 G HealthNet'\\n\\n\\n\\n\\n\",\n", - " '7': '7 No multiple ambulatory surgery payment methodology applies. PV 2020 amd Page 7 of 22 HealthNet\\n\\n\\nNone: Exclusions, 50% of Allowable Charges, not to exceed $3,820/visit: Inpatient and Outpatient, \\nAll other Outpatient Services: Surgical Implants, Prosthetics, Orthotics, Pacemakers, : Inpatient: Revenue Codes: 0274, 0275, 0276, 0278 Outpatient: Revenue Codes: 0274, 0275, 0276, 0278 or applicable HCPC Codes, 50% of Allowable Charges, not to exceed $3,820/visit: Included in above rates when each Revenue Code listed is less than $3,100 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: Included in above rates when each Revenue Code listed is less than $3,193 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: Included in above rates when each Revenue Code listed is less than $3,289 in aggregate Allowable Charges, \\nAll other Outpatient Services: Surgical Implants, Prosthetics, Orthotics, Pacemakers, : Inpatient: Revenue Codes: 0274, 0275, 0276, 0278 Outpatient: Revenue Codes: 0274, 0275, 0276, 0278 or applicable HCPC Codes, 50% of Allowable Charges, not to exceed $3,820/visit: 40% of Allowable Charges is applicable to each Revenue Code if at or over $3,100 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: 40.4% of Allowable Charges is applicable to each Revenue Code if at or over $3,193 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: 40.8% of Allowable Charges is applicable to each Revenue Code if at or over $3,289 in aggregate Allowable Charges, \\nAll other Outpatient Services: High Cost Drugs, : Revenue Codes: 0630-0636, 50% of Allowable Charges, not to exceed $3,820/visit: Included in above rates when each Revenue Code listed is less than $6,000 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: Included in above rates when each Revenue Code listed is less than $6,180 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: Included in above rates when each Revenue Code listed is less than $6,365 in aggregate Allowable Charges, \\nAll other Outpatient Services: High Cost Drugs, : Revenue Codes: 0630-0636, 50% of Allowable Charges, not to exceed $3,820/visit: 39.6% of Allowable Charges is applicable to each Revenue Code if at or over $6,000 in aggregate Allowable Charges, 50.5% of Allowable Charges, not to exceed $3,973/visit: 40% of Allowable Charges is applicable to each Revenue Code if at or over $6,180 in aggregate Allowable Charges, 51% of Allowable Charges, not to exceed $4,132/visit: 40.4% of Allowable Charges is applicable to each Revenue Code if at or over $6,365 in aggregate Allowable Charges, \\n\\n\\n',\n", - " '8': \"8 EXHIBIT C-1 MEDI-CAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBITCodes: Inpatient Services, \\nCategory of Service: Acute Rehabilitation, Codes: Revenue Codes: 118, 128, 138, 148, 158, Compensation: 105% of the Medi-Cal Rehabilitation Rate, \\nCategory of Service: Skilled Nursing Facility, Codes: Revenue Codes: 110, 120, 130, Compensation: 105% of the facility-specific published Medi-Cal per diem rate, \\nCategory of Service: Sub-Acute, Codes: Revenue Codes: 190-194, 199, Compensation: 105% of the facility-specific published Medi-Cal per diem rate, \\nCategory of Service: All other Inpatient Services, Compensation: 105% of APR-DRG based on the current statewide wage-adjusted base rate in effect at the time of service, \\nCodes: Outpatient Services, \\nCategory of Service: Outpatient Services, Compensation: 105% of the current Medi-Cal Fee Schedule, subject to any changes made by the State of California, \\nCategory of Service: Pharmaceuticals without an established Medi-Cal Value, Compensation: Average Wholesale Price (AWP) - 18%, \\nCategory of Service: Unlisted Procedures Procedures with Medi-Cal Units not Established but with a Medicare Value Procedures without Medi- Cal Units and without a Medicare Established Value, Compensation: 80% of the Current Medicare Allowable 15% of Allowable Charges, \\nCategory of Service: Surgical Implants, Prosthetics, Orthotics, Pacemakers for Ambulatory Surgery only, Codes: Revenue Codes 274, 275, 276, 278, Compensation: Cost + 5% for Ambulatory Surgery Only, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Exhibit C-3, Health Net shall pay and Provider shall accept as payment in full for non-capitated Medically Necessary Covered Services delivered pursuant to this Addendum C, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 8 of 22 Health Net\\n\\n\\n\\n\\n\",\n", - " '9': \"9 EXHIBIT D-1 ENHANCEDCARE PPO BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C- Section/Vaginal Delivery (Mom and Baby), Codes: DRG Codes: 768, 783- 788, 796-798, 805-807, 1/1/21 - 12/31/21: $9,500/case, 1/1/22 - 12/31/22: $9,880/case, 1/1/23 and thereafter: $10,275/case, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes: 0170, 0171, 0179, 1/1/21 - 12/31/21: $800/case, 1/1/22 - 12/31/22: $832/case, 1/1/23 and thereafter: $865/case, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services, Codes: All inpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 130% of Medicare Allowable, 1/1/22 - 12/31/22: 135.2% of Medicare Allowable, 1/1/23 and thereafter: 140.6% of Medicare Allowable, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Outpatient Services, Codes: All outpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 130% of Medicare Allowable, 1/1/22 - 12/31/22: 135.2% of Medicare Allowable, 1/1/23 and thereafter: 140.6% of Medicare Allowable, \\nCategory of Service: Provider-Based Billing, Codes: Rev Codes: 0510-0519, 1/1/21 - 12/31/21: Not covered, 1/1/22 - 12/31/22: Not covered, 1/1/23 and thereafter: Not covered, \\nCategory of Service: Outpatient Services that are not priced or codeable per Medicare guidelines, 1/1/21 - 12/31/21: 23.8% of Allowable Charges, not to exceed $2,980 per visit, 1/1/22 - 12/31/22: 24% of Allowable Charges, not to exceed $3,099 per visit, 1/1/23 and thereafter: 24.2% of Allowable Charges, not to exceed $3,223 per visit, \\n Subject to the terms of this Addendum, including without limitation the Payment Conditions set forth in Exhibit D-2 of this Addendum, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Contracted Services delivered under Benefit Programs pursuant to this Addendum, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 9 of 22 Health Net\\n\\n\\n\\n\\n\",\n", - " '10': '10 EXHIBIT D-2 ENHANCED PPO BENEFIT PROGRAMS FEE-FOR-SERVICE PAYMENT CONDITIONS The Payment Conditions set forth in this Exhibit D-2 supplement Health Net Policies. 1. Application of 72-Hour Rule. Payments made to Provider for inpatient Covered Services shall constitute payment for all of Provider\\'s charges relating to a Covered Person\\'s pre-admission testing for the same condition occurring within seventy-two (72) hours prior to an admission, including, but not limited to, charges for laboratory services, pathology services, radiology services, and medical/surgical supplies. If Member is admitted directly from the Emergency room or from an Observation level of care, the inpatient reimbursement shall be inclusive of all such emergency and/or observation related facility charges. 2. Admissions for Same or Related Diagnoses. Inpatient admissions for the same or a related diagnoses occurring within forty-eight (48) hours following a discharge in connection with a previous admission shall be considered part of the previous admission and are not separately reimbursable. 3. Hospital-Acquired Conditions and Provider Preventable Conditions. Payment to Provider under this Addendum shall comply with state and federal laws requiring reduction of payment or non-payment to a Provider for \"Hospital-Acquired Conditions\" and for \"Provider Preventable Conditions\" as such terms (or the reasonable equivalents thereof) are defined under applicable state and federal laws. 4. Never Events. Each Provider shall use best efforts to comply with applicable state and federal reporting or other requirements relating to Never Events and/or Serious Adverse Events, as the applicable term is defined by the National Quality Forum or by state or federal law. Providers shall not bill, charge, collect a deposit from, seek compensation, remuneration or reimbursement from, or have any recourse against Health Net or Members for any charges associated with Never Events and/or Serious Adverse Events. To the extent Provider receives any payment in connection with a Never Event or Serious Adverse Event, Provider shall promptly refund such amount. 5. Level of Care. Reimbursement under this Addendum shall not exceed the Allowed Amount corresponding to the level of care authorized by Health Net. 6. Payment for Professional Services. Payment for inpatient Covered Services under this Addendum includes payment for professional Covered Services (including but not limited to services provided by hospital-based physicians, Certified Registered Nurse Anesthetists or other professionals) that are provided in connection with such inpatient Covered Services and billed under the Provider\\'s tax identification number and provider identification number. Such professional Covered Services will not be separately reimbursable. Professional Covered Services billed by independent physicians under their own tax identification number and provider identification number are reimbursable directly to such physicians. 7. Payment for Multiple Procedures. Where multiple outpatient surgical or scope procedures performed on a Member during a single occasion of surgery, reimbursement will be as follows: i) the procedure for which the Allowed Amount under this Addendum is greatest will be reimbursed at one hundred percent (100%) of such Allowed Amount; ii) the procedures with second and third greatest Allowed Amounts under this Addendum shall each be reimbursed at fifty percent (50%) of such Allowed Amounts; iii) any additional procedures will not be eligible for reimbursement. 8. Multiple Dates of Service on a Single Claim Form. Provider shall identify on the applicable claim form each date of service when submitting claims spanning multiple dates of service. 9. Provider-Based Billing. Provider-Based Billing (as defined herein) will not be reimbursed under this Compensation Schedule as they are included as part of the compensation for professional fees under this Agreement. Neither Health Net nor the Member shall be responsible for such Provider-Based Billing. \"Provider-Based Billing\" are amounts charged by a clinic or facility as a technical component, or for PV 2020 amd Page 10 of 22\\n\\n',\n", - " '11': '11 overhead, in connection with professional services rendered in a clinic or facility, and include but are not limited to services billed using Revenue Codes 0510-0519. 10. Therapy Services. Provider shall bill therapy services with the appropriate modifier designating the type of therapy provided, when applicable. 11. National Provider Identifier/Effect of Exclusion or Suspension. Notwithstanding anything to contrary contained herein, Health Net shall not pay, and shall have no obligation to pay, any claim submitted by Provider based on an order or referral that does not include the National Provider Identifier (\"NPI\") for the ordering or referring physician. Neither Health Net nor the Member shall have any obligation to pay any claim submitted by Provider if the Provider has been excluded or suspended from participation in Medicare, Medicaid, or any other federal or state health care program. 12. Code Change Updates. Health Net utilizes nationally recognized coding structures (including, without limitation, revenue codes, CPT codes, HCPCS codes, ICD codes, national drug codes, ASA relative values, etc., or their successors) for basic coding and descriptions of the services rendered. Updates to billing-related codes shall become effective on the date (\"Code Change Effective Date\") that is the later of: (i) the first day of the month following sixty (60) days after publication by the governmental agency having authority over the applicable Product of such governmental agency\\'s acceptance of such code updates, (ii) the effective date of such code updates as determined by such governmental agency or (iii) if a date is not established by such governmental agency or the applicable Product is not regulated by such governmental agency, the date that changes are made to nationally recognized codes. Such updates may include changes to service groupings. Claims processed prior to the Code Change Effective Date shall not be reprocessed to reflect any such code updates. 13. Fee Change Updates. Updates to the fee schedule shall become effective on the effective date of such fee schedule updates, as determined by Health Net (\"Fee Change Effective Date\"). The date of implementation of any fee schedule updates, i.e. the date on which such fee change is first used for reimbursement (\"Fee Change Implementation Date\"), shall be the later of: (i) the first date on which Health Net is reasonably able to implement the update in the claims payment system; or (ii) the Fee Change Effective Date. Claims processed prior to the Fee Change Implementation Date shall not be reprocessed to reflect any updates to such fee schedule, even if service was provided after the Fee Change Effective Date. 14. Reimbursement Service Grouping. Not applicable. 15. Payment under this Compensation Schedule. All payments under this Compensation Schedule are subject to the terms and conditions set forth in the Agreement and Health Net Policies. 16. \"Allowable Charges\" means Provider\\'s billed charges for Contracted Services. 17. Medicare/CMS Allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 18. Health Net may use a vendor for the benefit administration of certain Covered Services related to substance abuse and behavioral health benefits. In the event of a conflict between this Agreement and the vendor\\'s contract with Provider, the vendor\\'s contract shall prevail when responsible for benefit administration. 19. Adjustments to Compensation Based on CMS Changes. In order to neutralize the impact of CMS changes to the DRG fee schedule for current and future years for all Commercial products, the parties agree to adjust the Inpatient rates based on DRGs in the above fee schedule as follows: 19.1 Inpatient. Health Net will pull a claims report of Provider\\'s Inpatient claims (both paid and denied) from Health Net\\'s system for dates of service January 1, 2019 through December 31, 2019 for PV 2020 amd Page 11 of 22 HealthNet\\n\\n',\n", - " '12': '12 Provider\\'s review and approval. Once the data set is agreed to, Health Net will price said claims using the DRG fee schedule effective October 1, 2019, and using the DRG fee schedule effective October 1, 2020. If Health Net\\'s analysis shows that the resulting trend comparing the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2019 DRG fee schedule to the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2020 DRG fee schedule is greater than or less than one percent (1%) as verified by Provider, the parties will execute an amendment with pro-rated Inpatient rates that counteract the DRG change. The parties will repeat this process each year using each year\\'s October 1 DRG fee schedule (i.e., the Inpatient claims data set for the 10/1/2021 DRG calculation will be cy2020, etc.). All Commercial products and all Prime Health hospitals will be included in the analysis. The data set will only include Commercial Inpatient claims that are reimbursed at the % of Medicare rate. Inpatient claims that are reimbursed at case rates etc. (i.e. OB rates) will be excluded from the analysis. 19.2 Medicare/CMS Allowable. Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 19.3 Time Frame for Analysis. CMS implements the Inpatient DRG fee schedule changes effective every October 1st, the parties will target January 31st of each year to complete the analysis. 19.4 Adjusting and Pro-rating Contract Rates Examples. Adjusted IP Contract Rate = \"Current IP Rate/(1+DRG Trend)\". Example A1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 167.5%/1.03=162.6%. Adjusted IP Contract Rate Year 2= 174.2%/1.03=169.1% Adjusted IP Contract Rate Year 3= 181.2%/1.03=175.9% Example A2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is 3%. Current HMO IP rates - Year 2: 169.1%, Year 3: 175.9%. Adjusted IP Contract Rate Year 2= 169.1%/1.03=164.2%. Adjusted IP Contract Rate Year 3= 175.9%/1.03=170.8%. Example A3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is 3%. Current HMO IP rates - Year 3: 170.8%. Adjusted IP Contract Rate Year 3= 170.8%/1.03=165.8%. Example B1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is - 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/0.97=172.7. Adjusted IP Contract Rate Year 2= 174.2%/0.97=179.6%. Adjusted IP Contract Rate Year 3= 181.2%/0.97=186.8%. Example B2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is - 3%. Current HMO IP rates - Year 2: 179.6%, Year 3: 186.8%. Adjusted IP Contract Rate Year 2= 179.6%/0.97=185.1%. Adjusted IP Contract Rate Year 3= 186.8%/0.97=192.6%. Example B3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is - 3%. PV 2020 amd Page 12 of 22\\n\\n',\n", - " '13': '13 Current HMO IP rates - Year 3: 192.6%. Adjusted IP Contract Rate Year 3= 192.6%/0.97=198.5%. Pro-Rating Adjustment Due to Delay in Implementing Adjusted Contract Rates: Pro-Rated Rate = \"Adjusted Contract Rate+[(Adjusted Contract Rate-Current Rate)*(x/(12-x))]. X = Delay in months in implementing the Adjusted Contract Rates. Inpatient: Compared to October (i.e. if rates implemented eff 2/1, X=4). Example A) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 162.6%+[(162.6%-167.5%)*(4/(12-4))]=160.2%. Pro-rating only applies to current year rates. No pro-rating for future years. Example B) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 172.7%+[(172.7%-167.5%)*(4/(12-4))]=175.3%. Pro-rating only applies to current year rates. No pro-rating for future years. If there is a need to adjust the Year 3 rates due to the changes in Medicare fee schedules (in January 2023), the Year 3 pro-rated rates will expire at the end of Year 3 (December 2023 - This will depend on the actual effective date of this agreement. Currently assumes it will be effective 12/1/2020). The rates thereafter will be \"Adjusted Contract Year 3 Rates\" (Non Pro-Rated). Pro-ration Calculation: PV 2020 amd Page 13 of 22\\n\\n\\n Example: No adjustment for delay in implementation and no inflationary increase to the base rate {\\'Example:\\': [\\'Example\\', \\'\\', \\'100 % of Medicare (no trend)\\', \\'Rate (no adjustment)\\', \\'Payment (not pro- rated)\\', \\'\\', \\'Example:\\', \\'100 % of Medicare (3% trend)\\', \\'Rate (pro- rated 4 months delay)\\', \\'Payment (pro-rated 4 months delay)\\'], \\'No adjustment\\': [\\'\\', \\'1\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'Base rate\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'for\\': [\\'Delayed\\', \\'2\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'increased\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'delay in\\': [\\'Months 1-4\\', \\'3\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'for\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'implementation\\': [\\'\\', \\'4\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'inflation,\\', \\'10,300\\', \\'167.5%\\', \\'17,253\\'], \\'\\': [\\'\\', \\'Total\\', \\'\\', \\'\\', \\'201,000\\', \\'\\', \\'\\', \\'\\', \\'\\', \\'201,000\\'], \\'and no\\': [\\'\\', \\'6\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'for the\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'inflationary\\': [\\'New\\', \\'7\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'delayed\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'increase\\': [\\'Rate Updated\\', \\'8\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'period\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'to the\\': [\\'Months\\', \\'9\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\'], \\'base rate\\': [\\'5-12\\', \\'10\\', \\'10,000\\', \\'167.5%\\', \\'16,750\\', \\'\\', \\'\\', \\'10,300\\', \\'160.2%\\', \\'16,499\\']} Example: No adjustment for delay in implementation and no inflationary increase to the base rate Pro-ration Calculation:Current Contract rate: DRG trend, 167.5%: 3%, \\nCurrent Contract rate: Adjust contract rate, 167.5%: 162.6%, \\nCurrent Contract rate: Pro-rated rate (x=4), 167.5%: 160.2%, \\n\\n\\n',\n", - " '14': '14 ADDENDUM E FEE-FOR-SERVICE PAYMENT CONDITIONS The Payment Conditions set forth in this Addendum E supplement Health Net Policies. I. PAYMENT CONDITIONS APPLICABLE TO (1) PAYMENT RATES FOR COMMERCIAL BENEFIT PROGRAMS, INCLUDING SALUD CON HEALTH NET AND COMMUNITYCARE HMO AND PURECARE PPO (EXCLUDING MEDI-CAL BENEFIT PROGRAM, ENHANCEDCARE PPO, and CALMEDICONNECT BENEFIT PROGRAM), NOT BASED ON MEDICARE/CMS ALLOWABLE; AND (2) PAYMENT RATES, FOR MEDICARE BENEFIT PROGRAMS, BASED ON MEDICARE/CMS ALLOWABLE: 1.1 The codes listed in the applicable rate exhibit shall be used by the parties for the purpose of defining the services for which payment will be made under this Agreement. Provider shall utilize the specified codes listed in the applicable rate exhibit when submitting Complete Claims for Covered Services under this Agreement. 1.2 The parties acknowledge that applicable coding agencies periodically issue coding modifications. Such modifications may be implemented by Health Net within sixty (60) days of the date Health Net receives the modification from the applicable coding agency or notification from Provider. The parties agree that in the event such coding modifications have the effect of changing a payment amount in this Agreement, the resulting payment amount change shall be effective on a prospective basis. The parties further agree to reasonably and in good faith discuss any contract rate amendment that may be appropriate based on comprehensive and substantive coding changes by the applicable coding agency within ninety (90) days of written notification by either party. Any and all rate modifications that may result from such contract amendment shall be effective on a prospective basis. 1.3 Only one per diem or case rate shall be payable a) for each Beneficiary who is admitted prior to midnight and remains past midnight; b) if a Beneficiary is admitted and discharged during the same day, provided that such admission and discharge is not within twenty-four (24) hours of a prior discharge; c) for mother and newborn child (children) when both mother and newborn child (children) are in the facility on the same day, unless the child (children) is in the neonatal intensive care unit. Pre-admission testing completed within one (1) day of admission will be included in the per diem and will not be billed separately by Provider. A per diem is an all-inclusive daily payment, including but not limited to observation services, supplies, implants, prosthetics, durable medical equipment, and pharmaceuticals with the exception of specific carve-outs or exclusions identified on the applicable rate exhibit. Inpatient Professional services billed by the Provider on behalf of the hospital-based physician(s) are included in the per diems or case rates. Admission kits are included in the per diem rate. Any Outpatient Services delivered to a Beneficiary within twenty-four (24) hours prior to an admission for Inpatient Services for the same medical condition at the same facility are included in the inpatient rate. 1.4 The rates set forth in this Agreement apply only to inpatient admissions commencing on or after the Effective Date of this Agreement. The applicable rate of payment that is in effect on the date an inpatient admission commences shall apply to the entire length of stay regardless of any rate change that may occur during the length of stay. Notwithstanding the foregoing, inpatient case rates are applicable beginning the first day of admission if a case rate procedure is performed at any time during the inpatient stay. 1.5 All inpatient case rates shall include any readmission charges if the patient is readmitted within forty-eight (48) hours following discharge for complications related to the prior admission. Should non-standard conditions occur, the second admission within forty-eight (48) hours may be eligible for reimbursement, provided such admission is in accordance with Health Net Policies. INTENTIONALLY LEFT BLANK PV 2020 amd Page 14 of 22 HealthNet\\n\\n',\n", - " '15': \"15 1.6 If a Beneficiary is admitted to the hospital from the emergency room and/or observation room, separate reimbursement for the emergency room, observation room and other outpatient charges shall not be paid; however, the applicable inpatient reimbursement shall be reimbursed as all-inclusive reimbursement. The rate payable for Emergency Services is an all-inclusive case rate, including but not limited to supplies, implants, prosthetics, durable medical equipment, observation charges, diagnostic services, therapeutic services, pharmaceuticals, and other services incidental to the emergency room visit, with the exception of specific exclusions identified on the applicable rate exhibit. 1.7 The rate payable for outpatient surgery is an all-inclusive case rate, including but not limited to supplies, implants, prosthetics, durable medical equipment, emergency room charges, observation charges, diagnostic services, therapeutic services and pharmaceuticals with the exception of specific exclusions identified on the applicable rate exhibit. Only one (1) case rate shall be paid for each Beneficiary per date of service. 1.8 Other Outpatient case rates are an all-inclusive payment, including but not limited to supplies, diagnostic services, therapeutic services and pharmaceuticals with the exception of specific exclusions identified on the applicable rate exhibit. Only one (1) case rate shall be paid for each Beneficiary per date of service; provided, however, that if one or more unique case rate procedure is performed on different body parts, each applicable case rate shall be reimbursed. 1.9 Intentionally Left Blank 1.10 In the event that a Beneficiary is transported either within Provider's Facility or to another facility for a service that is within Provider's customary scope of practice, but due to malfunction or other cause cannot be delivered, Provider shall be solely financially responsible for claims incurred for such transportation. 1.11 For purposes of this Agreement, New Service/Technology is defined as a service, procedure, device, test, or other item that, as of the effective date of this Agreement: (i) is not performed by Provider, or (ii) is not Covered by Health Net under the applicable Benefit Program, or (iii) for which there is no assigned CPT or other relevant code defined. New Service/Technology does not include a new code that is assigned as a change to an existing service, procedure, device, test, or other item. If Provider offers a New Service/Technology after the effective date of this Agreement for which Provider desires reimbursement under this Agreement, Provider agrees to give Health Net sixty (60) days prior written notice that Provider requests that such New Service/Technology be included in Contracted Services, and that Provider receive a different reimbursement rate for the use of such New Service/Technology for Beneficiaries. Health Net agrees to determine whether such New Service/Technology qualifies as a Covered Service and whether Health Net desires to include such New Service/Technology in Contracted Services within thirty (30) days of the date of Provider's notice hereunder. Health Net's determination of whether a New Service/Technology qualifies as a Covered Service shall be made in Health Net's sole discretion, and shall be final and binding on Provider with respect to this Agreement. Only if Health Net agrees that such New Service/Technology is a Covered Service and desires to include such New Service/Technology in Contracted Services, the parties shall enter into good faith negotiations to modify Provider's rates to reflect the New Service/Technology. Any rate modification shall be made only prospectively. 1.12 The lowest room and board billed charge rate shall apply to all days billed within the same revenue code level of care. 1.13 All professional Covered Services billed by Participating Providers on a CMS 1500 form under Provider's federal tax identification number shall be reimbursed at one hundred percent (100%) of CMS allowable in effect on the date of service. PV 2020 amd Page 15 of 22 HealthNet\\n\\n\",\n", - " '16': '16 1.14 If any payment rates are based on Provider\\'s billed charges, upon Health Net\\'s written request, Provider shall send Health Net an electronic copy of Provider\\'s current Charge Master. Provider shall notify Health Net in writing via certified letter at least sixty (60) days prior to making any changes to Provider\\'s Charge Master (\"Notice\"). The rates in Provider\\'s Charge Master on September 1, 2013, for example the date Provider\\'s most recent fiscal year began, shall be the \"Baseline Charge Master.\" Changes to the Baseline Charge Master shall be measured during the twelve (12) month period immediately preceding the date of the Notice (the \"Measurement Period\"). The Notice shall be signed by Provider\\'s Chief Financial Officer (or equivalent) and shall include a statement of any and all changes made to the Baseline Charge Master during the Measurement Period together with the effective date of such change. All changes included in the Notice shall be calculated using the methodology set forth herein. Any change to Provider\\'s Charge Master during the term of this Agreement shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. The terms of this section shall survive termination and/or subsequent renegotiation of this Agreement with respect to Contracted Services rendered during the term of the Agreement and Health Net shall retain its right to pursue recovery of any overpayments and/or to adjust the rates under the terms and provisions of the Agreement and applicable law. Health Net reserves all of its rights under this Agreement, at law and in equity related to any Charge Master dispute that may exist or arise in the future. The methodology for calculating Changes to the Charge Master and rates pursuant to this section is as follows: (a) First Year: Provider agrees that any Notice delivered during the first year of this Agreement shall reflect no changes made to the Baseline Charge Master. In the event Provider has made changes to its Baseline Charge Master, any such change shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement, such that there will not be a change to Health Net\\'s reimbursement to Provider during the first year of this Agreement. (b) Subsequent Years: For each subsequent Measurement Period, Provider shall not change Provider\\'s Baseline Charge Master in a manner that results in an increase in reimbursement under this Agreement by more than three percent (3%) in the aggregate, unless expressly agreed upon in writing by the parties hereto. Upon request, Provider shall promptly provide Health Net with a Notice that completely and accurately sets forth any and all changes during the Measurement Period. Health Net shall have the right to reconcile and reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. In the event Health Net exercises its right to reconcile and adjust payment, Health Net shall give Provider at least thirty (30) days prior written notice, which notice shall be attached to the applicable rate exhibit in this Agreement. Any resulting change in payment rates shall be effective upon thirty (30) days prior written notice to Provider and shall be applicable to all claims adjudicated after the thirty (30) day notice period has expired regardless of the dates of service for such claims. (c) Both parties agree that Charge Master changes shall be calculated using Provider\\'s aggregate weighted average charge increase where weights equal volume for specific charges. Health Net may periodically review Provider\\'s Charge Master filed with the State or Health Net\\'s historical claims data. If Health Net determines as a result of such review that Provider has made changes to the Baseline Charge Master but has failed to give Health Net Notice, the parties agree that such changes shall result in a proportional change to reduce the percentage for those categories of service reimbursed on a discount off of charges basis and increase the stop-loss threshold (if applicable) set forth in the applicable addendum and/or exhibit to this Agreement. Any such change in rates shall be effective upon thirty (30) days prior written notice to Provider and shall be applicable to all claims adjudicated after the thirty (30) day notice period has expired regardless of the dates of service for such claims. Health Net shall use the above methodology for calculating changes to the Charge Master and rates pursuant to this section. PV 2020 amd Page 16 of 22 Health Net\\n\\n',\n", - " '17': \"17 II. PAYMENT CONDITIONS APPLICABLE TO PAYMENT RATES BASED ON MEDICARE/CMS ALLOWABLE. The following payment conditions are in addition to those set forth in Article I above, and apply to payment rates based upon a percentage of Medicare/CMS allowable rates, and methodology. Notwithstanding the foregoing, to the extent any payment condition set forth in this Article II conflicts with a payment condition in Article I, the payment condition set forth in this Article II shall control with respect to payment based upon a percentage of Medicare/CMS allowable rates, and methodology: 2.1 Only one DRG/case rate shall be payable for each Beneficiary who is admitted, provided that such admission is not within 48 hours of a prior discharge and for Benefit Program Beneficiaries other than Medicare Beneficiaries, for mother and newborn child (children) when both mother and newborn child (children) are in the facility on the same day, unless the child (children) is in the neonatal intensive care unit. Any outpatient services delivered to a Beneficiary within forty-eight (48) hours prior to an admission for Inpatient Services at the same facility are included in the inpatient rate. 2.2 Provider shall be reimbursed in accordance with CMS guidelines for all Outpatient Services. 2.3 Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME) and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass-throughs. 2.4 Provider agrees to adhere to CMS/Medicare billing and compensation guidelines for all services. 2.5 Adjustments to Compensation Based on CMS Changes. In order to neutralize the impact of CMS changes to the DRG fee schedule for current and future years for all Commercial products, the parties agree to adjust the Inpatient rates based on DRGs in the above fee schedule as follows: 2.5.1 Inpatient. Health Net will pull a claims report of Provider's Inpatient claims (both paid and denied) from Health Net's system for dates of service January 1, 2019 through December 31, 2019 for Provider's review and approval. Once the data set is agreed to, Health Net will price said claims using the DRG fee schedule effective October 1, 2019, and using the DRG fee schedule effective October 1, 2020. If Health Net's analysis shows that the resulting trend comparing the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2019 DRG fee schedule to the sum of one hundred percent (100%) of Medicare Allowable for each claim in the agreed upon claim data set resulting from the October 1, 2020 DRG fee schedule is greater than or less than one percent (1%) as verified by Provider, the parties will execute an amendment with pro-rated Inpatient rates that counteract the DRG change. The parties will repeat this process each year using each year's October 1 DRG fee schedule (i.e., the Inpatient claims data set for the 10/1/2021 DRG calculation will be cy2020, etc.). All Commercial products and all Prime Health hospitals will be included in the analysis. The data set will only include Commercial Inpatient claims that are reimbursed at the % of Medicare rate. Inpatient claims that are reimbursed at case rates etc. (i.e. OB rates) will be excluded from the analysis. 2.5.2 Medicare/CMS Allowable. Medicare/CMS allowable for Inpatient Services is defined as Medicare DRG including, DME, DSH, Capital (including Capital IME), and other Medicare payments, and including outliers as defined by Medicare/CMS, but excluding Operating IME and pass- throughs. 2.5.3 Time Frame for Analysis. CMS implements the Inpatient DRG fee schedule changes effective every October 1st, the parties will target January 31st of each year to complete the analysis. INTENTIONALLY LEFT BLANK PV 2020 amd Page 17 of 22 HealthNet\\n\\n\",\n", - " '18': '18 2.5.4 Adjusting and Pro-rating Contract Rates Examples. Adjusted IP Contract Rate = \"Current IP Rate/(1+DRG Trend)\". Example A1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is 3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/1.03=162.6%. Adjusted IP Contract Rate Year 2= 174.2%/1.03=169.1%. Adjusted IP Contract Rate Year 3= 181.2%/1.03=175.9%. Example A2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is 3%. Current HMO IP rates - Year 2: 169.1%, Year 3: 175.9%. Adjusted IP Contract Rate Year 2= 169.1%/1.03=164.2%. Adjusted IP Contract Rate Year 3= 175.9%/1.03=170.8%. Example A3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is 3%. Current HMO IP rates - Year 3: 170.8%. Adjusted IP Contract Rate Year 3= 170.8%/1.03=165.8%. Example B1) for Year 1 Adjustment assuming 10/1/20 vs 10/1/19 DRG trend is -3%. Current HMO IP rates - Year 1: 167.5%, Year 2: 174.2%, Year 3: 181.2%. Adjusted IP Contract Rate Year 1= 167.5%/0.97=172.7%. Adjusted IP Contract Rate Year 2= 174.2%/0.97=179.6%. Adjusted IP Contract Rate Year 3= 181.2%/0.97=186.8%. Example B2) for Year 2 Adjustment assuming 10/1/21 vs 10/1/20 DRG Trend is -3%. Current HMO IP rates - Year 2: 179.6%, Year 3: 186.8%. Adjusted IP Contract Rate Year 2= 179.6%/0.97=185.1%. Adjusted IP Contract Rate Year 3= 186.8%/0.97=192.6%. Example B3) for Year 3 Adjustment assuming 10/1/22 vs 10/1/21 DRG Trend is -3%. Current HMO IP rates - Year 3: 192.6%. Adjusted IP Contract Rate Year 3= 192.6%/0.97=198.5%. Pro-Rating Adjustment Due to Delay in Implementing Adjusted Contract Rates: Pro-Rated Rate = \"Adjusted Contract Rate+[(Adjusted Contract Rate-Current Rate)*(x/(12-x))]. X = Delay in months in implementing the Adjusted Contract Rates. Inpatient: Compared to October (i.e. if rates implemented eff 2/1, X=4). Example A) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 162.6%+[(162.6%-167.5%)*(4/(12-4))]=160.2% Pro-rating only applies to current year rates. No pro-rating for future years. Example B) If the Adjusted Contract Year 1 Rates are implemented with a 4 months delay; Pro-rated Year 1 Rate: 172.7%+[(172.7%-167.5%)*(4/(12-4))]=175.3% Pro-rating only applies to current year rates. No pro-rating for future years. If there is a need to adjust the Year 3 rates due to the changes in Medicare fee schedules (in January 2023), the Year 3 pro-rated rates will expire at the end of Year 3 (December 2023 - This will depend on the actual effective date of this agreement. Currently assumes it will be effective 12/1/2020). The rates thereafter will be \"Adjusted Contract Year 3 Rates\" (Non Pro-Rated). PV 2020 amd Page 18 of 22\\n\\n',\n", - " '19': \"19 Pro-ration Calculation: PV 2020 amd Page 19 of 22 HealthNet\\n\\n\\n Example: No adjustment for delay in implementation and no inflationary increase to the base rate {'Example:': ['', '', '100 % of Medicare (no trend)', 'Rate (no adjustment)', 'Payment (not pro- rated)', '', 'Example:', '100 % of Medicare (3% trend)', 'Rate (pro- rated 4 months delay)', 'Payment (pro-rated 4 months delay)'], 'No adjustment': ['', '1', '10,000', '167.5%', '16,750', '', 'Base rate', '10,300', '167.5%', '17,253'], 'for': ['Delayed', '2', '10,000', '167.5%', '16,750', '', 'increased', '10,300', '167.5%', '17,253'], 'delay in': ['Months 1-4', '3', '10,000', '167.5%', '16,750', '', 'for', '10,300', '167.5%', '17,253'], 'implementation': ['', '4', '10,000', '167.5%', '16,750', '', 'inflation,', '10,300', '167.5%', '17,253'], '': ['', 'Total', '', '', '201,000', '', '', '', '', '201,000'], 'and no': ['', '6', '10,000', '167.5%', '16,750', '', 'for the', '10,300', '160.2%', '16,499'], 'inflationary': ['New', '7', '10,000', '167.5%', '16,750', '', 'delayed', '10,300', '160.2%', '16,499'], 'increase': ['Rate Updated', '8', '10,000', '167.5%', '16,750', '', 'period', '10,300', '160.2%', '16,499'], 'to the': ['Months', '9', '10,000', '167.5%', '16,750', '', '', '10,300', '160.2%', '16,499'], 'base rate': ['5-12', '10', '10,000', '167.5%', '16,750', '', '', '10,300', '160.2%', '16,499']} Example: No adjustment for delay in implementation and no inflationary increase to the base rate Pro-ration Calculation:Current Contract rate: DRG trend, 167.5%: 3%, \\nCurrent Contract rate: Adjust contract rate, 167.5%: 162.6%, \\nCurrent Contract rate: Pro-rated rate (x=4), 167.5%: 160.2%, \\n\\n\\n\",\n", - " '20': \"20 EXHIBIT F-1 COMMUNITYCARE AND PURECARE BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT1/1/21 - 12/31/21: Inpatient Services, \\nCategory of Service: OB/C- Section/Vaginal Delivery (Mom and Baby), Codes: DRG Codes: 768, 783-788, 796-798, 805-807, 1/1/21 - 12/31/21: $9,500/case, 1/1/22 - 12/31/22: $9,880/case, 1/1/23 and thereafter: $10,275/case, \\nCategory of Service: Boarder Baby, Codes: Revenue Codes:0170, 0171, 0179, 1/1/21 - 12/31/21: $800/case, 1/1/22 - 12/31/22: $832/case, 1/1/23 and thereafter: $865/case, \\nCategory of Service: Rehabilitation, Codes: Revenue Code: 0128, 1/1/21 - 12/31/21: $2,500/diem, 1/1/22 - 12/31/22: $2,600/diem, 1/1/23 and thereafter: $2,704/diem, \\nCategory of Service: All other Inpatient Services, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\n1/1/21 - 12/31/21: Outpatient Services, \\nCategory of Service: Outpatient Services, Codes: All outpatient codes, except those services set forth below, 1/1/21 - 12/31/21: 23.8% of Allowable Charges, not to exceed $3,094 per visit, 1/1/22 - 12/31/22: 24% of Allowable Charges, not to exceed $3,218 per visit, 1/1/23 and thereafter: 24.2% of Allowable Charges, not to exceed $3,346 per visit, \\nCategory of Service: Ambulatory Surgery, Codes: Revenue Codes 360, 369, 490, 499, 750, and applicable CPT codes, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Emergency Room Visit, Codes: Revenue Codes 450, 451, 452, 459, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Emergency Room - Urgent Care Services (Facility), Codes: Revenue Code 456, 1/1/21 - 12/31/21: 135% of Medicare Allowable, 1/1/22 - 12/31/22: 140.4% of Medicare Allowable, 1/1/23 and thereafter: 146% of Medicare Allowable, \\nCategory of Service: Observation, Codes: Revenue Codes: 0760, 0762, 1/1/21 - 12/31/21: 140% of Medicare Allowable, 1/1/22 - 12/31/22: 145.6% of Medicare Allowable, 1/1/23 and thereafter: 151.4% of Medicare Allowable, \\n Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under CommunityCare Benefit Programs pursuant to Addendum F, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 20 of 22 CP HealthNet\\n\\n\\n\\n\\n\",\n", - " '21': '21 EXHIBIT F-2 COMMUNITYCARE BENEFIT PROGRAMS HOSPITAL QUALITY PERFORMANCE PROGRAM I. FFS COMPENSATION. Provider shall render Covered Services to Members and shall accept the compensation set forth in the applicable Exhibit as payment-in-full for Covered Services rendered to such Members. II. INCENTIVE PROGRAM. Effective for measurement/calendar year 2020, Provider shall qualify for participation in the quality incentive program if at least four (4) of the following requirements are met: 2.1 Electronic Health Records (EHR) Access. Provider must, upon request, grant Health Net access to all Health Net Member EHRs (i.e., medical records access and individual export of medical records for Health Net members via Health Information Exchange that Health Net participates with or via direct EHR access on Provider EHR portal). 2.2 Nurse Care Manager Access. Provider must, upon request, allow Health Net care management nurses access to Provider facilities in order to perform any needed case management duties. 2.3 Notification of Admission/Discharge/Transfer. Provider must provide Health Net with notification within twenty-four (24) hours, or the next business day if admitted/discharged on a weekend, for ninety percent (90%) of Provider\\'s admissions, discharges, transfers (two events per member per occurrence). Alternatively, Provider must provide Health Net with Admission/Discharge data via a Health Information Exchange that Health Net participates with or send it directly to Health Net so that Health Net receives admission and discharge notifications electronically (i.e. HL7/industry standard messages) in near real time. 2.4 Hospital Consumer Assessment of Healthcare Providers and Systems (HCAHPS) Score. Provider achieves both of the following: 1 A score of seventy-five percent (75%) or greater for the HCAHPS question \"Patients who reported YES they would definitely recommend the hospital\", as reported in the most recent survey period by CMS. 2 A composite score that meets the most recent Medicare Stars 4 star threshold for the HCAHPS Composite Measure \"Discharge Information\", as reported in the most recent survey period by CMS, based on the following questions: a) .did hospital staff talk with you about whether you would have the help you needed when you left the hospital?\" b) \" did you get information in writing about what symptoms or health problems to look out for after you left the hospital?\" 2.5 California Maternal Quality Care Collaborative (CMQCC) Maternal Data Center Reporting and Data Release. Provider with maternity service participates in the CMQCC Maternal Data Center Reporting initiative and signs an authorization release to share their hospital-specific results with Health Net. III. PROVIDER PARTICIPATION. If Provider satisfies the requirements above, then participation in the quality incentive program shall be granted. VI. METRICS. The quality incentive program is comprised of selected metrics, each carrying its own separate award value. The metrics are as follows: PV 2020 amd Page 21 of 22 HealthNet\\n\\n',\n", - " '22': \"22 4.1 Hospital Acquired Conditions (HACs). Provider Standardized Infection Ratio (SIR) for each of the five (5) HACs listed below is either less than 1.0 for inpatient admissions during the calendar year, or Provider must reduce its SIR by at least 0.25 for inpatient admissions in relation to its prior year experience. Each individual HAC is measured separately and is worth twenty percent (20%) of the overall HACs incentive. HACs measured are listed below: 1. Central line-associated bloodstream infection (CLABSI) 2. Catheter-associated urinary tract infections (CAUTI) 3. Methicillin-resistant staphylococcus aureus (MRSA) 4. Clostridium difficile (C.Diff) 5. Colorectal Surgical Site Infections (SSI-Colon) Source: California Department of Public Health (CDPH) portal and National Healthcare Safety Network (NHSN), if a measure is not reported on the CDPH portal, as posted on January 31 following the calendar year. Value: Twenty percent (20%) of the maximum incentive for the measurement year. 4.2 All-Cause Readmissions. Provider must have a risk-adjusted all-cause/any facility Member readmission rate that is ten percent (10%) lower for the Benefit Program for the measurement/calendar year compared to the provider year-end baseline performance from the prior calendar year. Readmission rates at or below four percent (4%) automatically qualify for this incentive regardless of improvement level from prior measurement years. Readmission rates that are higher than the original baseline performance will not qualify for this incentive. Source: Claims submitted to Health Net Value: Forty percent (40%) of the maximum incentive for the measurement year 4.3 Discharge Instructions. Upon review, Provider must demonstrate to Health Net that at least ninety-five percent (95%) of Members received discharge instructions including discharge information summarizing care and the following: name of hospitalist during inpatient stay, treatment/procedures provided, discharge diagnosis, current medication list and allergies, test results and a notation of pending/no pending test(s), and patient care instructions. Source: Provider EMR system via direct access or extract or other electronic notification system approved by Health Net Value: Twenty percent (20%) of the maximum incentive for the measurement year. 4.4 NTSV C-Sections. Provider NTSV C-Section rate provided by CMQCC is either twenty- three and nine tenths percent (23.9%) or less or ten percent (10%) lower for the CommunityCare Benefit Program for the measurement/calendar year compared to the provider year-end baseline performance from the prior calendar year. NTSV C-Section rate is defined as a live baby born at or beyond thirty-seven (37.0) weeks during a first pregnancy, who is a singleton, and is in the vertex position during cesarean birth. Source: California Maternal Quality Care Collaborative (CMQCC) Value: Twenty percent (20%) of the maximum incentive for the measurement year. V. PAYMENT. The maximum incentive for this program shall be equal to two percent (2%) of the inpatient fee-for-service (FFS) payments Health Net made to Provider during the measurement year related to providing Covered Services to Members in the measurement year for the applicable Benefit Program(s). Health Net shall confirm each of the requirements for the metrics above are met during the measurement year. If Health Net confirms one or more of the requirement are met, Provider shall receive a quality incentive payment equal to the combined percentage values of the maximum incentive. Payment shall be made within one-hundred and eighty (180) days after the end of the measurement year. Health Net reserves the right to discontinue or modify this incentive program with thirty (30) days' notice prior to the beginning of a measurement year. PV 2020 amd Page 22 of 22 Health Net\"}" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import json\n", - "\n", - "def format_table(table_json):\n", - " table_text = \"\"\n", - " for i in range(len(table_json[list(table_json.keys())[0]])):\n", - " for key in table_json.keys():\n", - " if table_json[key][i]:\n", - " table_text += key + ': ' + table_json[key][i] + ', '\n", - " table_text += '\\n'\n", - " return table_text\n", - "\n", - "\n", - "def align_and_format_tables(text_dict):\n", - " aligned_text_dict = {}\n", - " for key, text in text_dict.items():\n", - " if 'Table Start' in text:\n", - " print(key)\n", - " table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL)\n", - " for table_text in table_texts:\n", - " # Extract pretable text\n", - " pretable = table_text.split('{')[0].strip()\n", - "\n", - " # Extract and format table text\n", - " table_only = \"{\" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace(\"'\", '\"') + \"}\"\n", - " table = json.loads(table_only)\n", - " table_formatted = format_table(table)\n", - "\n", - " # Align\n", - " if text.count(pretable) == 2:\n", - " # Remove original table\n", - " aligned_text = text.replace(table_text, \"\")\n", - " # Align new table\n", - " aligned_text = aligned_text.replace(pretable, pretable + table_formatted)\n", + "def clean_td(td):\n", + " td_clean = []\n", + " for d in td:\n", + " new_d = {}\n", + " for k in d.keys():\n", + " if k not in ['page_num', 'Filename']:\n", + " if ',' in d[k]:\n", + " if 'DATE' not in k:\n", + " new_d[k] = [v.strip() for v in d[k].split(',')]\n", + " elif d[k] == 'N/A':\n", + " new_d[k] = []\n", " else:\n", - " aligned_text = text.replace(table_text, pretable + table_formatted)\n", - " aligned_text_dict[key] = aligned_text.replace(\"-------Table Start--------\", \"\").replace(\"-------Table End--------\", \"\")\n", - " \n", - " else:\n", - " aligned_text_dict[key] = text\n", - " \n", - " return aligned_text_dict\n", + " new_d[k] = [d[k]]\n", + " else:\n", + " new_d[k] = d[k]\n", + " td_clean.append(new_d)\n", + " return td_clean\n", "\n", - "align_and_format_tables(text_dict)" + "def get_unique_td(td):\n", + " unique_td_dict = {}\n", + " for d in td:\n", + " for key, value in d.items():\n", + " if key not in ['page_num', 'Filename']:\n", + " for v in value:\n", + " if key not in unique_td_dict.keys():\n", + " unique_td_dict[key] = [v]\n", + " else:\n", + " unique_td_dict[key].append(v)\n", + " final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()}\n", + " return final_dict\n", + "\n", + "def td_bu_combine(td, bu, text_dict):\n", + " bu_dicts = bu.copy()\n", + " td_dicts = td.copy()\n", + "\n", + " td_dicts_clean = clean_td(td_dicts)\n", + " unique_td = get_unique_td(td_dicts_clean)\n", + " \n", + " combined_list = []\n", + " for bud_ in bu_dicts:\n", + " bud = bud_.copy()\n", + " for key, value in unique_td.items():\n", + " if len(value) == 0: # No value in contract\n", + " bud.update({key : \"\"})\n", + " elif len(value) == 1: # 1 value in contract\n", + " bud.update({key : value[0]})\n", + " else: # More than one value in contract\n", + " # Check for value on the page\n", + " page_value = []\n", + " for d in td_dicts_clean:\n", + " if (d['page_num'] == bud['page_num']) and (key in d.keys()):\n", + " for v in d[key]:\n", + " page_value.append(v)\n", + " \n", + " if len(page_value) == 1: # One value on page\n", + " bud.update({key : page_value[0]})\n", + " elif len(page_value) > 1: # Multiple values on page\n", + " bud.update({key : ', '.join(page_value)})\n", + " elif len(page_value) == 0:\n", + " bud.update({key : \"\"}) \n", + " combined_list.append(bud)\n", + " return combined_list" ] }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 94, + "metadata": {}, + "outputs": [], + "source": [ + "combined = td_bu_combine(td_results, bu_results, text_dict)" + ] + }, + { + "cell_type": "code", + "execution_count": 95, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "6 EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT Subject to the terms of this Agreement, including without limitation the Payment Conditions set forth in Addendum E, Health Net shall pay and Provider shall accept as payment in full for Medically Necessary Covered Services delivered under Commercial Benefit Programs pursuant to Addendum A, the lesser of the rates listed below, or one hundred percent (100%) of Provider's Allowable Charges. PV 2020 amd Page 6 of 22 G HealthNet'\n", - "\n", - "\n", - "-------Table Start-------- EXHIBIT A-1 COMMERCIAL BENEFIT PROGRAMS FEE-FOR-SERVICE RATE EXHIBIT {'Category of Service': ['', 'OB/C-Section and Vaginal Delivery (Mom and Baby)', 'Boarder Baby', 'Rehabilitation', 'All other Inpatient Services with a codeable Medicare DRG', 'All other Inpatient Services with a non- codeable Medicare DRG', '', 'Ambulatory Surgery', 'Emergency Room Visit', 'Emergency Room - Urgent Care Services (Facility)', 'Observation'], 'Codes': ['', 'MS-DRGs: 768, 783-788, 796-798, 805-807', 'Revenue Codes: 0170, 0171, 0179', 'Revenue Code: 0128', '', '', '', 'Revenue Codes: 0360, 0369, 0490, 0499, 0750, and applicable CPT Codes', 'Revenue Codes: 0450, 0451, 0452, 0459', 'Revenue Codes: 0456', 'Revenue Codes: 0760, 0762'], '1/1/21 - 12/31/21': ['Inpatient Services', '$12,600/3 day case rate. Additional days $4,000/day', '$1,000/day', '$2,500/diem', '167.5% of Medicare Allowable for the entire stay', '60% of Allowable Charges, not to exceed $7,000/day', 'Outpatient Services', '50% of Allowable Charges, not to exceed $6,000/visit', '50% of Allowable Charges, not to exceed $4,110/visit', '50% of Allowable Charges, not to exceed $3,820/visit', '50% of Allowable Charges, not to exceed $3,820/visit'], '1/1/22 - 12/31/22': ['', '$13,104/3 day case rate. Additional days $4,160/day', '$1,040/day', '$2,600/diem', '174.2% of Medicare Allowable for the entire stay', '60.6% of Allowable Charges, not to exceed $7,280/day', '', '50.5% of Allowable Charges, not to exceed $6,240/visit', '50.5% of Allowable Charges, not to exceed $4,274/visit', '50.5% of Allowable Charges, not to exceed $3,973/visit', '50.5% of Allowable Charges, not to exceed $3,973/visit'], '1/1/23 and thereafter': ['', '$13,628/3 day case rate. Additional days $4,326/day', '$1,082/day', '$2,704/diem', '181.2% of Medicare Allowable for the entire stay', '61.2% of Allowable Charges, not to exceed $7,571/day', '', '51% of Allowable Charges, not to exceed $6,490/visit', '51% of Allowable Charges, not to exceed $4,445/visit', '51% of Allowable Charges, not to exceed $4,132/visit', '51% of Allowable Charges, not to exceed $4,132/visit']} -------Table End--------\n", - "\n", - "\n" - ] + "data": { + "text/plain": [ + "['Medicaid',\n", + " 'Medicaid',\n", + " 'Medicaid',\n", + " 'Medicaid',\n", + " 'Medicaid',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " 'Commercial',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '',\n", + " '']" + ] + }, + "execution_count": 95, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "print(text_dict['6'])" + "[c['CONTRACT_LOB'] for c in combined]" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" ] }, { From 31cb01872ce6f248d0c4c10cab0d78a1f999997f Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 24 May 2024 17:19:21 -0500 Subject: [PATCH 29/78] Added comments --- src/prompt_funcs.py | 16 +++++++++------- src/prompts.py | 7 ++++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index c8ff729..c7831b5 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -161,6 +161,7 @@ def run_top_down(filename, text_dict): return td_final # List of list of dictionaries +# Takes list of dictionaries, updates vals in dictionaries to be lists instead of strings. N/A->empty list, single val -> [single_val], multiple val -> [val1, val2] def clean_td(td): td_clean = [] for d in td: @@ -177,8 +178,9 @@ def clean_td(td): else: new_d[k] = d[k] td_clean.append(new_d) - return td_clean + return td_clean # List of dictionaries where values in each dict are lists +# Takes list of dictionaries, returns one dictionary with unique values def get_unique_td(td): unique_td_dict = {} for d in td: @@ -197,19 +199,19 @@ def td_bu_combine(td, bu, text_dict): td_dicts = td.copy() td_dicts_clean = clean_td(td_dicts) - unique_td = get_unique_td(td_dicts_clean) + unique_td = get_unique_td(td_dicts_clean) # Unique values in entire contract combined_list = [] for bud_ in bu_dicts: bud = bud_.copy() - for key, value in unique_td.items(): + for key, value in unique_td.items(): # value is list of unique values if len(value) == 0: # No value in contract bud.update({key : ""}) elif len(value) == 1: # 1 value in contract bud.update({key : value[0]}) else: # More than one value in contract # Check for value on the page - page_value = [] + page_value = [] # page_value = list of all values of key on the same page as bud for d in td_dicts_clean: if (d['page_num'] == bud['page_num']) and (key in d.keys()): for v in d[key]: @@ -218,11 +220,11 @@ def td_bu_combine(td, bu, text_dict): if len(page_value) == 1: # One value on page bud.update({key : page_value[0]}) elif len(page_value) > 1: # Multiple values on page - bud.update({key : ', '.join(page_value)}) + bud.update({key : ', '.join(page_value)}) # Faizan - Change line to prompt check (pass in page_value) (commit ce3f4c17f281c4c3851d9967c51a14468ee06230) elif len(page_value) == 0: - bud.update({key : ""}) + bud.update({key : ""}) # Faizan - Change line to prompt check previous page (pass in page_value) - (ce3f4c17f281c4c3851d9967c51a14468ee06230) combined_list.append(bud) - return combined_list + return combined_list # List of dictionaries def run_prompts(file_object): diff --git a/src/prompts.py b/src/prompts.py index 2506edd..a465112 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -144,7 +144,12 @@ def LOB_CHECK(bu_dict, td_dicts, page): Service: {bu_dict['SERVICE']} Methodology: {bu_dict['FULL_METHODOLOGY']} -Listed above is page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. Here are the entities, represented as dictionary objects: +Listed above is page of a contract, as well as a Service and Methodology found on this page. +Your job is to identify which of the following entities this specific Service is associated with. + +----Change Below---- + +Here are the entities, represented as dictionary objects: {td_dicts} From 81f34edc8d37cad477fe3b1c7074ceba7b883ad7 Mon Sep 17 00:00:00 2001 From: FaizanM2000 Date: Mon, 3 Jun 2024 03:05:57 -0500 Subject: [PATCH 30/78] added new postprocessing and bottom up and top down merging. also carveouts --- .gitignore | 3 + src/carveouts.py | 138 +++++++++ src/config.py | 9 +- src/integration_testing.ipynb | 565 ++++++++++++++++++++++++++++++++++ src/postprocess.py | 27 ++ src/postprocessingfuncs.py | 95 ++++++ src/prompt_funcs.py | 3 +- src/prompts.py | 36 ++- src/test_notebook.ipynb | 359 --------------------- 9 files changed, 867 insertions(+), 368 deletions(-) create mode 100644 src/carveouts.py create mode 100644 src/integration_testing.ipynb create mode 100644 src/postprocess.py create mode 100644 src/postprocessingfuncs.py delete mode 100644 src/test_notebook.ipynb diff --git a/.gitignore b/.gitignore index 8a56daa..2b36f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ texts/ myenv/ *.env *.txt +subset/ +output/ +docs/ diff --git a/src/carveouts.py b/src/carveouts.py new file mode 100644 index 0000000..18a2bc0 --- /dev/null +++ b/src/carveouts.py @@ -0,0 +1,138 @@ +import pandas as pd +from difflib import SequenceMatcher +import re +import time +import prompts +import claude_funcs + +def get_closest_substring_match(val, valid_values): + """ Returns the first match from valid_values using a case-insensitive substring match. """ + if pd.isna(val): + return None + val = val.strip().upper() + for valid_value in valid_values: + if valid_value.upper() in val: + return valid_value + return None + +def get_best_carveout_from_claude(carveout_list, service): + print("using claude to get best carveout...") + prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + +def check_prov_type_similarity(prov_type, carveout): + print("checking similarity between prov_type and carveout with claude...") + prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'" + while True: + try: + response = claude_funcs.invoke_claude_3(prompt, max_tokens=10) + return response.strip() + except Exception as e: + print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...") + time.sleep(60) + +def label_services(filepath, carveout_list, output_filepath): + # Read the CSV file + df = pd.read_csv(filepath) + + # Select the first 50 rows of the DataFrame + df = df.head(500) + + # Define primary service terms + primary_terms = ['Covered Services', 'Inpatient Services', 'Outpatient Services', 'Physician Services', + 'Inpatient', 'Outpatient', 'Physician', 'Medical', 'Surgical', 'Diagnostic', 'Therapeutic'] + + # Initialize columns for the results + df['carveout_matched'] = '' + df['label'] = '' + df['IS_CARVEOUT'] = '' + + # Initialize a nested dictionary to track occurrences for each Filename and TD_LOB + occurrence_tracker = {} + + carveout_count = 0 + + # Iterate through the rows in the DataFrame + for index, row in df.iterrows(): + service = str(row['SERVICE']) + prov_type = str(row['PROV_TYPE']) if pd.notna(row['PROV_TYPE']) else '' + td_lob = row['TD_LOB'] + filename = row['Filename'] + + # Initialize the nested dictionary for the filename and TD_LOB if not present + if filename not in occurrence_tracker: + occurrence_tracker[filename] = {} + if td_lob not in occurrence_tracker[filename]: + occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms} + + # Check if the service is a primary or similar to a primary + primary = get_closest_substring_match(service, primary_terms) + if primary: + # Ensure the primary term is initialized in the occurrence tracker + if primary not in occurrence_tracker[filename][td_lob]: + occurrence_tracker[filename][td_lob][primary] = 0 + + # Determine the label based on the count of this primary term for the given Filename and TD_LOB + count = occurrence_tracker[filename][td_lob][primary] + if count == 0: + df.at[index, 'label'] = 'primary' + elif count == 1: + df.at[index, 'label'] = 'secondary' + elif count == 2: + df.at[index, 'label'] = 'tertiary' + else: + df.at[index, 'label'] = 'additional' + occurrence_tracker[filename][td_lob][primary] += 1 + df.at[index, 'IS_CARVEOUT'] = 'N' + else: + # If PROV_TYPE is blank, continue with carveout determination + best_carveout_response = get_best_carveout_from_claude(carveout_list, service) + matched_carveout = best_carveout_response # Use the model response directly + + df.at[index, 'carveout_matched'] = matched_carveout + + if matched_carveout: + # Check similarity between PROV_TYPE and carveout + prov_type_similarity_score = SequenceMatcher(None, prov_type, matched_carveout).ratio() + if prov_type_similarity_score < 0.8: + prov_type_similarity = check_prov_type_similarity(prov_type, matched_carveout) + else: + prov_type_similarity = "True" + + if re.search(r'\b(True|Yes)\b', prov_type_similarity, re.IGNORECASE): + if matched_carveout not in occurrence_tracker[filename][td_lob]: + print('claude returned a match between provider type and service') + occurrence_tracker[filename][td_lob][matched_carveout] = 0 + count = occurrence_tracker[filename][td_lob][matched_carveout] + if count == 0: + df.at[index, 'label'] = 'primary' + elif count == 1: + df.at[index, 'label'] = 'secondary' + elif count == 2: + df.at[index, 'label'] = 'tertiary' + else: + df.at[index, 'label'] = 'additional' + occurrence_tracker[filename][td_lob][matched_carveout] += 1 + df.at[index, 'IS_CARVEOUT'] = 'N' + else: + df.at[index, 'IS_CARVEOUT'] = 'Y' + else: + df.at[index, 'IS_CARVEOUT'] = 'Y' + + carveout_count += 1 + + # Save the updated DataFrame to a new CSV file after every 10 carveouts + if carveout_count % 10 == 0: + df.to_csv(output_filepath, index=False) + print(f"Saved progress after processing {carveout_count} carveouts.") + + # Final save of the updated DataFrame to a new CSV file + df.to_csv(output_filepath, index=False) + +label_services('service.csv', prompts.get_carveout_list(), 'carveouts50.csv') diff --git a/src/config.py b/src/config.py index 3c8e05f..418548a 100644 --- a/src/config.py +++ b/src/config.py @@ -3,6 +3,7 @@ from datetime import datetime import boto3 from llama_index.llms.bedrock import Bedrock +#pip install llama-index-llms-bedrock # General Settings TEST = False # True to run test prompt - just for testing model connection @@ -28,12 +29,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXNEUYUPUY" -AWS_SECRET_ACCESS_KEY="3HljGiz+dqWBtj+dkqs/syyxoguUoTahdLyceEeo" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEBUaCXVzLWVhc3QtMiJHMEUCIQCX9LcYh9A28cLNA/lDEHJ6ZqrulN0FampT1ptNyX+9JAIgIFrmDiZS5KfHbZWptuvveZt2s7x5r0xDwCU6t6EgBAEqjAMIjv//////////ARAAGgw2NjAxMzEwNjg3ODIiDOLRiVmy5kw46/5mzCrgAscMrf+1rEm7kpwpmd+BHjLNV4ggEy0oqQQ6z5Ny5BV6Ouo0tRmXqOqSaI5/YKXeKsRo8s4lrDNNq8qIK0GxpugvmbqguKYfEzKwyihGi5IVVb2JGL1nBkL495v0w5j1fuXqNwB7VCPkJyJCqxDHeehgtQ1AymuVAiRowkWmr7vVvFQyKKcGsBJlIzBZyguTtJhT2HPCp8IW9LhQMS2XBAa4xCnHqpPZSYIoKyp2zPPwi3pBKm+7eE3NGH7V7/Vrl6IL29zNkk5K/IeqAcyBzaFgaCPilr5N6lSMeKlkz8w0ls3guZeaEjDRf/Mz3NvMD9HeUVqGfi2M+ODxudgEfmfH3Op0EC2g5fsIBu/uvcjJf7PgUo9xT6CH5P5HY/e6TLaZH55ioz468uaTfoNw10TZ/6iymqYEXSpLfqHejC6qCx2pfcYf1UizgD2rn22GfcGs8u6oKFSThJ4fETq1mogwzafCsgY6pgHl1EzoiRwiXqCDl3GsA162dYrapmEPqcIVXTTuNuYXbl7B3GQfxTmjHjqSqKFhieWEvydD4SSQAhSXz9tPsN69Myyelwg3t6r2EWGL+JiZXEFpO1peWJgu3xfd3F1tPnp7Nfv7bVGmuIvft1JYTmx20W/vH9Q4J7jM4tPqEeQs5fRORTdXcm10Yysuq5FYo+9bahUJnPYblcJveO30vVirwzVWvmPm" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFULN3ZDJ" +AWS_SECRET_ACCESS_KEY="biHj+LwFkcquA7hoOgWst66u7AAQcik+A8TnMSoO" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELr//////////wEaCXVzLWVhc3QtMiJHMEUCIEIZfQMj70zYiV1czvy0RdOdugmJ9YGZIPBk/wqUE8m1AiEAzA9jGoZfv0ZMZlBKTLHGT3Ax3rAQNVLYWqgo9jy3Ui8qiQMIQxAAGgw2NjAxMzEwNjg3ODIiDOYEfYdB5ykwPYhKOSrmAiHPCbfUR2lXoqyEvURpTFyOqLEmeVFHCHQoVyeFcXwQmjjZor+ui1dclq0asj35ZSScClpyQKZsfk5o5+76pxzTm1tlBez1buIFkel/9OngHNG4MvMwLCMLdI+ILtJy0ufLmky4oVgtSor5n2e5dLENqstqhXEREbGdxZ5gaF7zKg9Dk/VPQ8arzV0oEvxuPGbZ+Z640jtVR27oSU1pQQ4ayK+CM/jBMANlzO9BptRB+Phfm6oOfxYQskyIXRV+aff/3d/MVJs6X08U/YjN+qXx0pDLsyBxYQIpKZXIZvWrYRD/NM7DnPJF8DlTAWhNSDU0QJ4j2oTMKcaG0LHXmd1SejjgHXhK0ZiG2tcpNHPaIBFm/lWnaixOdxM9XUWckoTe+1FRmcblQ7YI2HtWsbgzU3B8BLmLvEPX+lJSDeJBuv1MGFA4ITa4YaRywBBdZTcSl9cwLGB1X9ksTga5cd1XZeW/218w5cfmsgY6pgEsq/5MsjzbPqT20XDAhlC7GUmY3K+glAHfm8nVnihVdfmfU2apbx0bcDu5Er9mngykbj0u24P8VbkmlozS2H2dDmCjqQeE2Ux2pOuUxfqtMlJVSjAO8MCVvjc2nxsv55iHDTHGSlaN8TmsoBqVRbsOmauWFPQ7gFJDqF7YADS+20NDsgblAeeE23eZU4A/mG1dWfIK5E58nAoKWvnumFyQnJ0sMPpN" # File Paths -LOCAL_PATH = 'docs/priority_health/' # Replace with local +LOCAL_PATH = 'subset/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/integration_testing.ipynb b/src/integration_testing.ipynb new file mode 100644 index 0000000..e6dce29 --- /dev/null +++ b/src/integration_testing.ipynb @@ -0,0 +1,565 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import pandas as pd\n", + "import re\n", + "import ast\n", + "import json\n", + "\n", + "import prompt_funcs\n", + "import config\n", + "import utils\n", + "import preprocess\n", + "import table_funcs\n", + "import claude_funcs\n", + "import prompts" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "function clean_td(td):\n", + " Initialize td_clean as an empty list\n", + " for each dictionary d in td:\n", + " Initialize new_d as an empty dictionary\n", + " for each key k and value v in d:\n", + " if 'DATE' in k:\n", + " new_d[k] = v as a list\n", + " else if k is not 'page_num' or 'Filename':\n", + " if v is a string with commas:\n", + " new_d[k] = split v into a list by commas\n", + " else if v is 'N/A':\n", + " new_d[k] = an empty list\n", + " else:\n", + " new_d[k] = v as a list\n", + " else:\n", + " new_d[k] = v\n", + " Append new_d to td_clean\n", + " return td_clean\n", + "\n", + "function get_unique_keys(dicts):\n", + " Initialize keys as an empty set\n", + " for each dictionary d in dicts:\n", + " Add all keys in d to keys\n", + " return keys\n", + "\n", + "function get_unique_date_fields(td_results):\n", + " Initialize unique_date_fields as an empty dictionary\n", + " for each dictionary td in td_results:\n", + " for each key k and value v in td:\n", + " if 'DATE' is in k:\n", + " if k is not in unique_date_fields:\n", + " Initialize unique_date_fields[k] as an empty set\n", + " Add v to unique_date_fields[k] as a list\n", + " Convert sets in unique_date_fields to lists\n", + " return unique_date_fields\n", + "\n", + "function GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return a prompt string with page, field_name, and values\n", + " else:\n", + " return a prompt string with page, SERVICE, and METHODOLOGY from bu_dict\n", + "\n", + "function merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) union get_unique_keys(bu_results)\n", + " date_fields = get_unique_date_fields(td_results)\n", + " Initialize merged_results as an empty list\n", + " for each dictionary bu in bu_results:\n", + " Initialize merged as a dictionary with keys from all_keys and values from bu\n", + " td_on_page = filter td_results by page_num matching bu's page_num\n", + " for each dictionary td in td_on_page:\n", + " for each key k and value v in td:\n", + " if merged[k] is empty:\n", + " merged[k] = v\n", + " else if v is a list and merged[k] is not a list:\n", + " merged[k] = v\n", + " else if v is a list:\n", + " Extend merged[k] with v\n", + " for each date_key and date_values in date_fields:\n", + " if date_key is not in merged or merged[date_key] is empty:\n", + " merged[date_key] = date_values\n", + " else:\n", + " Extend merged[date_key] with values not already in merged[date_key]\n", + " for each key in all_keys:\n", + " if merged[key] is a list with more than one value:\n", + " page_text = get text for bu's page_num from text_dict\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, key, merged[key])\n", + " response = invoke LLM with prompt\n", + " if response is valid:\n", + " merged[key] = response\n", + " else:\n", + " merged[key] = join merged[key] into a string\n", + " Append merged to merged_results\n", + " return merged_results\n", + "\n", + "function process_file(filename, input_dict):\n", + " contract_text = read contract text from input_dict using filename\n", + " Clean contract_text by removing newlines\n", + " Split contract_text into pages stored in text_dict\n", + " Run Top Down analysis and store results in td_results\n", + " Run Bottom Up analysis and store results in bu_results\n", + " Combine td_results and bu_results into combined_results\n", + " Create output directory based on filename\n", + " Save td_results, bu_results, and combined_results as CSV files in the output directory\n", + "\n", + "function process_all_files(input_dict):\n", + " for each filename in input_dict:\n", + " Call process_file with filename and input_dict\n", + " Print a message indicating the file has been processed\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "\n", + "\n", + "\n", + "def clean_td(td):\n", + " td_clean = []\n", + " for d in td:\n", + " new_d = {}\n", + " for k, v in d.items():\n", + " if 'DATE' in k:\n", + " new_d[k] = v if isinstance(v, list) else [v]\n", + " elif k not in ['page_num', 'Filename']:\n", + " if isinstance(v, str) and ',' in v:\n", + " new_d[k] = [item.strip() for item in v.split(',')]\n", + " elif v == 'N/A':\n", + " new_d[k] = []\n", + " else:\n", + " new_d[k] = [v] if isinstance(v, str) else v\n", + " else:\n", + " new_d[k] = v\n", + " td_clean.append(new_d)\n", + " return td_clean\n", + "\n", + "def get_unique_keys(dicts):\n", + " keys = set()\n", + " for d in dicts:\n", + " keys.update(d.keys())\n", + " return keys\n", + "\n", + "def get_unique_date_fields(td_results):\n", + " unique_date_fields = {}\n", + " for td in td_results:\n", + " for key, value in td.items():\n", + " if 'DATE' in key:\n", + " if key not in unique_date_fields:\n", + " unique_date_fields[key] = set()\n", + " unique_date_fields[key].update(value if isinstance(value, list) else [value])\n", + " # Convert sets to lists for consistency\n", + " for key in unique_date_fields:\n", + " unique_date_fields[key] = list(unique_date_fields[key])\n", + " return unique_date_fields\n", + "\n", + "def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Field: {field_name}\n", + "Values: {values}\n", + "\n", + "Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n", + "Your job is to identify which of these values is the most appropriate based on the page content.\n", + "\n", + "Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n", + "\"\"\"\n", + " else:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Service: {bu_dict['SERVICE']}\n", + "Methodology: {bu_dict['FULL_METHODOLOGY']}\n", + "\n", + "Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n", + "Your job is to identify which of the following entities this specific Service is associated with. \n", + "\n", + "----Change Below----\n", + "\n", + "Here are the entities, represented as dictionary objects:\n", + "\n", + "{td_dicts}\n", + "\n", + "Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", + "\"\"\"\n", + "\n", + "def merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n", + " print(\"All Keys:\", all_keys)\n", + " \n", + " date_fields = get_unique_date_fields(td_results)\n", + " print(\"Date Fields:\", date_fields)\n", + " \n", + " merged_results = []\n", + " for bu in bu_results:\n", + " merged = {key: bu.get(key, \"\") for key in all_keys}\n", + " td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n", + " \n", + " for td in td_on_page:\n", + " for key, value in td.items():\n", + " if not merged[key]:\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value and not isinstance(merged[key], list):\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value:\n", + " merged[key].extend(value)\n", + "\n", + " # Include unique date fields\n", + " for date_key, date_values in date_fields.items():\n", + " if date_key not in merged or not merged[date_key]:\n", + " merged[date_key] = date_values\n", + " else:\n", + " merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n", + "\n", + " # Use LLM for disambiguation and inference if necessary\n", + " for key in all_keys:\n", + " if isinstance(merged[key], list) and len(merged[key]) > 1:\n", + " page_num = bu['page_num']\n", + " page_text = text_dict.get(page_num, \"\")\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n", + " response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " try:\n", + " selected_value = response.strip()\n", + " merged[key] = selected_value\n", + " except Exception as e:\n", + " print(f\"Error processing LLM response for key {key}: {e}\")\n", + " merged[key] = ', '.join(merged[key])\n", + " \n", + " merged_results.append(merged)\n", + " \n", + " return merged_results\n", + "\n", + "def process_file(filename, input_dict):\n", + " contract_text = input_dict[filename]\n", + "\n", + " # Preprocess\n", + " contract_text = preprocess.clean_newlines(contract_text)\n", + " text_dict = preprocess.split_text(contract_text)\n", + " \n", + " # Run Top Down\n", + " td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n", + " print(\"TD Results:\", td_results)\n", + " \n", + " # Run Bottom Up\n", + " bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", + " print(\"BU Results:\", bu_results)\n", + " \n", + " # Combine\n", + " combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n", + " print(\"Combined Results:\", combined_results)\n", + " \n", + " # Create directories\n", + " base_filename = os.path.splitext(filename)[0]\n", + " output_dir = os.path.join('output', base_filename)\n", + " os.makedirs(output_dir, exist_ok=True)\n", + " \n", + " # Save results\n", + " pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n", + " pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n", + " pd.DataFrame(combined_results).to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + " \n", + "def process_all_files(input_dict):\n", + " for filename in input_dict.keys():\n", + " process_file(filename, input_dict)\n", + " print(f\"Processed {filename}\")\n", + "\n", + "# Example usage\n", + "input_dict = utils.read_input(path='subset')\n", + "process_all_files(input_dict)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Preprocessing:\n", + "clean_newlines(contract_text):\n", + "\n", + "Removes unwanted newline characters from the contract text.\n", + "split_text(contract_text):\n", + "\n", + "Splits the contract text into individual pages and stores them in a dictionary.\n", + "Top Down and Bottom Up Analysis:\n", + "run_top_down(filename, text_dict):\n", + "\n", + "Extracts data from each page of the contract text and returns a list of dictionaries.\n", + "run_bottom_up(filename, text_dict):\n", + "\n", + "Extracts data from the entire contract document and returns a list of dictionaries.\n", + "Cleaning and Combining Results:\n", + "clean_td(td_results):\n", + "\n", + "Ensures values in the Top Down results are in list format and preserves date fields.\n", + "get_unique_keys(td_results, bu_results):\n", + "\n", + "Collects all unique field names from both Top Down and Bottom Up results.\n", + "get_unique_date_fields(td_results):\n", + "\n", + "Aggregates all unique date fields from the Top Down results.\n", + "merge_results(td_results, bu_results, text_dict):\n", + "\n", + "Combines Top Down and Bottom Up results, ensuring all relevant fields are included.\n", + "Uses the LLM to resolve ambiguities and infer the most appropriate values when necessary when multiple are there.\n", + "Post-Processing:\n", + "post_process_combined(combined_df):\n", + "\n", + "Cleans and validates specific fields (CONTRACT_LOB, PRODUCT, CONTRACT_NETWORK) in the combined dataset.\n", + "Adds new columns (Corrected_LOB, Corrected_PRODUCT, Corrected_NETWORK) to the combined dataset.\n", + "Extracts and standardizes page numbers.\n", + "clean_columns_combined(df, column_name, valid_values, new_column_name):\n", + "\n", + "Cleans a column by applying an exact match check and updates it to a new column.\n", + "clean_pagenumbers(df):\n", + "\n", + "Extracts and standardizes page numbers in the combined dataset.\n", + "Saving the Results:\n", + "process_file(filename, input_dict):\n", + "\n", + "Runs the entire process for a single file, from reading and preprocessing to saving the final results.\n", + "Calls the necessary functions to extract, combine, and post-process the data.\n", + "Saves the results as CSV files in the output directory.\n", + "process_all_files(input_dict):\n", + "\n", + "Iterates through all input files and processes each one using process_file." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "import postprocess\n", + "import utils\n", + "\n", + "def clean_td(td):\n", + " td_clean = []\n", + " for d in td:\n", + " new_d = {}\n", + " for k, v in d.items():\n", + " if 'DATE' in k:\n", + " new_d[k] = v if isinstance(v, list) else [v]\n", + " elif k not in ['page_num', 'Filename']:\n", + " if isinstance(v, str) and ',' in v:\n", + " new_d[k] = [item.strip() for item in v.split(',')]\n", + " elif v == 'N/A':\n", + " new_d[k] = []\n", + " else:\n", + " new_d[k] = [v] if isinstance(v, str) else v\n", + " else:\n", + " new_d[k] = v\n", + " td_clean.append(new_d)\n", + " return td_clean\n", + "\n", + "def get_unique_keys(dicts):\n", + " keys = set()\n", + " for d in dicts:\n", + " keys.update(d.keys())\n", + " return keys\n", + "\n", + "def get_unique_date_fields(td_results):\n", + " unique_date_fields = {}\n", + " for td in td_results:\n", + " for key, value in td.items():\n", + " if 'DATE' in key:\n", + " if key not in unique_date_fields:\n", + " unique_date_fields[key] = set()\n", + " unique_date_fields[key].update(value if isinstance(value, list) else [value])\n", + " for key in unique_date_fields:\n", + " unique_date_fields[key] = list(unique_date_fields[key])\n", + " return unique_date_fields\n", + "\n", + "def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None):\n", + " if field_name and values:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Field: {field_name}\n", + "Values: {values}\n", + "\n", + "Listed above is a page of a contract, as well as the potential values for the {field_name} field.\n", + "Your job is to identify which of these values is the most appropriate based on the page content.\n", + "\n", + "Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself.\n", + "\"\"\"\n", + " else:\n", + " return f\"\"\"### PAGE START ### {page} ### PAGE END\n", + " \n", + "Service: {bu_dict['SERVICE']}\n", + "Methodology: {bu_dict['FULL_METHODOLOGY']}\n", + "\n", + "Listed above is a page of a contract, as well as a Service and Methodology found on this page. \n", + "Your job is to identify which of the following entities this specific Service is associated with. \n", + "\n", + "----Change Below----\n", + "\n", + "Here are the entities, represented as dictionary objects:\n", + "\n", + "{td_dicts}\n", + "\n", + "Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. \n", + "\"\"\"\n", + "\n", + "def merge_results(td_results, bu_results, text_dict):\n", + " all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results)\n", + " print(\"All Keys:\", all_keys)\n", + " \n", + " date_fields = get_unique_date_fields(td_results)\n", + " print(\"Date Fields:\", date_fields)\n", + " \n", + " merged_results = []\n", + " for bu in bu_results:\n", + " merged = {key: bu.get(key, \"\") for key in all_keys}\n", + " td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']]\n", + " \n", + " for td in td_on_page:\n", + " for key, value in td.items():\n", + " if not merged[key]:\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value and not isinstance(merged[key], list):\n", + " merged[key] = value\n", + " elif isinstance(value, list) and value:\n", + " merged[key].extend(value)\n", + "\n", + " for date_key, date_values in date_fields.items():\n", + " if date_key not in merged or not merged[date_key]:\n", + " merged[date_key] = date_values\n", + " else:\n", + " merged[date_key].extend([val for val in date_values if val not in merged[date_key]])\n", + "\n", + " for key in all_keys:\n", + " if isinstance(merged[key], list) and len(merged[key]) > 1:\n", + " page_num = bu['page_num']\n", + " page_text = text_dict.get(page_num, \"\")\n", + " prompt = GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key])\n", + " response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000)\n", + " try:\n", + " selected_value = response.strip()\n", + " merged[key] = selected_value\n", + " except Exception as e:\n", + " print(f\"Error processing LLM response for key {key}: {e}\")\n", + " merged[key] = ', '.join(merged[key])\n", + " \n", + " merged_results.append(merged)\n", + " \n", + " return merged_results\n", + "\n", + "def process_file(filename, input_dict):\n", + " contract_text = input_dict[filename]\n", + "\n", + " # Preprocess\n", + " contract_text = preprocess.clean_newlines(contract_text)\n", + " text_dict = preprocess.split_text(contract_text)\n", + " \n", + " # Run Top Down\n", + " td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page\n", + " print(\"TD Results:\", td_results)\n", + " \n", + " # Run Bottom Up\n", + " bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", + " print(\"BU Results:\", bu_results)\n", + " \n", + " # Combine\n", + " combined_results = merge_results(clean_td(td_results), bu_results, text_dict)\n", + " print(\"Combined Results:\", combined_results)\n", + " \n", + " # Convert to DataFrame\n", + " combined_df = pd.DataFrame(combined_results)\n", + " \n", + " # Post-process combined results\n", + " post_processed_combined_df = postprocess.postprocess_results(combined_df)\n", + " print(\"Post-processed Results:\", post_processed_combined_df)\n", + " \n", + " # Create directories\n", + " base_filename = os.path.splitext(filename)[0]\n", + " output_dir = os.path.join('output', base_filename)\n", + " os.makedirs(output_dir, exist_ok=True)\n", + " \n", + " # Save results\n", + " pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n", + " pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n", + " post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + "\n", + "def process_all_files(input_dict):\n", + " for filename in input_dict.keys():\n", + " process_file(filename, input_dict)\n", + " print(f\"Processed {filename}\")\n", + "\n", + "# Example usage\n", + "input_dict = utils.read_input(path='subset')\n", + "process_all_files(input_dict)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "# Usage example:\n", + "#save_combined_to_csv(combined, 'combined_output.csv')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "[c['CONTRACT_LOB'] for c in combined]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/postprocess.py b/src/postprocess.py new file mode 100644 index 0000000..29d4149 --- /dev/null +++ b/src/postprocess.py @@ -0,0 +1,27 @@ +import pandas as pd +import postprocessingfuncs + +def sanitize_combined(df): + """ Sanitize the combined data by removing brackets and cleaning values. """ + for column in df.columns: + df[column] = df[column].apply(postprocessingfuncs.sanitize_value) + return df + +def post_process_combined(df): + VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] + VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] + VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] + + # Sanitize the combined data + df = sanitize_combined(df) + + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') + postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT') + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + clean_df = postprocessingfuncs.clean_pagenumbers(df) + return clean_df + +def postprocess_results(combined_df): + combined_df = post_process_combined(combined_df) + return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py new file mode 100644 index 0000000..9dfdd49 --- /dev/null +++ b/src/postprocessingfuncs.py @@ -0,0 +1,95 @@ +import pandas as pd +import re +import difflib + +def sanitize_value(value): + """ Remove brackets from list items and clean the values. """ + if pd.isna(value): + return value + if isinstance(value, str): + value = value.strip('[]') + if ',' in value: + value = ', '.join([item.strip(" '") for item in value.split(',')]) + else: + value = value.strip(" '") + return value + +def exact_match(val, valid_values): + val = val.strip().upper() + for valid_val in valid_values: + if val == valid_val.upper(): + return valid_val + return None + +def clean_columns_combined(df, column_name, valid_values, new_column_name): + """ Cleans a column by applying an exact match check and updates it to a new column. """ + changes = {} + original_values = df[column_name].unique() + + def update_column(entry): + if pd.notna(entry): + terms = sanitize_value(entry).split(',') + for term in terms: + match = exact_match(term, valid_values) + if match: + return match + return None + + df[new_column_name] = df[column_name].apply(update_column) + cleaned_values = df[new_column_name].unique() + return original_values, cleaned_values, changes + +def get_closest_match(val, valid_values, similarity_threshold=0.5): + if pd.isna(val): + return None + val = val.strip().upper() + matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold) + return matches[0] if matches else None + +def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): + """ Applies fuzzy matching to a column in the dataframe and logs changes. """ + changes = {} + original_values = df[column_name].unique() + + def log_and_clean(entry): + if pd.notna(entry): + words = sanitize_value(entry).split(',') + cleaned_words = [] + for word in words: + cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold) + if cleaned_word and word.strip().upper() != cleaned_word: + changes[word.strip()] = cleaned_word + cleaned_words.append(cleaned_word if cleaned_word else word.strip()) + return ', '.join(cleaned_words) + return None + + df[column_name] = df[column_name].apply(log_and_clean) + cleaned_values = df[column_name].unique() + return original_values, cleaned_values, changes + +def extract_page_number(page_text): + match = re.search(r'Pages\s+(\d+)-\d+', page_text) + if match: + return match.group(1) + else: + return page_text + +def clean_pagenumbers(df): + df['Page'] = df['Page'].apply(extract_page_number) + return df + +def consolidate_csv(df, output_csv='output/consolidated.csv'): + df['page_num'] = df['Page'].apply(lambda x: x.split()[1]) + + grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({ + 'PRODUCT': lambda x: ', '.join(x.dropna().unique()), + 'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()), + 'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()), + 'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()), + 'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()), + 'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()), + 'page_num' : lambda x : ', '.join(x.dropna().unique()) + }).reset_index() + + grouped.to_csv(output_csv, index=False) + print(f"Consolidated CSV has been saved to {output_csv}") diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index c7831b5..8811716 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -8,7 +8,7 @@ import json from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate from llama_index.core import SummaryIndex, Document from llama_index.llms.bedrock import Bedrock - +#pip install llama-index-llms-bedrock import config import prompts import claude_funcs @@ -227,6 +227,7 @@ def td_bu_combine(td, bu, text_dict): return combined_list # List of dictionaries + def run_prompts(file_object): filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") diff --git a/src/prompts.py b/src/prompts.py index a465112..9e8fa06 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -186,10 +186,38 @@ ONLY return the answer, with no other commentary or explanation. """ - - - - +def get_carveout_list(): + return ['Emergency Department/Emergency Room', 'Emergency Department', 'Emergency Room', 'Observation', 'Surgery', 'General Surgery', +'Ambulatory Surgery', 'Intensive Care', 'Trauma', 'HIV', 'Human Immunodeficiency Virus', 'Major Joint Replacement', 'Transplant', 'OBGYN', +'Obstetrician/Gynecologist', 'Obstetrician', 'Gynecologist', 'Opthalmology & Vision', 'Opthalmology', 'Vision', 'Never Events', +'Medically Unnecessary', 'Medically Unnecessary Procedures', 'Not Medically Necessary', 'Not Medically Necessary Procedures', +'Experimental', 'Investigational', 'Unlisted Codes', 'Durable Medical Equipment', 'DME', 'Prosthetics & Orthotics', 'Prosthetics', +'Orthotics', 'Implants', 'Hearing Aids', 'Hearing', 'Anesthesia', 'Anesthesiology', 'Medical Pharmacy', 'Physician Administered Drugs', +'Global', 'Bundled or Unbundled Codes', 'Bundled Codes', 'Unbundled Codes', 'Multiple Procedure Reductions', 'Second Surgery', 'Subsequent Surgeries', +'Non-Behavioral Health Mid-Level Professionals', 'Physician Assistant', 'PA', 'Nurse Practicioner', 'NP', 'Non-Physician', +'Non-Physician Health Professionals', 'Technical Component', 'Professional Component', 'Laboratory', 'Pathology', 'Lab', 'Path', 'Lab/Path', +'Laboratory/Pathology', 'Radiology', 'Imaging', 'Radiology/Imaging', 'Mammography', 'Diagnostic', 'Pre-Admission Procedures', +'Post-Discharge Procedures', 'Readmission', 'Status Indicators', 'Stop Loss', 'Emergency Medical', 'EMS', 'NICU', 'Neonatal Intensive Care Unit', +'Vaccine for Children', 'VFC', 'Surgical Assistant', 'Physicians/Clinical Psychologists', 'Doctor of Nursing Practice', 'Osteopathic Medicine', +'Clinical Psychology', 'Audiologist', 'Chiropractors', 'Registered Dietician', 'AUD', 'DC', 'RD', 'Board Certified Behavioral Analysis', 'Behavioral Analysis', +'BCBA', 'Independent Licensures', 'Licensed Professional Counselor', 'Marriage and Family Therapist', 'Substance Abuse Counselor', 'Clinical Social Worker', +'Behavioral Health Outpatient Clinic', 'Physical Therapist', 'Occupational Therapist', 'Speech Therapist', 'PT', 'OT', 'ST', 'Transportation', +'Primary Care', 'Primary Care Behavioral Health', 'Behavioral Physician', 'Clinical Psychologist', 'Mid-Level Practicioner', 'Dentist', 'Dental', +'Supplies and Devices', 'Supplies', 'Devices', 'Immunizations', 'Obstetrical Epidural', 'Pediatric Subspecialties', 'Orthopedic Surgery', 'All Other Specialists', +'Specialty Care Physician', 'Pediatric Primary Care', 'Nurse Anesthetist', 'Early Periodic Screening, Diagnostic, and Treatment', 'EPSDT', +'Ancillary', 'Oncology', 'Cancer', 'Inpatient Physical Rehabilitation', 'Inpatient Rehabilitation', 'Outpatient Rehabilitation', 'Rehabilitation', 'Non-Behavioral Health Rehabilitation', +'Extracorporeal Shock Wave Lithotripsy', 'Cardiac', 'Special Care Unit', 'Skilled Nursing', 'Infusion', 'Specialty Care', +'Specialty Care Physician', 'Organ Acquisition', 'Blood Products', 'Blood Products Outpatient', 'Blood Products Inpatient', 'High Cost Drugs', 'Sleep Studies', +'NICU', 'Newborn Intensive Care Unit', 'Extracorporeal Membrane Oxygenation', 'Burns', 'Kyphoplasty', 'Cryosurgical Ablation of the Prostate', +'Transurethral Thermal Ablation', 'TUMT', 'Transurethral Needle Ablation', 'TUNA', 'Hyperbaric Treatment', 'Clinic Visit', 'Boarder Baby', +'Pediatric Intensive Care Unit', 'PICU', 'Psychiatric', 'Mental Health', 'Behavioral Health', 'Substance Abuse', +'Behavioral Health and Substance Abuse', 'Sub-Acute Facility Care', 'Unrouped Inpatient', 'All Other Acute', +'Neurology', 'Neurology Subspecialties', 'Automatic Implantable Cardioverter Defibrillator', 'Percutaneous Transluminal Coronary Angioplasty', +'Non-Coronary Angioplasty', 'Cardiac Catheters', 'Cardiovascular Surgery', 'Cardiac Surgery', 'Cesarean Birth', 'Cesarean Section', 'C-Section', +'Gamma-Knife Radio-Surgery Outpatient', 'DaVinci Robotic Assisted Surgery', 'Outpatient Electrophysiology with Ablation', 'Outpatient Electrophysiology', +'Magnetic Resonance Image', 'MRI', 'Computed Tomography Scan', 'CT Scan', 'Radiation Therapy', 'Dialysis', 'Gastric Bypass', 'Lap Band', +'Obesity', 'Laparoscopic Cholecystectomy', 'Lap Chole', 'Laparoscopic Hysterectomy', 'Laparoscopic Prostatectomy', 'Laparoscopic Hysteroscopy', +'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology'] diff --git a/src/test_notebook.ipynb b/src/test_notebook.ipynb deleted file mode 100644 index 9361dbd..0000000 --- a/src/test_notebook.ipynb +++ /dev/null @@ -1,359 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Imports\n", - "import pandas as pd\n", - "import re\n", - "import ast\n", - "import json\n", - "\n", - "import prompt_funcs\n", - "import config\n", - "import utils\n", - "import preprocess\n", - "import table_funcs\n", - "import claude_funcs\n", - "import prompts" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "input_dict = utils.read_input(path='../docs/test/')\n", - "input_dict.keys()\n", - "filename = '47-1928508 Advanced Anesthesia.txt'" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "contract_text = input_dict[filename]" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "#### PREPROCESS ####\n", - "contract_text = preprocess.clean_newlines(contract_text)\n", - "text_dict = preprocess.split_text(contract_text)\n", - "#text_dict = table_funcs.align_and_format_tables(text_dict)\n", - "#text_dict = preprocess.highlight_rates(text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "#### TOP DOWN ####\n", - "td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page\n", - "\n", - "#### BOTTOM UP ####\n", - "bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries\n", - "\n", - "#### COMBINE ####\n", - "# combined_results = prompt_funcs.td_bu_combine(td_results, bu_results, text_dict)\n", - "\n", - "# answer_df = pd.DataFrame(combined_results)" - ] - }, - { - "cell_type": "code", - "execution_count": 92, - "metadata": {}, - "outputs": [], - "source": [ - "def clean_td(td):\n", - " td_clean = []\n", - " for d in td:\n", - " new_d = {}\n", - " for k in d.keys():\n", - " if k not in ['page_num', 'Filename']:\n", - " if ',' in d[k]:\n", - " if 'DATE' not in k:\n", - " new_d[k] = [v.strip() for v in d[k].split(',')]\n", - " elif d[k] == 'N/A':\n", - " new_d[k] = []\n", - " else:\n", - " new_d[k] = [d[k]]\n", - " else:\n", - " new_d[k] = d[k]\n", - " td_clean.append(new_d)\n", - " return td_clean\n", - "\n", - "def get_unique_td(td):\n", - " unique_td_dict = {}\n", - " for d in td:\n", - " for key, value in d.items():\n", - " if key not in ['page_num', 'Filename']:\n", - " for v in value:\n", - " if key not in unique_td_dict.keys():\n", - " unique_td_dict[key] = [v]\n", - " else:\n", - " unique_td_dict[key].append(v)\n", - " final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()}\n", - " return final_dict\n", - "\n", - "def td_bu_combine(td, bu, text_dict):\n", - " bu_dicts = bu.copy()\n", - " td_dicts = td.copy()\n", - "\n", - " td_dicts_clean = clean_td(td_dicts)\n", - " unique_td = get_unique_td(td_dicts_clean)\n", - " \n", - " combined_list = []\n", - " for bud_ in bu_dicts:\n", - " bud = bud_.copy()\n", - " for key, value in unique_td.items():\n", - " if len(value) == 0: # No value in contract\n", - " bud.update({key : \"\"})\n", - " elif len(value) == 1: # 1 value in contract\n", - " bud.update({key : value[0]})\n", - " else: # More than one value in contract\n", - " # Check for value on the page\n", - " page_value = []\n", - " for d in td_dicts_clean:\n", - " if (d['page_num'] == bud['page_num']) and (key in d.keys()):\n", - " for v in d[key]:\n", - " page_value.append(v)\n", - " \n", - " if len(page_value) == 1: # One value on page\n", - " bud.update({key : page_value[0]})\n", - " elif len(page_value) > 1: # Multiple values on page\n", - " bud.update({key : ', '.join(page_value)})\n", - " elif len(page_value) == 0:\n", - " bud.update({key : \"\"}) \n", - " combined_list.append(bud)\n", - " return combined_list" - ] - }, - { - "cell_type": "code", - "execution_count": 94, - "metadata": {}, - "outputs": [], - "source": [ - "combined = td_bu_combine(td_results, bu_results, text_dict)" - ] - }, - { - "cell_type": "code", - "execution_count": 95, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Medicaid',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " 'Commercial',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '',\n", - " '']" - ] - }, - "execution_count": 95, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "[c['CONTRACT_LOB'] for c in combined]" - ] - }, - { - "cell_type": "code", - "execution_count": 98, - "metadata": {}, - "outputs": [], - "source": [ - "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From f2e3423d6163e0cd39e6267ce1e0d5f73b67d3a8 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 3 Jun 2024 14:53:40 -0700 Subject: [PATCH 31/78] TD/BU integration changes --- src/config.py | 9 ++- src/file_processing.py | 134 ++++++++++++++++++++++++++++++++++ src/integration_testing.ipynb | 58 +++++++-------- src/main.py | 10 +-- src/prompt_funcs.py | 102 +------------------------- src/prompts.py | 32 ++++++++ 6 files changed, 207 insertions(+), 138 deletions(-) create mode 100644 src/file_processing.py diff --git a/src/config.py b/src/config.py index 418548a..1885743 100644 --- a/src/config.py +++ b/src/config.py @@ -13,6 +13,7 @@ TODAY = datetime.now().strftime("%Y%m%d") # I/O Options WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written +OUTPUT_FOLDER = 'output' READ_MODE = '_LOCAL_' # OR '_S3_' OUTPUT_MODE = '_INDIVIDUAL_' # or '_CONSOLIDATED_' @@ -29,12 +30,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFULN3ZDJ" -AWS_SECRET_ACCESS_KEY="biHj+LwFkcquA7hoOgWst66u7AAQcik+A8TnMSoO" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELr//////////wEaCXVzLWVhc3QtMiJHMEUCIEIZfQMj70zYiV1czvy0RdOdugmJ9YGZIPBk/wqUE8m1AiEAzA9jGoZfv0ZMZlBKTLHGT3Ax3rAQNVLYWqgo9jy3Ui8qiQMIQxAAGgw2NjAxMzEwNjg3ODIiDOYEfYdB5ykwPYhKOSrmAiHPCbfUR2lXoqyEvURpTFyOqLEmeVFHCHQoVyeFcXwQmjjZor+ui1dclq0asj35ZSScClpyQKZsfk5o5+76pxzTm1tlBez1buIFkel/9OngHNG4MvMwLCMLdI+ILtJy0ufLmky4oVgtSor5n2e5dLENqstqhXEREbGdxZ5gaF7zKg9Dk/VPQ8arzV0oEvxuPGbZ+Z640jtVR27oSU1pQQ4ayK+CM/jBMANlzO9BptRB+Phfm6oOfxYQskyIXRV+aff/3d/MVJs6X08U/YjN+qXx0pDLsyBxYQIpKZXIZvWrYRD/NM7DnPJF8DlTAWhNSDU0QJ4j2oTMKcaG0LHXmd1SejjgHXhK0ZiG2tcpNHPaIBFm/lWnaixOdxM9XUWckoTe+1FRmcblQ7YI2HtWsbgzU3B8BLmLvEPX+lJSDeJBuv1MGFA4ITa4YaRywBBdZTcSl9cwLGB1X9ksTga5cd1XZeW/218w5cfmsgY6pgEsq/5MsjzbPqT20XDAhlC7GUmY3K+glAHfm8nVnihVdfmfU2apbx0bcDu5Er9mngykbj0u24P8VbkmlozS2H2dDmCjqQeE2Ux2pOuUxfqtMlJVSjAO8MCVvjc2nxsv55iHDTHGSlaN8TmsoBqVRbsOmauWFPQ7gFJDqF7YADS+20NDsgblAeeE23eZU4A/mG1dWfIK5E58nAoKWvnumFyQnJ0sMPpN" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMFZB42PP" +AWS_SECRET_ACCESS_KEY="Q70JV6ph4cALcn3jM0JM55KhrRO7vZcClt7fDZab" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEAsaCXVzLWVhc3QtMiJHMEUCIQC4ajBdrraawj7K5uNmeJdwRKQBuy3uWN3mDirDnyilkQIgI7L85Oz5K5VG+N6XoVBa/dGH5cUUIpXUVWl4MDmgou8qjAMIlP//////////ARAAGgw2NjAxMzEwNjg3ODIiDLf4pcE0sfOJvHbZwirgAqehe2Qhyersv7D0rehPy+Abx+lYnIfmViM/mHL2FIbAp6rht8Af3R08/SiRywF+wtVqeWERKrpoiW8DmYkDLxCosPdLskFfrjC0RNiLKLyrZDksTBaSwcJIWA2r4pu1M6tI+QprJrb7e3nF9Z4V3l/gRj9owL9CVlS0kvFR5JwqbJIHgydGNWaTuN+XhgkxMyvZqQcdRD6lEdqlezFkVNUhV0fRI+dhOoOwR/zXjz+L1xIkDx7/FzYq8J0HhsLqn1HUhVj7/X9kJi+Fx6qdWOiePLtAJlJPLcsuTfehFr4FAp3Pk2hf6AlzuJfemGnlOaSJS8pKh6bcPtIoRqPwc2eo3h8D2cl1mo67+NZsTkmlPynuYiSZOQUxu6PEjVml8jIW4xkvg+GGMAkG+jFn22PMgLHhLnMjL2JC9Z+BfyiyIwsOwta+hic0rxHLS9DT2FG6V75+/I3nHSqcgq2Xx3Mw76D4sgY6pgFpxpII4v+R5eX2lGt4tG0V+PU3OWS1GKEmDvLL5Ahnsnfw/iLnUXvcqSOM+6Y9T3vG+gX9ELq9sPFe+11PFezen6LuUJlJdY0t2mq7TNh2/RflL6Khb6gAfSFGCgauAdKfj4UOApd8wBuJ6WE4iJ4ep7XviqHQnI62dfulGL7qHZv6MDTxoinWUlGtSUOwmeR/XawOSyvYSB012MDJn3+iLshLA81x" # File Paths -LOCAL_PATH = 'subset/' # Replace with local +LOCAL_PATH = 'data/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py new file mode 100644 index 0000000..1507097 --- /dev/null +++ b/src/file_processing.py @@ -0,0 +1,134 @@ + +import os +import pandas as pd + +import config +import preprocess +import table_funcs +import prompt_funcs +import prompts +import claude_funcs + +def clean_td(td): + td_clean = [] + for d in td: + new_d = {} + for k, v in d.items(): + if 'DATE' in k: + new_d[k] = v if isinstance(v, list) else [v] + elif k not in ['page_num', 'Filename']: + if isinstance(v, str) and ',' in v: + new_d[k] = [item.strip() for item in v.split(',')] + elif v == 'N/A': + new_d[k] = [] + else: + new_d[k] = [v] if isinstance(v, str) else v + else: + new_d[k] = v + td_clean.append(new_d) + return td_clean + + +def get_unique_keys(dicts): + keys = set() + for d in dicts: + keys.update(d.keys()) + return keys + +def get_unique_date_fields(td_results): + unique_date_fields = {} + for td in td_results: + for key, value in td.items(): + if 'DATE' in key: + if key not in unique_date_fields: + unique_date_fields[key] = set() + unique_date_fields[key].update(value if isinstance(value, list) else [value]) + for key in unique_date_fields: + unique_date_fields[key] = list(unique_date_fields[key]) + return unique_date_fields + +def merge_results(td_results, bu_results, text_dict): + all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) + print("All Keys:", all_keys) + + date_fields = get_unique_date_fields(td_results) + print("Date Fields:", date_fields) + + merged_results = [] + for bu in bu_results: + merged = {key: bu.get(key, "") for key in all_keys} + td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']] + + for td in td_on_page: + for key, value in td.items(): + if not merged[key]: + merged[key] = value + elif isinstance(value, list) and value and not isinstance(merged[key], list): + merged[key] = value + elif isinstance(value, list) and value: + merged[key].extend(value) + + for date_key, date_values in date_fields.items(): + if date_key not in merged or not merged[date_key]: + merged[date_key] = date_values + else: + merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) + + for key in all_keys: + if isinstance(merged[key], list) and len(merged[key]) > 1: + page_num = bu['page_num'] + page_text = text_dict.get(page_num, "") + prompt = prompts.GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key]) + response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + try: + selected_value = response.strip() + merged[key] = selected_value + except Exception as e: + print(f"Error processing LLM response for key {key}: {e}") + merged[key] = ', '.join(merged[key]) + + merged_results.append(merged) + + return merged_results + + +def process_file(file_object): + filename, contract_text = file_object + if config.VERBOSE: print(f"Processing {filename}...") + + #### PREPROCESS #### + contract_text = preprocess.clean_newlines(contract_text) + text_dict = preprocess.split_text(contract_text) + text_dict = table_funcs.align_and_format_tables(text_dict) + text_dict = preprocess.highlight_rates(text_dict) + + # Run Top Down + td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page + print("TD Results:", td_results) + + # Run Bottom Up + bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries + print("BU Results:", bu_results) + + # Combine + combined_results = merge_results(clean_td(td_results), bu_results, text_dict) + print("Combined Results:", combined_results) + + # Convert to DataFrame + combined_df = pd.DataFrame(combined_results) + + # Post-process combined results + # post_processed_combined_df = postprocess.postprocess_results(combined_df) + # print("Post-processed Results:", post_processed_combined_df) + + # Create directories + base_filename = os.path.splitext(filename)[0] + output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) + os.makedirs(output_dir, exist_ok=True) + + # Save results + pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) + pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) + combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) + # post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) + diff --git a/src/integration_testing.ipynb b/src/integration_testing.ipynb index e6dce29..8df98fe 100644 --- a/src/integration_testing.ipynb +++ b/src/integration_testing.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -124,8 +124,6 @@ "import os\n", "import pandas as pd\n", "\n", - "\n", - "\n", "def clean_td(td):\n", " td_clean = []\n", " for d in td:\n", @@ -336,9 +334,28 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TD Results: [{'CONTRACT_LOB': 'Commercial, Medicare', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'HMO, PPO, EPO', 'PRODUCT': 'Medicare Select', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007'}, {'CONTRACT_LOB': 'Medicare', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Health Net', 'page_num': '2', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': '11/20/07', 'LOB_PRICING_TERMS_TERMINATION_DATE': '11/20/07'}]\n", + "BU Results: [{'SERVICE': 'Commercial HMO Members Contracted Services', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '241%', 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': '(b) one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_RATE': 100, 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '1', 'SERVICE': 'Commercial HMO Members Contracted Services', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'SERVICE': 'Commercial HMO Members Contracted Services', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '254%', 'FULL_METHODOLOGY': 'For Commercial HMO Members, effective December 1, 2008 Physician Group shall be compensated for Contracted Services, less applicable Copayments, in an amount equal to the lesser of: (a) two hundred and fifty-four percent (254%) of the Medicare allowable charges based on the Medicare Resource based Relative Value Scalo (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': 'one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_RATE': '100', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '1', 'SERVICE': 'Commercial HMO Members Contracted Services', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'SERVICE': 'Medicare HMO and Medicare Select products Contracted Services', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '100%', 'FULL_METHODOLOGY': '(a) one hundred percent (100%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register;', 'REIMBURSEMENT_RATE': '241', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '1', 'SERVICE': 'Medicare HMO and Medicare Select products Contracted Services', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '241%', 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register; or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'RBRVS', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': '(b) one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_RATE': 100, 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '1', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '254%', 'FULL_METHODOLOGY': 'Effective December 1, 2008 Physician Group shall be compensated for Covered Services in an amount, less applicable Copayments and/or coinsurance, that is equal to the lesser of: (a) two hundred and fifty-four percent (254%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register; or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'page_num': '1', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': 'one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_RATE': '100', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '1', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'SERVICE': 'Medicare Resource Based Relative Value Scale (RBRVS)', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'In the event that potential CMS changes to Medicare Resource Based Relative Value Scale (RBRVS) result in an aggregate 3% change, either positively or negatively, either party may initiate a meet and confer process which will determine impact and/or potential changes to reimbursement.', 'page_num': '2', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare Resource Based Relative Value Scale (RBRVS)', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'Y', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'In the event that potential CMS changes to Medicare Resource Based Relative Value Scale (RBRVS) result in an aggregate 3% change, either positively or negatively, either party may initiate a meet and confer process which will determine impact and/or potential changes to reimbursement.', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}]\n", + "All Keys: {'REIMBURSEMENT_STATUS_INDICATOR_CODES', 'CONTRACT_NETWORK', 'REIMBURSEMENT_GROUPER_CODES', 'GREATER_OF_LANGUAGE_IND', 'LESSER_OF_LANGUAGE_IND', 'REIMBURSEMENT_EXCEPTION_IND', 'SERVICE', 'LOB_PRICING_TERMS_TERMINATION_DATE', 'REIMBURSEMENT_REVENUE_CODES', 'PRODUCT', 'CONTRACT_LOB', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', 'REIMBURSEMENT_PROC_CODES', 'RATE_ESCALATOR_IND', 'REIMBURSEMENT_RATE', 'REIMBURSEMENT_METHODOLOGY', 'REIMBURSEMENT_FLAT_FEE', 'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_ADMITTYPE_CODES', 'CONTRACT_PROGRAM', 'FULL_METHODOLOGY', 'REIMBURSEMENT_DESCRIBE_EXCEPTION', 'page_num', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION', 'REIMBURSEMENT_FEE_SCHEDULE', 'CONTRACT_MARKETPLACE_METAL_LEVEL', 'LOB_PRICING_TERMS_EFFECTIVE_DATE', 'Filename'}\n", + "Date Fields: {'LOB_PRICING_TERMS_EFFECTIVE_DATE': ['11/20/07', 'December 1, 2007'], 'LOB_PRICING_TERMS_TERMINATION_DATE': ['11/20/07', 'December 1, 2007']}\n", + "Combined Results: [{'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': '', 'LESSER_OF_LANGUAGE_IND': '', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Commercial HMO Members Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': '241%', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'Y', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Commercial HMO Members Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': 100, 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(b) one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': '', 'LESSER_OF_LANGUAGE_IND': '', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Commercial HMO Members Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': '254%', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For Commercial HMO Members, effective December 1, 2008 Physician Group shall be compensated for Contracted Services, less applicable Copayments, in an amount equal to the lesser of: (a) two hundred and fifty-four percent (254%) of the Medicare allowable charges based on the Medicare Resource based Relative Value Scalo (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'Y', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Commercial HMO Members Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': '100', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': '', 'LESSER_OF_LANGUAGE_IND': '', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Medicare HMO and Medicare Select products Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': '100%', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(a) one hundred percent (100%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register: or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'Y', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Medicare HMO and Medicare Select products Contracted Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': '241', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register;', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': '', 'LESSER_OF_LANGUAGE_IND': '', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': '241%', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(a) two hundred and forty-one percent (241%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register; or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'RBRVS', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'Y', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': 100, 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': '(b) one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': '', 'LESSER_OF_LANGUAGE_IND': '', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'REIMBURSEMENT_RATE': '254%', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule, Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'Effective December 1, 2008 Physician Group shall be compensated for Covered Services in an amount, less applicable Copayments and/or coinsurance, that is equal to the lesser of: (a) two hundred and fifty-four percent (254%) of the Medicare allowable charges based on the Medicare Resource Based Relative Value Scale (RBRVS) unit values and CMS Geographical Practice Cost Indices as published in the most current published edition of the Federal Register; or (b) one hundred percent (100%) of Physician Group allowable billed charges.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'Most current published edition', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare RBRVS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'Y', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Preferred Provider Organization (PPO) Exclusive Provider Organization (EPO) Benefit Programs Covered Services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'December 1, 2007', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Medicare Select'], 'CONTRACT_LOB': 'Commercial', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': '100', 'REIMBURSEMENT_METHODOLOGY': 'Billed Charges', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'one hundred percent (100%) of Physician Group allowable billed charges', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '1', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'December 1, 2007', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'Y', 'SERVICE': 'Medicare Resource Based Relative Value Scale (RBRVS)', 'LOB_PRICING_TERMS_TERMINATION_DATE': '11/20/07', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': ['Health Net'], 'CONTRACT_LOB': ['Medicare'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'In the event that potential CMS changes to Medicare Resource Based Relative Value Scale (RBRVS) result in an aggregate 3% change, either positively or negatively, either party may initiate a meet and confer process which will determine impact and/or potential changes to reimbursement.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'In the event that potential CMS changes to Medicare Resource Based Relative Value Scale (RBRVS) result in an aggregate 3% change, either positively or negatively, either party may initiate a meet and confer process which will determine impact and/or potential changes to reimbursement.', 'page_num': '2', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicare Resource Based Relative Value Scale (RBRVS)', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': '11/20/07', 'Filename': '2007-12-01 California Emergency Physicians AMD_MU.txt'}]\n", + "Processed 2007-12-01 California Emergency Physicians AMD_MU.txt\n", + "TD Results: [{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'moda IMAGING SERVICES', 'page_num': '1', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, {'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '2', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, {'CONTRACT_LOB': 'Medicare Advantage', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '3', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013'}, {'CONTRACT_LOB': 'Medicare Advantage', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '4', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013'}, {'CONTRACT_LOB': 'Medicare, Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'ODS Plus Network, ODS PPO Advantage', 'PRODUCT': 'Oregon Community Health, Inc., ODS Health Plan, Inc.', 'page_num': '5', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}]\n", + "BU Results: [{'SERVICE': 'Rural Health Clinic (RHC) services', 'REIMBURSEMENT_FLAT_FEE': '79.17', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a UB-04 (including any future editions) will be reimbursed at the RHC per visit rate of $79.17.', 'page_num': '3', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Per Visit', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732', 'G0008', '90655-90663'], 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'SERVICE': 'Pneumococcal and influenza vaccinations', 'REIMBURSEMENT_FLAT_FEE': '87.83', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83', 'page_num': '3', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'], 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'SERVICE': 'Vaccinations', 'REIMBURSEMENT_FLAT_FEE': '47.09', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0008, 90655-90663 $47.09', 'page_num': '3', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': ['G0008', '90655-90663'], 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'SERVICE': 'Rural Health Clinic services billed on UB-04', 'REIMBURSEMENT_FLAT_FEE': '79.17', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a UB-04 (including any future editions) will be reimbursed at the RHC per visit rate of $79.17.', 'page_num': '4', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Per Visit', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'G0009, 90669, 90670, 90732, G0008, 90655-90663', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'SERVICE': 'Pneumococcal and influenza vaccinations billed on CMS 1500', 'REIMBURSEMENT_FLAT_FEE': '87.83', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83', 'page_num': '4', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': ['G0009', '90669', '90670', '90732'], 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, {'SERVICE': 'Other vaccinations billed on CMS 1500', 'REIMBURSEMENT_FLAT_FEE': '47.09', 'REIMBURSEMENT_RATE': 'N/A', 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0008, 90655-90663 $47.09', 'page_num': '4', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': ['G0008', '90655-90663'], 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}]\n", + "All Keys: {'REIMBURSEMENT_STATUS_INDICATOR_CODES', 'CONTRACT_NETWORK', 'REIMBURSEMENT_GROUPER_CODES', 'GREATER_OF_LANGUAGE_IND', 'LESSER_OF_LANGUAGE_IND', 'REIMBURSEMENT_EXCEPTION_IND', 'SERVICE', 'LOB_PRICING_TERMS_TERMINATION_DATE', 'REIMBURSEMENT_REVENUE_CODES', 'PRODUCT', 'CONTRACT_LOB', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', 'REIMBURSEMENT_PROC_CODES', 'RATE_ESCALATOR_IND', 'REIMBURSEMENT_RATE', 'REIMBURSEMENT_METHODOLOGY', 'REIMBURSEMENT_FLAT_FEE', 'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_ADMITTYPE_CODES', 'CONTRACT_PROGRAM', 'FULL_METHODOLOGY', 'REIMBURSEMENT_DESCRIBE_EXCEPTION', 'page_num', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION', 'REIMBURSEMENT_FEE_SCHEDULE', 'CONTRACT_MARKETPLACE_METAL_LEVEL', 'LOB_PRICING_TERMS_EFFECTIVE_DATE', 'Filename'}\n", + "Date Fields: {'LOB_PRICING_TERMS_EFFECTIVE_DATE': ['N/A', 'January 1, 2013'], 'LOB_PRICING_TERMS_TERMINATION_DATE': ['N/A', 'January 1, 2013']}\n", + "Combined Results: [{'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Rural Health Clinic (RHC) services', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'G0009, 90669, 90670, 90732', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Per Visit', 'REIMBURSEMENT_FLAT_FEE': '79.17', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a UB-04 (including any future editions) will be reimbursed at the RHC per visit rate of $79.17.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '3', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Pneumococcal and influenza vaccinations', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': '90732', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': '87.83', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '3', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Vaccinations', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': '90655-90663', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': '47.09', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0008, 90655-90663 $47.09', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '3', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Rural Health Clinic services billed on UB-04', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'G0009, 90669, 90670, 90732, G0008, 90655-90663', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Per Visit', 'REIMBURSEMENT_FLAT_FEE': '79.17', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a UB-04 (including any future editions) will be reimbursed at the RHC per visit rate of $79.17.', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '4', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Pneumococcal and influenza vaccinations billed on CMS 1500', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': '90732', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': '87.83', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0009, 90669, 90670, 90732 $87.83', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '4', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}, {'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'CONTRACT_NETWORK': [], 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'GREATER_OF_LANGUAGE_IND': 'N', 'LESSER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'SERVICE': 'Other vaccinations billed on CMS 1500', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'January 1, 2013', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'PRODUCT': [], 'CONTRACT_LOB': ['Medicare Advantage'], 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': '90655-90663', 'RATE_ESCALATOR_IND': 'N', 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_METHODOLOGY': 'Medicare Fee Schedule', 'REIMBURSEMENT_FLAT_FEE': '47.09', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'CONTRACT_PROGRAM': [], 'FULL_METHODOLOGY': 'For services billed on a CMS 1500 (including any future editions) for pneumococcal and influenza vaccinations will be allowed as follows: G0008, 90655-90663 $47.09', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'page_num': '4', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'CMS', 'CONTRACT_MARKETPLACE_METAL_LEVEL': [], 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'January 1, 2013', 'Filename': '2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt'}]\n", + "Processed 2013-01-01 Amendment MA FE PeaceHealth TIN 931251530 & 800537724.txt\n" + ] + } + ], "source": [ "import os\n", "import pandas as pd\n", @@ -479,8 +496,8 @@ " combined_df = pd.DataFrame(combined_results)\n", " \n", " # Post-process combined results\n", - " post_processed_combined_df = postprocess.postprocess_results(combined_df)\n", - " print(\"Post-processed Results:\", post_processed_combined_df)\n", + " # post_processed_combined_df = postprocess.postprocess_results(combined_df)\n", + " # print(\"Post-processed Results:\", post_processed_combined_df)\n", " \n", " # Create directories\n", " base_filename = os.path.splitext(filename)[0]\n", @@ -490,7 +507,8 @@ " # Save results\n", " pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False)\n", " pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False)\n", - " post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + " combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", + " # post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False)\n", "\n", "def process_all_files(input_dict):\n", " for filename in input_dict.keys():\n", @@ -498,7 +516,7 @@ " print(f\"Processed {filename}\")\n", "\n", "# Example usage\n", - "input_dict = utils.read_input(path='subset')\n", + "input_dict = utils.read_input(path='../data/')\n", "process_all_files(input_dict)\n", "\n" ] @@ -509,19 +527,8 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "\n", "# Usage example:\n", - "#save_combined_to_csv(combined, 'combined_output.csv')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "[c['CONTRACT_LOB'] for c in combined]" + "#save_combined_to_csv(combined, 'combined_output.csv')" ] }, { @@ -532,13 +539,6 @@ "source": [ "utils.consolidate_individual(input_folder='../results/', output_folder='../output/')" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -557,7 +557,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.3" } }, "nbformat": 4, diff --git a/src/main.py b/src/main.py index e9c1f79..b183f03 100644 --- a/src/main.py +++ b/src/main.py @@ -8,6 +8,8 @@ import utils import config import prompt_funcs import claude_funcs +import file_processing + def main(): if config.TEST: @@ -15,8 +17,8 @@ def main(): else: # Read contract txt input_dict = utils.read_input() - already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('results/')] - input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} + # already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('results/')] + # input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} # Set up results folder if not os.path.exists('results'): @@ -33,7 +35,7 @@ def main(): def process_item(item): key, value = item try: - prompt_funcs.run_prompts(item) + file_processing.process_file(item) #prompt_funcs.bottom_up(item) except Exception as e: # Print the error message and traceback @@ -61,5 +63,3 @@ def main(): if __name__ == "__main__": main() - - diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 8811716..effb73d 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -124,7 +124,6 @@ def run_top_down_date(type_, d, page): answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) return answer - def top_down_secondary(td_results, text_dict): updated_dicts = [] for d in td_results: @@ -133,6 +132,8 @@ def top_down_secondary(td_results, text_dict): # Dates d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) + + # if auto_renewal == N: else: 'N/A' d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) updated_dicts.append(d) @@ -161,104 +162,5 @@ def run_top_down(filename, text_dict): return td_final # List of list of dictionaries -# Takes list of dictionaries, updates vals in dictionaries to be lists instead of strings. N/A->empty list, single val -> [single_val], multiple val -> [val1, val2] -def clean_td(td): - td_clean = [] - for d in td: - new_d = {} - for k in d.keys(): - if k not in ['page_num', 'Filename']: - if ',' in d[k]: - if 'DATE' not in k: - new_d[k] = [v.strip() for v in d[k].split(',')] - elif d[k] == 'N/A': - new_d[k] = [] - else: - new_d[k] = [d[k]] - else: - new_d[k] = d[k] - td_clean.append(new_d) - return td_clean # List of dictionaries where values in each dict are lists - -# Takes list of dictionaries, returns one dictionary with unique values -def get_unique_td(td): - unique_td_dict = {} - for d in td: - for key, value in d.items(): - if key not in ['page_num', 'Filename']: - for v in value: - if key not in unique_td_dict.keys(): - unique_td_dict[key] = [v] - else: - unique_td_dict[key].append(v) - final_dict = {key : list(set(unique_td_dict[key])) for key in unique_td_dict.keys()} - return final_dict - -def td_bu_combine(td, bu, text_dict): - bu_dicts = bu.copy() - td_dicts = td.copy() - - td_dicts_clean = clean_td(td_dicts) - unique_td = get_unique_td(td_dicts_clean) # Unique values in entire contract - - combined_list = [] - for bud_ in bu_dicts: - bud = bud_.copy() - for key, value in unique_td.items(): # value is list of unique values - if len(value) == 0: # No value in contract - bud.update({key : ""}) - elif len(value) == 1: # 1 value in contract - bud.update({key : value[0]}) - else: # More than one value in contract - # Check for value on the page - page_value = [] # page_value = list of all values of key on the same page as bud - for d in td_dicts_clean: - if (d['page_num'] == bud['page_num']) and (key in d.keys()): - for v in d[key]: - page_value.append(v) - - if len(page_value) == 1: # One value on page - bud.update({key : page_value[0]}) - elif len(page_value) > 1: # Multiple values on page - bud.update({key : ', '.join(page_value)}) # Faizan - Change line to prompt check (pass in page_value) (commit ce3f4c17f281c4c3851d9967c51a14468ee06230) - elif len(page_value) == 0: - bud.update({key : ""}) # Faizan - Change line to prompt check previous page (pass in page_value) - (ce3f4c17f281c4c3851d9967c51a14468ee06230) - combined_list.append(bud) - return combined_list # List of dictionaries - -def run_prompts(file_object): - filename, contract_text = file_object - if config.VERBOSE: print(f"Processing {filename}...") - - #### PREPROCESS #### - contract_text = preprocess.clean_newlines(contract_text) - text_dict = preprocess.split_text(contract_text) - text_dict = table_funcs.align_and_format_tables(text_dict) - text_dict = preprocess.highlight_rates(text_dict) - - #### TOP DOWN #### - td_results = run_top_down(filename, text_dict) # Returns list of list of dictionaries for each page - - #### BOTTOM UP #### - bu_results = run_bottom_up(filename, text_dict) # Returns list of dictionaries - - #### COMBINE #### - combined_results = td_bu_combine(td_results, bu_results, text_dict) - - answer_df = pd.DataFrame(combined_results) - - ### OUTPUT #### - if config.WRITE_OUTPUT: - if config.OUTPUT_MODE == '_INDIVIDUAL_': - answer_df.to_csv(f"results/{filename}_results.csv") - elif config.OUTPUT_MODE == '_CONSOLIDATED_': - if not os.path.exists('temp'): - os.makedirs('temp') - answer_df.to_csv(f"temp/{filename}_results.csv") - else: - if config.VERBOSE: print(answer_df) - - # Postprocess (added later) - diff --git a/src/prompts.py b/src/prompts.py index 9e8fa06..86fffde 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -1,5 +1,37 @@ +def GENERATE_PROMPT(bu_dict, td_dicts, page, field_name=None, values=None): + if field_name and values: + return f"""### PAGE START ### {page} ### PAGE END + +Field: {field_name} +Values: {values} + +Listed above is a page of a contract, as well as the potential values for the {field_name} field. +Your job is to identify which of these values is the most appropriate based on the page content. + +Write ONLY the value that is most appropriate. Do not return any additional text beyond the value itself. +""" + else: + return f"""### PAGE START ### {page} ### PAGE END + +Service: {bu_dict['SERVICE']} +Methodology: {bu_dict['FULL_METHODOLOGY']} + +Listed above is a page of a contract, as well as a Service and Methodology found on this page. +Your job is to identify which of the following entities this specific Service is associated with. + +----Change Below---- + +Here are the entities, represented as dictionary objects: + +{td_dicts} + +Write ONLY the number of the dictionary that is most associated with the Service and Methodology of interest. Do not return any additional text beyond the digit itself. +""" + + + def BOTTOM_UP_PRIMARY(page, payer): return f""" ### PAGE START ### {page} ### PAGE END From ba1e0ca2cb44c9c7c0d9904c22602e13e8360ee1 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 5 Jun 2024 07:31:33 -0700 Subject: [PATCH 32/78] Modify table preprocessing --- src/config.py | 13 +++++++++---- src/main.py | 8 -------- src/postprocess.py | 2 +- src/preprocess.py | 2 +- src/prompt_funcs.py | 1 - src/table_funcs.py | 5 +++-- 6 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/config.py b/src/config.py index 1885743..c4c65c6 100644 --- a/src/config.py +++ b/src/config.py @@ -11,6 +11,11 @@ VERBOSE = True CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") +# Valid values +VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] +VALID_PROGRAMS = [] +VALID_LOB = [] + # I/O Options WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written OUTPUT_FOLDER = 'output' @@ -30,12 +35,12 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMFZB42PP" -AWS_SECRET_ACCESS_KEY="Q70JV6ph4cALcn3jM0JM55KhrRO7vZcClt7fDZab" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEAsaCXVzLWVhc3QtMiJHMEUCIQC4ajBdrraawj7K5uNmeJdwRKQBuy3uWN3mDirDnyilkQIgI7L85Oz5K5VG+N6XoVBa/dGH5cUUIpXUVWl4MDmgou8qjAMIlP//////////ARAAGgw2NjAxMzEwNjg3ODIiDLf4pcE0sfOJvHbZwirgAqehe2Qhyersv7D0rehPy+Abx+lYnIfmViM/mHL2FIbAp6rht8Af3R08/SiRywF+wtVqeWERKrpoiW8DmYkDLxCosPdLskFfrjC0RNiLKLyrZDksTBaSwcJIWA2r4pu1M6tI+QprJrb7e3nF9Z4V3l/gRj9owL9CVlS0kvFR5JwqbJIHgydGNWaTuN+XhgkxMyvZqQcdRD6lEdqlezFkVNUhV0fRI+dhOoOwR/zXjz+L1xIkDx7/FzYq8J0HhsLqn1HUhVj7/X9kJi+Fx6qdWOiePLtAJlJPLcsuTfehFr4FAp3Pk2hf6AlzuJfemGnlOaSJS8pKh6bcPtIoRqPwc2eo3h8D2cl1mo67+NZsTkmlPynuYiSZOQUxu6PEjVml8jIW4xkvg+GGMAkG+jFn22PMgLHhLnMjL2JC9Z+BfyiyIwsOwta+hic0rxHLS9DT2FG6V75+/I3nHSqcgq2Xx3Mw76D4sgY6pgFpxpII4v+R5eX2lGt4tG0V+PU3OWS1GKEmDvLL5Ahnsnfw/iLnUXvcqSOM+6Y9T3vG+gX9ELq9sPFe+11PFezen6LuUJlJdY0t2mq7TNh2/RflL6Khb6gAfSFGCgauAdKfj4UOApd8wBuJ6WE4iJ4ep7XviqHQnI62dfulGL7qHZv6MDTxoinWUlGtSUOwmeR/XawOSyvYSB012MDJn3+iLshLA81x" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFTD5MQUK" +AWS_SECRET_ACCESS_KEY="yGCDnqRupWsJyABleBXd3NkzySjf2tuU1L5pPsUh" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJIMEYCIQC79KaOSV9wiHBrCPKOdoGQ10Hhl2BlX9kN2LI0AWtbzQIhAOniVNrVlB0wbuaJUVcFjuV8LRO/ZZdCk4ac9O9HftrdKowDCL7//////////wEQABoMNjYwMTMxMDY4NzgyIgz6LOHKqD7pmvQXBf8q4AJkRxmxk5UhNmr6iAx+zsccBvcFR2XBFywrgrPCHk5mUzQXfgPFBmcTW7XTB2F2Rie5y/oZGV1YxK8s2JYXd9gkbJvwUt7K1bNjQYg/IMQ1goecxH2dRHq//JYDs9TQiU01nacnI6eMPQqSCpmtc2yI45GovWwONKJviddUI5Ln/IS7Ktq4dcmy7xx/GqbgqsWb3bCFhmT/9UrOGaBSOHA0fhshKaw8ihbWgz1BtVFSt2i8FsTXRlNXn5l79iwo+vWs4CQzmHgRjw3ZIytrw2V2nppgJlw1Tb+AEwBdbC9wW9WsmDmw9QmBeyxcKAKkmPf66wWqmBqK0/2w8r4q6H7gEta/Ibc08137zd0fLWEu74iZE0I5jhLYOGwh+U0DhsR0ZDufWurcA+YIfWZAsfG0PePonv8+6drcqtNjTqKs0KIfAQ6PHAb4xuoCYEzvLeyMcY1t5sOl/oU+RWxkPrsIMMLLgbMGOqUBfx6TS5599bddNIEV2CH3AHym/18n3lyA19PHcVcfldqU7nMrYTZdgNBMy1Or1OVjcxYV988OI8ZOqWPiYzMGR6cVSzwA1ESjLUVma3vKcOTq4GbgBvRxA44PMfBm/P3ow3Vhx3ONe/oaxqgKre26JBheTN5s0Uua4lEcO4lvgB6/Kap+/aaR9vnEwLtJZ8u5d84k/Ajw1nG8bDMfLPsyWXjtUUZI" # File Paths -LOCAL_PATH = 'data/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/main.py b/src/main.py index b183f03..d6ccb75 100644 --- a/src/main.py +++ b/src/main.py @@ -23,14 +23,6 @@ def main(): # Set up results folder if not os.path.exists('results'): os.makedirs('results') - - # Extract all tables as dfs - #tables_dict = table_funcs.extract_all_tables(input_dict) - #print(tables_dict) - - # # Run bottom up primary with multithreading - # with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: - # executor.map(prompt_funcs.bottom_up, input_dict.items()) def process_item(item): key, value = item diff --git a/src/postprocess.py b/src/postprocess.py index 29d4149..ee807f1 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -7,7 +7,7 @@ def sanitize_combined(df): df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df -def post_process_combined(df): +def post_process_combined(df, ): VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] diff --git a/src/preprocess.py b/src/preprocess.py index 90c33f3..7accf76 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -47,5 +47,5 @@ def clean_billed_charges(text): if '%' in pre_text: return match.group() else: - return '100% of billed charges' + return r'100% of billed charges' return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index effb73d..e6f3475 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -109,7 +109,6 @@ def run_bottom_up(filename, text_dict): d['Filename'] = filename return results_dicts # List of dictionaries - def run_top_down_metal_level(d, page): if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper(): prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page) diff --git a/src/table_funcs.py b/src/table_funcs.py index 86b5db6..1c7af74 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -26,8 +26,9 @@ def format_table(table_json): for i in range(len(table_json[list(table_json.keys())[0]])): for key in table_json.keys(): if table_json[key][i]: - table_text += key + ': ' + table_json[key][i] + ', ' - table_text += '\n' + #table_text += key + ': ' + table_json[key][i] + ', ' + table_text += table_json[key][i] + ' : ' + table_text += r'\n' return table_text def align_and_format_tables(text_dict): From 38d18f97c51fa988b90f4be4a62c9e2b661df281 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 5 Jun 2024 09:52:38 -0700 Subject: [PATCH 33/78] Add Flat Fee as Reimbursement Methodology --- src/prompt_funcs.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index e6f3475..f6b4890 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -64,12 +64,15 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): for d in temp_dicts: if d is not None: page_num = d['page_num'] - + # Bottom Up Methodology if config.RUN_METHODOLOGY: - prompt = prompts.BOTTOM_UP_METHODOLOGY(d) - methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) - d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + if d['REIMBURSEMENT_FLAT_FEE'] == 'N/A': + prompt = prompts.BOTTOM_UP_METHODOLOGY(d) + methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + else: + d['REIMBURSEMENT_METHODOLOGY'] = 'Flat Fee' # # Bottom Up FS if config.RUN_FS: From a5fe33730951accc488b4d71d092c5c8fd87544d Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 5 Jun 2024 15:00:25 -0700 Subject: [PATCH 34/78] Merged postprocessing updates --- src/config.py | 6 ++--- src/file_processing.py | 17 ++++++------ src/postprocess.py | 55 ++++++++++++++++++++++++++++++++++++-- src/postprocessingfuncs.py | 55 ++++++++++++++++++++++---------------- src/prompts.py | 2 ++ src/test.py | 23 +++++++++++++--- 6 files changed, 119 insertions(+), 39 deletions(-) diff --git a/src/config.py b/src/config.py index c4c65c6..9ba5975 100644 --- a/src/config.py +++ b/src/config.py @@ -35,9 +35,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFTD5MQUK" -AWS_SECRET_ACCESS_KEY="yGCDnqRupWsJyABleBXd3NkzySjf2tuU1L5pPsUh" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJIMEYCIQC79KaOSV9wiHBrCPKOdoGQ10Hhl2BlX9kN2LI0AWtbzQIhAOniVNrVlB0wbuaJUVcFjuV8LRO/ZZdCk4ac9O9HftrdKowDCL7//////////wEQABoMNjYwMTMxMDY4NzgyIgz6LOHKqD7pmvQXBf8q4AJkRxmxk5UhNmr6iAx+zsccBvcFR2XBFywrgrPCHk5mUzQXfgPFBmcTW7XTB2F2Rie5y/oZGV1YxK8s2JYXd9gkbJvwUt7K1bNjQYg/IMQ1goecxH2dRHq//JYDs9TQiU01nacnI6eMPQqSCpmtc2yI45GovWwONKJviddUI5Ln/IS7Ktq4dcmy7xx/GqbgqsWb3bCFhmT/9UrOGaBSOHA0fhshKaw8ihbWgz1BtVFSt2i8FsTXRlNXn5l79iwo+vWs4CQzmHgRjw3ZIytrw2V2nppgJlw1Tb+AEwBdbC9wW9WsmDmw9QmBeyxcKAKkmPf66wWqmBqK0/2w8r4q6H7gEta/Ibc08137zd0fLWEu74iZE0I5jhLYOGwh+U0DhsR0ZDufWurcA+YIfWZAsfG0PePonv8+6drcqtNjTqKs0KIfAQ6PHAb4xuoCYEzvLeyMcY1t5sOl/oU+RWxkPrsIMMLLgbMGOqUBfx6TS5599bddNIEV2CH3AHym/18n3lyA19PHcVcfldqU7nMrYTZdgNBMy1Or1OVjcxYV988OI8ZOqWPiYzMGR6cVSzwA1ESjLUVma3vKcOTq4GbgBvRxA44PMfBm/P3ow3Vhx3ONe/oaxqgKre26JBheTN5s0Uua4lEcO4lvgB6/Kap+/aaR9vnEwLtJZ8u5d84k/Ajw1nG8bDMfLPsyWXjtUUZI" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXG7BOCV4A" +AWS_SECRET_ACCESS_KEY="xDx1H2SOL71C1N9jl9pv5HqCyLccRsYs8vQvPU5y" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjED0aCXVzLWVhc3QtMiJIMEYCIQCJo1mxCG+aqqh1B5Jy1vvEcANPejqBUIOVnbPcUs5/+QIhAJmpSAygER0WddjIenyY8hu0V4aqiFCsdBBQbkupzvwzKowDCMf//////////wEQABoMNjYwMTMxMDY4NzgyIgxuroZvh2Y2VPEKEYgq4AK36l5Ls8q/xw+e0A1s6+TpUjvecAr9tUrwEF5MyY0ToWSyac7/N8y/g7p8Kd9lMiJqJ1C7YlePsgGQen9BBxLeLWp/yEWsKuQsyJmh2mcO9DtSFYkWeZgRmxT6KVflCySKzMDOtytwU1vzwrbjQeh8pRG7zijvO0WvGPfBykHYOp9dIGIG8kGa6IUq58EhpzKtCSahOFbaddamvzf+HabTZ45tv5RJbG/Lrw3FqEFAzrj7RkiUS3dnJoScH595PtEHZ3IHDYE2jvyeS/FqV+t454IvTV4avq9SePDtAAjw4CnQVyFR7gchPil+qIKgt2+IDHaqTAIRbR3isC/X2TTS8kw7pkm1Uqgz60DWsFrjrpWpIGqHAYmxmnbEX0rNkGzJgaX6YdDF2o7LTm6z0rBvjmY96uKk1QPs07tVJMxklx1+QFzMzpnZEuDp8KEaKW9xQg3IGP2TuSJM8ZAKDQ4uMNaug7MGOqUBRaIqNEebTiBJ9b1LvNOyonXyhqeq8GsA4y0MPcUDnk5JDFUfUt/1o6gefIV6pFsel5RIwHf49rfsrIsGQs5YzuViwP3whkloYPbg8ZXlQTJRVTXHaOqjQHVfeT26aU9S4VOFfqH8N/dhLVNUMpesutnkzrngyv0nUTubYt7LtBF5A0RvcTqMMfmU9pbjUTNk1fhsd4mOBAuHDrOTMXm+1LMvDJyc" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/file_processing.py b/src/file_processing.py index 1507097..823c296 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -8,6 +8,8 @@ import table_funcs import prompt_funcs import prompts import claude_funcs +import postprocessingfuncs +import postprocess def clean_td(td): td_clean = [] @@ -104,22 +106,22 @@ def process_file(file_object): # Run Top Down td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print("TD Results:", td_results) + print("Top Down complete...") # Run Bottom Up bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries - print("BU Results:", bu_results) + print("Bottom Up complete...") # Combine combined_results = merge_results(clean_td(td_results), bu_results, text_dict) - print("Combined Results:", combined_results) + print("TD/BU Merge complete...") # Convert to DataFrame combined_df = pd.DataFrame(combined_results) # Post-process combined results - # post_processed_combined_df = postprocess.postprocess_results(combined_df) - # print("Post-processed Results:", post_processed_combined_df) + combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) + post_processed_combined_df = postprocess.postprocess_results(combined_df) # Create directories base_filename = os.path.splitext(filename)[0] @@ -129,6 +131,5 @@ def process_file(file_object): # Save results pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) - combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) - # post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results.csv'), index=False) - + post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) + \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index ee807f1..787dbc4 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,4 +1,5 @@ import pandas as pd +import re import postprocessingfuncs def sanitize_combined(df): @@ -7,17 +8,66 @@ def sanitize_combined(df): df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df -def post_process_combined(df, ): +def extract_codes(value, pattern): + """ + Extract codes from a given string using a specified regex pattern. + """ + if pd.isna(value): + return value + matches = re.findall(pattern, str(value)) + if matches: + return ', '.join(matches) + else: + # Check if there are any numbers in the value + if any(char.isdigit() for char in value): + return value + else: + return "" + +def clean_code_columns(df, column_name, pattern): + """ + Clean the specified column in the DataFrame by extracting codes based on the provided regex pattern. + """ + df[column_name] = df[column_name].apply(lambda x: extract_codes(x, pattern)) + return df + +def post_process_combined(df): VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] + + # Define valid values dictionary + valid_values_dict = { + 'CONTRACT_LOB': VALID_LOBS, + 'CONTRACT_PROGRAM': VALID_PROGRAMS, + 'CONTRACT_NETWORK': VALID_NETWORKS + } # Sanitize the combined data df = sanitize_combined(df) + # Clean specific columns for exact matches postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - postprocessingfuncs.clean_columns_combined(df, 'PRODUCT', VALID_PROGRAMS, 'Corrected_PRODUCT') + postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + # Apply fuzzy matching to the columns + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) + postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) + + # Correct misplaced values across columns + df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + + # Define the patterns for the different code types + cpt_code_pattern = r'[A-Za-z0-9]{4}[A-Za-z]|[A-Za-z0-9]{3}[A-Za-z][0-9]|[A-Za-z0-9]{2}[A-Za-z][0-9]{2}|[A-Za-z0-9][A-Za-z][0-9]{3}|[A-Za-z][0-9]{4}' + drg_code_pattern = r'\b\d{3}\b' + rev_code_pattern = r'\b\d[A-Za-z]\d{2}|\d{2}[A-Za-z]\d|\d{3}[A-Za-z]|[A-Za-z]\d{3}|\d{4}\b' + + # Clean the specified columns + df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES', cpt_code_pattern) + df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES', drg_code_pattern) + df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES', rev_code_pattern) clean_df = postprocessingfuncs.clean_pagenumbers(df) return clean_df @@ -25,3 +75,4 @@ def post_process_combined(df, ): def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) return combined_df + diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 9dfdd49..0325a76 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -6,12 +6,11 @@ def sanitize_value(value): """ Remove brackets from list items and clean the values. """ if pd.isna(value): return value + if isinstance(value, list): + value = ', '.join(str(v) for v in value) if isinstance(value, str): value = value.strip('[]') - if ',' in value: - value = ', '.join([item.strip(" '") for item in value.split(',')]) - else: - value = value.strip(" '") + value = ', '.join([item.strip(" '") for item in value.split(',')]) return value def exact_match(val, valid_values): @@ -24,11 +23,12 @@ def exact_match(val, valid_values): def clean_columns_combined(df, column_name, valid_values, new_column_name): """ Cleans a column by applying an exact match check and updates it to a new column. """ changes = {} + df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() def update_column(entry): if pd.notna(entry): - terms = sanitize_value(entry).split(',') + terms = entry.split(',') for term in terms: match = exact_match(term, valid_values) if match: @@ -39,7 +39,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes -def get_closest_match(val, valid_values, similarity_threshold=0.5): +def get_closest_match(val, valid_values, similarity_threshold=0.7): if pd.isna(val): return None val = val.strip().upper() @@ -49,11 +49,12 @@ def get_closest_match(val, valid_values, similarity_threshold=0.5): def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): """ Applies fuzzy matching to a column in the dataframe and logs changes. """ changes = {} + df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() def log_and_clean(entry): if pd.notna(entry): - words = sanitize_value(entry).split(',') + words = entry.split(',') cleaned_words = [] for word in words: cleaned_word = get_closest_match(word.strip(), valid_values, similarity_threshold=threshold) @@ -75,21 +76,29 @@ def extract_page_number(page_text): return page_text def clean_pagenumbers(df): - df['Page'] = df['Page'].apply(extract_page_number) + df['page_num'] = df['page_num'].apply(extract_page_number) return df -def consolidate_csv(df, output_csv='output/consolidated.csv'): - df['page_num'] = df['Page'].apply(lambda x: x.split()[1]) - - grouped = df.groupby(['Filename', 'Corrected_LOB']).agg({ - 'PRODUCT': lambda x: ', '.join(x.dropna().unique()), - 'Corrected_NETWORK': lambda x: ', '.join(x.dropna().unique()), - 'CONTRACT_SERVICE_AREA': lambda x: ', '.join(x.dropna().unique()), - 'LOB_PRICING_TERMS_EFFECTIVE_DT': lambda x: ', '.join(x.dropna().unique()), - 'LOB_PRICING_TERMS_TERMINATION_DT': lambda x: ', '.join(x.dropna().unique()), - 'CONTRACT_MARKETPLACE_METAL_LEVEL': lambda x: ', '.join(x.dropna().unique()), - 'page_num' : lambda x : ', '.join(x.dropna().unique()) - }).reset_index() - - grouped.to_csv(output_csv, index=False) - print(f"Consolidated CSV has been saved to {output_csv}") +def correct_misplaced_values(df, columns, valid_values_dict): + """ + Check and correct misplaced values across specified columns. + """ + for index, row in df.iterrows(): + for col in columns: + if pd.notna(row[col]): + terms = row[col].split(',') + for term in terms: + term = term.strip() + for target_col, valid_values in valid_values_dict.items(): + if target_col != col: + match = exact_match(term, valid_values) + if match: + if pd.isna(row[target_col]) or not row[target_col].strip(): + df.at[index, target_col] = match + df.at[index, col] = None + else: + current_value = row[target_col].strip() + if get_closest_match(match, [current_value], 0.8) is None: + df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell" + df.at[index, col] = None + return df diff --git a/src/prompts.py b/src/prompts.py index 86fffde..0e6567b 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -148,6 +148,8 @@ Read and analyze the page, then populate a JSON dictionary with the following co Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. +Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. + Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ diff --git a/src/test.py b/src/test.py index 9d3aa07..86893aa 100644 --- a/src/test.py +++ b/src/test.py @@ -1,5 +1,22 @@ +import utils +import preprocess +import table_funcs +import prompts +import prompt_funcs + +input_dict = utils.read_input() + +filename = list(input_dict.keys())[0] +contract_text = input_dict[filename] + + +contract_text = preprocess.clean_newlines(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) + +bu_results = prompt_funcs.run_bottom_up(filename, text_dict) + +print(bu_results) -#import llama_index -import openai -print(openai.__version__) \ No newline at end of file From 42956469b8f46e51578c2c08627f7fdb357ba84d Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 6 Jun 2024 07:26:52 -0700 Subject: [PATCH 35/78] Add implied 100% logic to lesser of prompt --- src/config.py | 6 +++--- src/file_processing.py | 8 +++++--- src/prompts.py | 2 ++ src/test.py | 3 ++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/config.py b/src/config.py index 9ba5975..65b4bdc 100644 --- a/src/config.py +++ b/src/config.py @@ -35,9 +35,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXG7BOCV4A" -AWS_SECRET_ACCESS_KEY="xDx1H2SOL71C1N9jl9pv5HqCyLccRsYs8vQvPU5y" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjED0aCXVzLWVhc3QtMiJIMEYCIQCJo1mxCG+aqqh1B5Jy1vvEcANPejqBUIOVnbPcUs5/+QIhAJmpSAygER0WddjIenyY8hu0V4aqiFCsdBBQbkupzvwzKowDCMf//////////wEQABoMNjYwMTMxMDY4NzgyIgxuroZvh2Y2VPEKEYgq4AK36l5Ls8q/xw+e0A1s6+TpUjvecAr9tUrwEF5MyY0ToWSyac7/N8y/g7p8Kd9lMiJqJ1C7YlePsgGQen9BBxLeLWp/yEWsKuQsyJmh2mcO9DtSFYkWeZgRmxT6KVflCySKzMDOtytwU1vzwrbjQeh8pRG7zijvO0WvGPfBykHYOp9dIGIG8kGa6IUq58EhpzKtCSahOFbaddamvzf+HabTZ45tv5RJbG/Lrw3FqEFAzrj7RkiUS3dnJoScH595PtEHZ3IHDYE2jvyeS/FqV+t454IvTV4avq9SePDtAAjw4CnQVyFR7gchPil+qIKgt2+IDHaqTAIRbR3isC/X2TTS8kw7pkm1Uqgz60DWsFrjrpWpIGqHAYmxmnbEX0rNkGzJgaX6YdDF2o7LTm6z0rBvjmY96uKk1QPs07tVJMxklx1+QFzMzpnZEuDp8KEaKW9xQg3IGP2TuSJM8ZAKDQ4uMNaug7MGOqUBRaIqNEebTiBJ9b1LvNOyonXyhqeq8GsA4y0MPcUDnk5JDFUfUt/1o6gefIV6pFsel5RIwHf49rfsrIsGQs5YzuViwP3whkloYPbg8ZXlQTJRVTXHaOqjQHVfeT26aU9S4VOFfqH8N/dhLVNUMpesutnkzrngyv0nUTubYt7LtBF5A0RvcTqMMfmU9pbjUTNk1fhsd4mOBAuHDrOTMXm+1LMvDJyc" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXINEKTKP3" +AWS_SECRET_ACCESS_KEY="+CBv7rMWD03nbdEx0UNoZ4HzwGvWHxk5JcnsgjEy" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEE4aCXVzLWVhc3QtMiJIMEYCIQC/lDrqlB5C3FLvINHn7ZJ0Ae3vgZueO0sFYnZ7kk8X0AIhAL/GYR8hsoMC7jyE0gQ3wAEOZKv6+aSgB7e0hGyLecEnKowDCNf//////////wEQABoMNjYwMTMxMDY4NzgyIgzjw/Ad98FNbwU4nwkq4AIdE/cCiDWuHkwFtSWm+9Nwijwyjo7X8RJxtrXl1qldj/wWWk9PbZw1Jd8wPA/dA5U5QwCdQ4XKjAyPkeqs93o6oOUqOHKgw8315izvbCF1Hc6jyjh9w3Z2kcennbGC6fz3repCgC4Mc5z3HHTpx8UAU92R8SZrJ4h0VsNwvWWmCemTEG60DzVmsZrlf07xwDQqOR7a5fs2VedzU3nvQmz/HMaEb0cz4qiIfp9bjC+hcQwShaXgU2mpS2vzGd4/L1zrVyJywsX2usM1ggpcjiZQZA9AuxOaPmExLOV8jLkARogd7Ma9NVL1BxfhSCVWqHDjrEt7bkjTvAeMwu/uNqkTS9ikF0/XRFd9q9VER9NSeueyi4kUkXzxlxMrCJtaByaJmk4m4IFhF7MPkOBrCjKozXOytTAcSHwhsiYM6XYUQznkrmde3SzXCNC6QQK2zOUNa1HQ/XREktxqp97YuSxQMLX7hrMGOqUBLOjzFxEGAFTHPRKb3vMorBPQyhqhzhEGgi/cnOSehvGLrZ1O70Ngyq6tBhLhinEJKpwzN/4XZU5MDPsAb3sxHsqwHFEmWgPuCyHb5JsX7jb4qIPFQcX6rklZcA8QpzeHK1ffXphknVDk5OiVcDf4tW6dWzE6J9fDyMY7FGzqWe/Aw+YtB3l/Of5FV61M2/uks5msY1/MyPy3nIjvDDF/3CpW9sAA" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/file_processing.py b/src/file_processing.py index 823c296..2e9f2e6 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -106,15 +106,15 @@ def process_file(file_object): # Run Top Down td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print("Top Down complete...") + print(f"{filename} Top Down complete...") # Run Bottom Up bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries - print("Bottom Up complete...") + print(f"{filename} Bottom Up complete...") # Combine combined_results = merge_results(clean_td(td_results), bu_results, text_dict) - print("TD/BU Merge complete...") + print(f"{filename} TD/BU Merge complete...") # Convert to DataFrame combined_df = pd.DataFrame(combined_results) @@ -122,6 +122,7 @@ def process_file(file_object): # Post-process combined results combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) post_processed_combined_df = postprocess.postprocess_results(combined_df) + print(f"{filename} Postprocessing complete...") # Create directories base_filename = os.path.splitext(filename)[0] @@ -132,4 +133,5 @@ def process_file(file_object): pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) + print(f"{filename} Output complete...") \ No newline at end of file diff --git a/src/prompts.py b/src/prompts.py index 0e6567b..f9fa46e 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -77,6 +77,8 @@ Read and analyze the page, then populate a JSON dictionary with the following fi For REIMBURSEMENT_RATE and REIMBURSEMENT_FLAT_FEE, do NOT include the Rate or Flat Fee from Original_Methodology. +Note that in some cases, the REIMBURSEMENT_RATE will not be explicitly written. It may say something like "Provider's billed charges", which is essentially equivalent to "100% of Provider's billed charges". In these cases, assign REIMBURSEMENT_RATE a value of 100%. + Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ diff --git a/src/test.py b/src/test.py index 86893aa..d9a7dd9 100644 --- a/src/test.py +++ b/src/test.py @@ -1,3 +1,4 @@ +import pandas as pd import utils import preprocess @@ -18,5 +19,5 @@ text_dict = preprocess.highlight_rates(text_dict) bu_results = prompt_funcs.run_bottom_up(filename, text_dict) -print(bu_results) +print(pd.DataFrame(bu_results)) From ff7814d63101324e996e1f771e4ed859dff54f57 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 6 Jun 2024 13:57:53 -0700 Subject: [PATCH 36/78] Add modifiers to PROC_CODE column --- src/file_processing.py | 20 +++++++++----------- src/prompts.py | 2 ++ src/test.py | 38 +++++++++++++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/src/file_processing.py b/src/file_processing.py index 2e9f2e6..a0fecf7 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -51,10 +51,8 @@ def get_unique_date_fields(td_results): def merge_results(td_results, bu_results, text_dict): all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) - print("All Keys:", all_keys) date_fields = get_unique_date_fields(td_results) - print("Date Fields:", date_fields) merged_results = [] for bu in bu_results: @@ -71,10 +69,10 @@ def merge_results(td_results, bu_results, text_dict): merged[key].extend(value) for date_key, date_values in date_fields.items(): - if date_key not in merged or not merged[date_key]: - merged[date_key] = date_values - else: - merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) + if date_key not in merged or not merged[date_key]: + merged[date_key] = date_values + else: + merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) for key in all_keys: if isinstance(merged[key], list) and len(merged[key]) > 1: @@ -106,15 +104,15 @@ def process_file(file_object): # Run Top Down td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print(f"{filename} Top Down complete...") + print(f"Top Down Complete - {filename} ") # Run Bottom Up bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries - print(f"{filename} Bottom Up complete...") + print(f"Bottom Up Complete - {filename}") # Combine combined_results = merge_results(clean_td(td_results), bu_results, text_dict) - print(f"{filename} TD/BU Merge complete...") + print(f"TD/BU Merge Complete - {filename} ") # Convert to DataFrame combined_df = pd.DataFrame(combined_results) @@ -122,7 +120,7 @@ def process_file(file_object): # Post-process combined results combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) post_processed_combined_df = postprocess.postprocess_results(combined_df) - print(f"{filename} Postprocessing complete...") + print(f"Postprocessing Complete - {filename} ") # Create directories base_filename = os.path.splitext(filename)[0] @@ -133,5 +131,5 @@ def process_file(file_object): pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) - print(f"{filename} Output complete...") + print(f"Output Complete - {filename} ") \ No newline at end of file diff --git a/src/prompts.py b/src/prompts.py index f9fa46e..2b65a07 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -150,6 +150,8 @@ Read and analyze the page, then populate a JSON dictionary with the following co Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. +Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them. + Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. diff --git a/src/test.py b/src/test.py index d9a7dd9..92f5122 100644 --- a/src/test.py +++ b/src/test.py @@ -5,19 +5,51 @@ import preprocess import table_funcs import prompts import prompt_funcs +import claude_funcs + +def BOTTOM_UP_CODES(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Your job is to identify codes associated with this specific term. + +Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided): +'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility. +'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes. +'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar. +'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options. +'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service. +'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures. +'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes. +'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status. + +Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. + +Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them. + +Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. + +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" input_dict = utils.read_input() filename = list(input_dict.keys())[0] contract_text = input_dict[filename] - contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) -bu_results = prompt_funcs.run_bottom_up(filename, text_dict) +d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} +prompt = BOTTOM_UP_CODES(d, text_dict['3']) + +answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + +print(answer) -print(pd.DataFrame(bu_results)) From 9f63064c86f1b6e7e5b1fa4ec25c82064b4b7217 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 6 Jun 2024 17:23:26 -0700 Subject: [PATCH 37/78] Modified table preprocessing, postprocessing non functional in this commit --- src/postprocess.py | 47 +++++++--------- src/postprocessingfuncs.py | 19 +++++-- src/table_funcs.py | 38 +++++++++++-- src/test.py | 112 ++++++++++++++++++++++++++----------- 4 files changed, 146 insertions(+), 70 deletions(-) diff --git a/src/postprocess.py b/src/postprocess.py index 787dbc4..25b863f 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -3,32 +3,28 @@ import re import postprocessingfuncs def sanitize_combined(df): - """ Sanitize the combined data by removing brackets and cleaning values. """ + for column in df.columns: df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df -def extract_codes(value, pattern): - """ - Extract codes from a given string using a specified regex pattern. - """ +def extract_codes_CPT(value): if pd.isna(value): return value - matches = re.findall(pattern, str(value)) + + value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE) + + pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b' + matches = re.findall(pattern, value) if matches: - return ', '.join(matches) + + return ', '.join([''.join(match) for match in matches]) else: - # Check if there are any numbers in the value - if any(char.isdigit() for char in value): - return value - else: - return "" + return "" -def clean_code_columns(df, column_name, pattern): - """ - Clean the specified column in the DataFrame by extracting codes based on the provided regex pattern. - """ - df[column_name] = df[column_name].apply(lambda x: extract_codes(x, pattern)) +def clean_code_columns(df, column_name): + + df[column_name] = df[column_name].apply(extract_codes) return df def post_process_combined(df): @@ -59,15 +55,13 @@ def post_process_combined(df): # Correct misplaced values across columns df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) - # Define the patterns for the different code types - cpt_code_pattern = r'[A-Za-z0-9]{4}[A-Za-z]|[A-Za-z0-9]{3}[A-Za-z][0-9]|[A-Za-z0-9]{2}[A-Za-z][0-9]{2}|[A-Za-z0-9][A-Za-z][0-9]{3}|[A-Za-z][0-9]{4}' - drg_code_pattern = r'\b\d{3}\b' - rev_code_pattern = r'\b\d[A-Za-z]\d{2}|\d{2}[A-Za-z]\d|\d{3}[A-Za-z]|[A-Za-z]\d{3}|\d{4}\b' - - # Clean the specified columns - df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES', cpt_code_pattern) - df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES', drg_code_pattern) - df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES', rev_code_pattern) + # Clean the specified code columns + df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES') + df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES') + df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES') + + # Filter out specific SERVICE rows + df = postprocessingfuncs.filter_service_column(df) clean_df = postprocessingfuncs.clean_pagenumbers(df) return clean_df @@ -75,4 +69,3 @@ def post_process_combined(df): def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) return combined_df - diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 0325a76..12ed2b5 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -3,7 +3,7 @@ import re import difflib def sanitize_value(value): - """ Remove brackets from list items and clean the values. """ + if pd.isna(value): return value if isinstance(value, list): @@ -21,7 +21,7 @@ def exact_match(val, valid_values): return None def clean_columns_combined(df, column_name, valid_values, new_column_name): - """ Cleans a column by applying an exact match check and updates it to a new column. """ + changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -47,7 +47,7 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7): return matches[0] if matches else None def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): - """ Applies fuzzy matching to a column in the dataframe and logs changes. """ + changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -80,9 +80,7 @@ def clean_pagenumbers(df): return df def correct_misplaced_values(df, columns, valid_values_dict): - """ - Check and correct misplaced values across specified columns. - """ + for index, row in df.iterrows(): for col in columns: if pd.notna(row[col]): @@ -102,3 +100,12 @@ def correct_misplaced_values(df, columns, valid_values_dict): df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell" df.at[index, col] = None return df + +def filter_service_column(df): + + keywords = [ + 'LIABILITY', 'RISK' ,'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' + ] + pattern = '|'.join(keywords) + df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] + return df diff --git a/src/table_funcs.py b/src/table_funcs.py index 1c7af74..9e93b8b 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -3,6 +3,8 @@ import re import pandas as pd import json import ast +import csv +from io import StringIO def extract_tables(text): pattern = re.compile(r"-------Table Start--------(.*?)-------Table End--------", re.DOTALL) @@ -21,9 +23,33 @@ def extract_all_tables(input_dict): return tables_dict -def format_table(table_json): +def convert_to_dict(table_text): + table_text = table_text.strip('{}') + table_text_list = table_text.split(']') + table_text_list = [item for item in table_text_list if len(item) > 0] + + final_dict = {} + num_elements = 0 + for key_value_text in table_text_list: + key = key_value_text.split(':')[0].strip(', \'') + value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') + + f = StringIO(value) + reader = csv.reader(f, delimiter=',') + + value_list = [] + for row in reader: + for r in row: + value_list.append(r) + + final_dict[key] = value_list + num_elements = len(value_list) + return final_dict, num_elements + + +def format_table(table_json, table_size): table_text = "" - for i in range(len(table_json[list(table_json.keys())[0]])): + for i in range(table_size): for key in table_json.keys(): if table_json[key][i]: #table_text += key + ': ' + table_json[key][i] + ', ' @@ -42,10 +68,12 @@ def align_and_format_tables(text_dict): pretable = table_text.split('{')[0].strip() # Extract and format table text - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table = json.loads(table_only) - table_formatted = format_table(table) + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + # Align if text.count(pretable) == 2: # Remove original table diff --git a/src/test.py b/src/test.py index 92f5122..2f2b869 100644 --- a/src/test.py +++ b/src/test.py @@ -1,4 +1,8 @@ import pandas as pd +import re +import json +import csv +from io import StringIO import utils import preprocess @@ -7,49 +11,93 @@ import prompts import prompt_funcs import claude_funcs -def BOTTOM_UP_CODES(d, page): - return f"""### PAGE START ### {page} ### PAGE END - -The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: -Service: {d['SERVICE']} -Methodology: {d['FULL_METHODOLOGY']} - -Your job is to identify codes associated with this specific term. - -Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided): -'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility. -'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes. -'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar. -'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options. -'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service. -'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures. -'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes. -'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status. - -Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. - -Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them. - -Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. - -Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. -""" input_dict = utils.read_input() filename = list(input_dict.keys())[0] contract_text = input_dict[filename] +def convert_to_dict(table_text): + table_text = table_text.strip('{}') + table_text_list = table_text.split(']') + table_text_list = [item for item in table_text_list if len(item) > 0] + + final_dict = {} + num_elements = 0 + for key_value_text in table_text_list: + key = key_value_text.split(':')[0].strip(', \'') + value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') + + f = StringIO(value) + reader = csv.reader(f, delimiter=',') + + value_list = [] + for row in reader: + for r in row: + value_list.append(r) + + final_dict[key] = value_list + num_elements = len(value_list) + return final_dict, num_elements + + +def format_table(table_json, table_size): + table_text = "" + for i in range(table_size): + for key in table_json.keys(): + if table_json[key][i]: + #table_text += key + ': ' + table_json[key][i] + ', ' + table_text += table_json[key][i] + ' : ' + table_text += r'\n' + return table_text + + +def align_and_format_tables(text_dict): + aligned_text_dict = {} + for key, text in text_dict.items(): + if 'Table Start' in text: + table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) + for table_text in table_texts: + try: + # Extract pretable text + pretable = table_text.split('{')[0].strip() + + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + + # Align + if text.count(pretable) == 2: + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: + aligned_text = text.replace(table_text, pretable + table_formatted) + except: + aligned_text = text + aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") + else: + aligned_text_dict[key] = text + + return aligned_text_dict + contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) -d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} -prompt = BOTTOM_UP_CODES(d, text_dict['3']) +#print(text_dict['13']) -answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +# d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} +# prompt = BOTTOM_UP_CODES(d, text_dict['3']) + +# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + +# print(answer) -print(answer) From ce73b9d1a09c1e36d0e5cab9a340a2f68cabde11 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 06:28:41 -0700 Subject: [PATCH 38/78] Modified table preprocessing to incorporate dict keys --- src/table_funcs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/table_funcs.py b/src/table_funcs.py index 9e93b8b..0d62314 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -49,8 +49,10 @@ def convert_to_dict(table_text): def format_table(table_json, table_size): table_text = "" + keys = [key for key in table_json.keys()] + table_text += ' : '.join(keys) + "\n" for i in range(table_size): - for key in table_json.keys(): + for key in keys: if table_json[key][i]: #table_text += key + ': ' + table_json[key][i] + ', ' table_text += table_json[key][i] + ' : ' From de8e94827d7521182b842643396b42e1dfea9578 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 07:06:00 -0700 Subject: [PATCH 39/78] Postprocessing bug fixes --- src/config.py | 8 ++++---- src/postprocess.py | 39 ++++++++++++++++++++++++++++++-------- src/postprocessingfuncs.py | 27 ++++++++++++++++++++------ 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/src/config.py b/src/config.py index 65b4bdc..5e3d8c8 100644 --- a/src/config.py +++ b/src/config.py @@ -6,7 +6,7 @@ from llama_index.llms.bedrock import Bedrock #pip install llama-index-llms-bedrock # General Settings -TEST = False # True to run test prompt - just for testing model connection +TEST = True # True to run test prompt - just for testing model connection VERBOSE = True CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") @@ -35,9 +35,9 @@ RUN_EXCEPTION = True RUN_CODES = True # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXINEKTKP3" -AWS_SECRET_ACCESS_KEY="+CBv7rMWD03nbdEx0UNoZ4HzwGvWHxk5JcnsgjEy" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEE4aCXVzLWVhc3QtMiJIMEYCIQC/lDrqlB5C3FLvINHn7ZJ0Ae3vgZueO0sFYnZ7kk8X0AIhAL/GYR8hsoMC7jyE0gQ3wAEOZKv6+aSgB7e0hGyLecEnKowDCNf//////////wEQABoMNjYwMTMxMDY4NzgyIgzjw/Ad98FNbwU4nwkq4AIdE/cCiDWuHkwFtSWm+9Nwijwyjo7X8RJxtrXl1qldj/wWWk9PbZw1Jd8wPA/dA5U5QwCdQ4XKjAyPkeqs93o6oOUqOHKgw8315izvbCF1Hc6jyjh9w3Z2kcennbGC6fz3repCgC4Mc5z3HHTpx8UAU92R8SZrJ4h0VsNwvWWmCemTEG60DzVmsZrlf07xwDQqOR7a5fs2VedzU3nvQmz/HMaEb0cz4qiIfp9bjC+hcQwShaXgU2mpS2vzGd4/L1zrVyJywsX2usM1ggpcjiZQZA9AuxOaPmExLOV8jLkARogd7Ma9NVL1BxfhSCVWqHDjrEt7bkjTvAeMwu/uNqkTS9ikF0/XRFd9q9VER9NSeueyi4kUkXzxlxMrCJtaByaJmk4m4IFhF7MPkOBrCjKozXOytTAcSHwhsiYM6XYUQznkrmde3SzXCNC6QQK2zOUNa1HQ/XREktxqp97YuSxQMLX7hrMGOqUBLOjzFxEGAFTHPRKb3vMorBPQyhqhzhEGgi/cnOSehvGLrZ1O70Ngyq6tBhLhinEJKpwzN/4XZU5MDPsAb3sxHsqwHFEmWgPuCyHb5JsX7jb4qIPFQcX6rklZcA8QpzeHK1ffXphknVDk5OiVcDf4tW6dWzE6J9fDyMY7FGzqWe/Aw+YtB3l/Of5FV61M2/uks5msY1/MyPy3nIjvDDF/3CpW9sAA" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAI7RYNWW" +AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/postprocess.py b/src/postprocess.py index 25b863f..c7e3bc5 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -3,7 +3,6 @@ import re import postprocessingfuncs def sanitize_combined(df): - for column in df.columns: df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df @@ -13,18 +12,38 @@ def extract_codes_CPT(value): return value value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE) - + pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b' matches = re.findall(pattern, value) if matches: - return ', '.join([''.join(match) for match in matches]) else: return "" -def clean_code_columns(df, column_name): +def extract_codes_Diagnosis(value): + if pd.isna(value): + return value - df[column_name] = df[column_name].apply(extract_codes) + pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b' + matches = re.findall(pattern, value) + if matches: + return ', '.join(matches) + else: + return "" + +def extract_codes_Revenue(value): + if pd.isna(value): + return value + + pattern = r'\b\d{3,4}\b' + matches = re.findall(pattern, value) + if matches: + return ', '.join(matches) + else: + return "" + +def clean_code_column(df, column_name, extract_function): + df[column_name] = df[column_name].apply(extract_function) return df def post_process_combined(df): @@ -56,13 +75,17 @@ def post_process_combined(df): df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) # Clean the specified code columns - df = clean_code_columns(df, 'REIMBURSEMENT_PROC_CODES') - df = clean_code_columns(df, 'REIMBURSEMENT_DIAG_CODES') - df = clean_code_columns(df, 'REIMBURSEMENT_REVENUE_CODES') + df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) + df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) + df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) # Filter out specific SERVICE rows df = postprocessingfuncs.filter_service_column(df) + # Move percentages and large numbers to correct columns + df = postprocessingfuncs.move_percentage_to_rate(df) + df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) + clean_df = postprocessingfuncs.clean_pagenumbers(df) return clean_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 12ed2b5..7fbb9eb 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -3,7 +3,6 @@ import re import difflib def sanitize_value(value): - if pd.isna(value): return value if isinstance(value, list): @@ -21,7 +20,6 @@ def exact_match(val, valid_values): return None def clean_columns_combined(df, column_name, valid_values, new_column_name): - changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -47,7 +45,6 @@ def get_closest_match(val, valid_values, similarity_threshold=0.7): return matches[0] if matches else None def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): - changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -80,7 +77,6 @@ def clean_pagenumbers(df): return df def correct_misplaced_values(df, columns, valid_values_dict): - for index, row in df.iterrows(): for col in columns: if pd.notna(row[col]): @@ -102,10 +98,29 @@ def correct_misplaced_values(df, columns, valid_values_dict): return df def filter_service_column(df): - keywords = [ - 'LIABILITY', 'RISK' ,'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' + 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' ] pattern = '|'.join(keywords) df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] return df + +def move_percentage_to_rate(df): + def move_percentage(value): + if isinstance(value, str) and '%' in value: + return True + return False + + df['REIMBURSEMENT_RATE'] = df.apply(lambda row: row['REIMBURSEMENT_FLAT_FEE'] if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_RATE'], axis=1) + df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) + return df + +def move_large_numbers_to_flat_fee(df): + def move_large_number(value): + if isinstance(value, str) and value.isdigit() and int(value) > 100: + return True + return False + + df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: row['REIMBURSEMENT_RATE'] if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) + df['REIMBURSEMENT_RATE'] = df.apply(lambda row: None if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_RATE'], axis=1) + return df From f4153c0520cecabdb553b86ab72339fcfefa6f33 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 07:49:04 -0700 Subject: [PATCH 40/78] Updated outputs --- src/config.py | 4 ++-- src/file_processing.py | 2 ++ src/main.py | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/config.py b/src/config.py index 5e3d8c8..432bec4 100644 --- a/src/config.py +++ b/src/config.py @@ -6,7 +6,7 @@ from llama_index.llms.bedrock import Bedrock #pip install llama-index-llms-bedrock # General Settings -TEST = True # True to run test prompt - just for testing model connection +TEST = False # True to run test prompt - just for testing model connection VERBOSE = True CLIENT_NAME = 'Payer' TODAY = datetime.now().strftime("%Y%m%d") @@ -40,7 +40,7 @@ AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/priority_health/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index a0fecf7..e35f8c7 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -130,6 +130,8 @@ def process_file(file_object): # Save results pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) + combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) + print(f"Output Complete - {filename} ") \ No newline at end of file diff --git a/src/main.py b/src/main.py index d6ccb75..18dc9e3 100644 --- a/src/main.py +++ b/src/main.py @@ -10,7 +10,6 @@ import prompt_funcs import claude_funcs import file_processing - def main(): if config.TEST: print(claude_funcs.invoke_claude_3("Write 'test', nothing more.", max_tokens = 10)) From 4710c55db24474781d91ab276e1d47ea97eb6b13 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 08:48:16 -0700 Subject: [PATCH 41/78] Add error handling for postprocessing --- src/config.py | 5 ++- src/file_processing.py | 38 +++++++++++--------- src/postprocess.py | 73 ++++++++++++++++++++++++++++++-------- src/postprocessingfuncs.py | 2 +- 4 files changed, 84 insertions(+), 34 deletions(-) diff --git a/src/config.py b/src/config.py index 432bec4..ab922f1 100644 --- a/src/config.py +++ b/src/config.py @@ -34,13 +34,16 @@ RUN_FS = True RUN_EXCEPTION = True RUN_CODES = True +# Postprocessing Settings +FUZZY_MATCH_THRESHOLD = 0.8 + # AWS Keys AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAI7RYNWW" AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths -LOCAL_PATH = 'data/priority_health/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index e35f8c7..c0afa8e 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -93,45 +93,49 @@ def merge_results(td_results, bu_results, text_dict): def process_file(file_object): + + # Start Processing filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") + # Create directories + base_filename = os.path.splitext(filename)[0].strip() + output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) + os.makedirs(output_dir, exist_ok=True) + + #### PREPROCESS #### contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) - # Run Top Down + #### Run Top Down #### td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - print(f"Top Down Complete - {filename} ") + # Write TD results + pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) + print(f"Top Down Complete - {filename}") - # Run Bottom Up + #### Run Bottom Up #### bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries + # Write BU results + pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) print(f"Bottom Up Complete - {filename}") - # Combine + #### Combine #### combined_results = merge_results(clean_td(td_results), bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") + #### Write unprocessed output #### # Convert to DataFrame combined_df = pd.DataFrame(combined_results) + combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) - # Post-process combined results + #### POSTPROCESSING #### combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) post_processed_combined_df = postprocess.postprocess_results(combined_df) - print(f"Postprocessing Complete - {filename} ") - - # Create directories - base_filename = os.path.splitext(filename)[0] - output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) - os.makedirs(output_dir, exist_ok=True) - - # Save results - pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) - pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) - combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) - + print(f"Postprocessing Complete - {filename} ") + print(f"Output Complete - {filename} ") \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index c7e3bc5..e822a0f 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,4 +1,5 @@ import pandas as pd +import config import re import postprocessingfuncs @@ -62,31 +63,73 @@ def post_process_combined(df): df = sanitize_combined(df) # Clean specific columns for exact matches - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') - postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + for field_list in [['CONTRACT_LOB', 'Corrected_LOB', VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', VALID_NETWORKS]]: + try: + postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) + except: + pass + + try: + postprocessingfuncs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) + except: + pass + + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - # Apply fuzzy matching to the columns - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) - postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') + # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') + + # # Apply fuzzy matching to the columns + # try: + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) + # except: + # pass + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) + # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) # Correct misplaced values across columns - df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + try: + df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + except: + pass # Clean the specified code columns - df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) - df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) - df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) + try: + df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) + except Exception as e: + print(f"Error: {e}") + try: + df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) + except Exception as e: + print(f"Error: {e}") + try: + df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) + except Exception as e: + print(f"Error: {e}") # Filter out specific SERVICE rows - df = postprocessingfuncs.filter_service_column(df) + try: + df = postprocessingfuncs.filter_service_column(df) + except Exception as e: + print(f"Error: {e}") # Move percentages and large numbers to correct columns - df = postprocessingfuncs.move_percentage_to_rate(df) - df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) + try: + df = postprocessingfuncs.move_percentage_to_rate(df) + except Exception as e: + print(f"Error: {e}") + + try: + df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) + except Exception as e: + print(f"Error: {e}") - clean_df = postprocessingfuncs.clean_pagenumbers(df) + try: + clean_df = postprocessingfuncs.clean_pagenumbers(df) + except Exception as e: + print(f"Error: {e}") + clean_df = df + return clean_df def postprocess_results(combined_df): diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 7fbb9eb..ab94eb1 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -32,7 +32,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): if match: return match return None - + df[new_column_name] = df[column_name].apply(update_column) cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes From a944dee2f89cff9c1f8f42c0ca2f18dceb4e5996 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 09:36:07 -0700 Subject: [PATCH 42/78] Error handling, streamlined IO --- src/config.py | 19 ++++-- src/file_processing.py | 30 ++++----- src/main.py | 11 +--- src/test.py | 140 +++++++++++++++++++++-------------------- src/utils.py | 28 +++++++++ 5 files changed, 130 insertions(+), 98 deletions(-) diff --git a/src/config.py b/src/config.py index ab922f1..9652321 100644 --- a/src/config.py +++ b/src/config.py @@ -8,7 +8,7 @@ from llama_index.llms.bedrock import Bedrock # General Settings TEST = False # True to run test prompt - just for testing model connection VERBOSE = True -CLIENT_NAME = 'Payer' +CLIENT_NAME = 'Priority Health' TODAY = datetime.now().strftime("%Y%m%d") # Valid values @@ -16,11 +16,18 @@ VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] VALID_PROGRAMS = [] VALID_LOB = [] -# I/O Options -WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written -OUTPUT_FOLDER = 'output' +# Input Settings READ_MODE = '_LOCAL_' # OR '_S3_' -OUTPUT_MODE = '_INDIVIDUAL_' # or '_CONSOLIDATED_' + +# Output Settings +WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written +OUTPUT_DIRECTORY = 'output' +CONSOLIDATED_OUTPUT_DIRECTORY = 'consolidated_output' +OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.csv' +TD_RESULTS_NAME = 'td_results.csv' +BU_RESULTS_NAME = 'bu_results.csv' +UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' +PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings MAX_WORKERS = 20 @@ -43,7 +50,7 @@ AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/priority_health/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index c0afa8e..cc3caa6 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -94,47 +94,43 @@ def merge_results(td_results, bu_results, text_dict): def process_file(file_object): - # Start Processing + ################## INITIATE PROCESSING ################## filename, contract_text = file_object if config.VERBOSE: print(f"Processing {filename}...") - # Create directories + ################## CREATE OUTPUT DIRECTORIES ################## base_filename = os.path.splitext(filename)[0].strip() - output_dir = os.path.join(config.OUTPUT_FOLDER, base_filename) + output_dir = os.path.join(config.OUTPUT_DIRECTORY, base_filename) os.makedirs(output_dir, exist_ok=True) - - #### PREPROCESS #### + ################## PREPROCESS ################## contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) - #### Run Top Down #### + ################## RUN TOP DOWN ################## td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page - # Write TD results - pd.DataFrame(td_results).to_csv(os.path.join(output_dir, 'td_results.csv'), index=False) + pd.DataFrame(td_results).to_csv(os.path.join(output_dir, config.TD_RESULTS_NAME), index=False) print(f"Top Down Complete - {filename}") - #### Run Bottom Up #### + ################## RUN BOTTOM UP ################## bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries - # Write BU results - pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, 'bu_results.csv'), index=False) + pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, config.BU_RESULTS_NAME), index=False) print(f"Bottom Up Complete - {filename}") - #### Combine #### + ################## COMBINE TOP DOWN AND BOTTOM UP ################## combined_results = merge_results(clean_td(td_results), bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") - #### Write unprocessed output #### - # Convert to DataFrame + ################## WRITE UNPROCESSED OUTPUT ################## combined_df = pd.DataFrame(combined_results) - combined_df.to_csv(os.path.join(output_dir, 'combined_results_unprocessed.csv'), index=False) + combined_df.to_csv(os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False) - #### POSTPROCESSING #### + ################## RUN POSTPROCESSING ################## combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) post_processed_combined_df = postprocess.postprocess_results(combined_df) - post_processed_combined_df.to_csv(os.path.join(output_dir, 'combined_results_post_processed.csv'), index=False) + post_processed_combined_df.to_csv(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) print(f"Postprocessing Complete - {filename} ") print(f"Output Complete - {filename} ") diff --git a/src/main.py b/src/main.py index 18dc9e3..4912bfa 100644 --- a/src/main.py +++ b/src/main.py @@ -18,10 +18,6 @@ def main(): input_dict = utils.read_input() # already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('results/')] # input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} - - # Set up results folder - if not os.path.exists('results'): - os.makedirs('results') def process_item(item): key, value = item @@ -45,10 +41,9 @@ def main(): except Exception as e: print(f"Error: {e}") - - # Write Output - if config.WRITE_OUTPUT and config.OUTPUT_MODE == '_CONSOLIDATED_': - consolidated_df = utils.consolidate_individual(input_folder='temp') # Consolidate temp results to one file, then write output + # Write Consolidated Output + if config.WRITE_OUTPUT: + utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) if __name__ == "__main__": diff --git a/src/test.py b/src/test.py index 2f2b869..f5b7790 100644 --- a/src/test.py +++ b/src/test.py @@ -3,6 +3,7 @@ import re import json import csv from io import StringIO +import config import utils import preprocess @@ -12,92 +13,97 @@ import prompt_funcs import claude_funcs -input_dict = utils.read_input() +if config.WRITE_OUTPUT: + utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) -filename = list(input_dict.keys())[0] -contract_text = input_dict[filename] -def convert_to_dict(table_text): - table_text = table_text.strip('{}') - table_text_list = table_text.split(']') - table_text_list = [item for item in table_text_list if len(item) > 0] - final_dict = {} - num_elements = 0 - for key_value_text in table_text_list: - key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') +# input_dict = utils.read_input() + +# filename = list(input_dict.keys())[0] +# contract_text = input_dict[filename] + +# def convert_to_dict(table_text): +# table_text = table_text.strip('{}') +# table_text_list = table_text.split(']') +# table_text_list = [item for item in table_text_list if len(item) > 0] + +# final_dict = {} +# num_elements = 0 +# for key_value_text in table_text_list: +# key = key_value_text.split(':')[0].strip(', \'') +# value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') - f = StringIO(value) - reader = csv.reader(f, delimiter=',') +# f = StringIO(value) +# reader = csv.reader(f, delimiter=',') - value_list = [] - for row in reader: - for r in row: - value_list.append(r) +# value_list = [] +# for row in reader: +# for r in row: +# value_list.append(r) - final_dict[key] = value_list - num_elements = len(value_list) - return final_dict, num_elements +# final_dict[key] = value_list +# num_elements = len(value_list) +# return final_dict, num_elements -def format_table(table_json, table_size): - table_text = "" - for i in range(table_size): - for key in table_json.keys(): - if table_json[key][i]: - #table_text += key + ': ' + table_json[key][i] + ', ' - table_text += table_json[key][i] + ' : ' - table_text += r'\n' - return table_text +# def format_table(table_json, table_size): +# table_text = "" +# for i in range(table_size): +# for key in table_json.keys(): +# if table_json[key][i]: +# #table_text += key + ': ' + table_json[key][i] + ', ' +# table_text += table_json[key][i] + ' : ' +# table_text += r'\n' +# return table_text -def align_and_format_tables(text_dict): - aligned_text_dict = {} - for key, text in text_dict.items(): - if 'Table Start' in text: - table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) - for table_text in table_texts: - try: - # Extract pretable text - pretable = table_text.split('{')[0].strip() +# def align_and_format_tables(text_dict): +# aligned_text_dict = {} +# for key, text in text_dict.items(): +# if 'Table Start' in text: +# table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) +# for table_text in table_texts: +# try: +# # Extract pretable text +# pretable = table_text.split('{')[0].strip() - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" +# # Extract and format table text +# #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" +# table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) +# table, table_size = convert_to_dict(table_only) +# table_formatted = format_table(table, table_size) - # Align - if text.count(pretable) == 2: - # Remove original table - aligned_text = text.replace(table_text, "") - # Align new table - aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - else: - aligned_text = text.replace(table_text, pretable + table_formatted) - except: - aligned_text = text - aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") - else: - aligned_text_dict[key] = text +# # Align +# if text.count(pretable) == 2: +# # Remove original table +# aligned_text = text.replace(table_text, "") +# # Align new table +# aligned_text = aligned_text.replace(pretable, pretable + table_formatted) +# else: +# aligned_text = text.replace(table_text, pretable + table_formatted) +# except: +# aligned_text = text +# aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") +# else: +# aligned_text_dict[key] = text - return aligned_text_dict +# return aligned_text_dict -contract_text = preprocess.clean_newlines(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = align_and_format_tables(text_dict) -text_dict = preprocess.highlight_rates(text_dict) +# contract_text = preprocess.clean_newlines(contract_text) +# text_dict = preprocess.split_text(contract_text) +# text_dict = align_and_format_tables(text_dict) +# text_dict = preprocess.highlight_rates(text_dict) -#print(text_dict['13']) +# #print(text_dict['13']) -# d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} -# prompt = BOTTOM_UP_CODES(d, text_dict['3']) +# # d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} +# # prompt = BOTTOM_UP_CODES(d, text_dict['3']) -# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +# # answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) -# print(answer) +# # print(answer) diff --git a/src/utils.py b/src/utils.py index e05e140..43a16b0 100644 --- a/src/utils.py +++ b/src/utils.py @@ -107,3 +107,31 @@ def format_td_check(td_dicts, dont_include_list): final_str += '\n' dict_count += 1 return final_str + + +def consolidate_csvs(output_dir, output_file): + os.makedirs(config.CONSOLIDATED_OUTPUT_DIRECTORY, exist_ok=True) + + df_list = [] + # Walk through each folder in the output directory + for root, dirs, files in os.walk(output_dir): + for dir_name in dirs: + dir_path = os.path.join(root, dir_name) + file_path = os.path.join(dir_path, config.PROCESSED_RESULTS_NAME) + if os.path.isfile(file_path): + try: + df = pd.read_csv(file_path) + df_list.append(df) + except: + pass + + concatenated_df = pd.concat(df_list, ignore_index=True) + concatenated_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False) + print(f"All CSV files have been consolidated into {output_file}") + + + + + + + From 84f59341b85f792346ee102b3bb03a0ba5fb5b5b Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 11:34:07 -0700 Subject: [PATCH 43/78] Top Down Primary modifications --- src/prompts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/prompts.py b/src/prompts.py index 2b65a07..00bdfe4 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -157,6 +157,7 @@ Ensure you ONLY return codes if they are associated directly with the Service li Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ + def TOP_DOWN_PRIMARY(page): return f"""### PAGE START ### {page} ### PAGE END @@ -165,7 +166,7 @@ The preceding text is one page of a contract between a Payer and a provider in t Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: 'CONTRACT_LOB' : List any Line of Business mentioned on the page. Choose ONLY from the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs. If none of these are found on the page, simply write 'N/A' for this answer. 'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS -'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS +'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer. 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list. From 3c05bebdf83334d1167e0f99c4c9583fb3c3cf5b Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 11:55:05 -0700 Subject: [PATCH 44/78] Fix move_large_numbers_to_flat_fee --- src/config.py | 2 +- src/postprocessingfuncs.py | 4 +- src/prompts.py | 6 +- src/test.py | 132 ++++++++++++------------------------- 4 files changed, 47 insertions(+), 97 deletions(-) diff --git a/src/config.py b/src/config.py index 9652321..5cf3485 100644 --- a/src/config.py +++ b/src/config.py @@ -50,7 +50,7 @@ AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" # File Paths -LOCAL_PATH = 'data/priority_health/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index ab94eb1..1ee41b4 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -32,7 +32,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): if match: return match return None - + df[new_column_name] = df[column_name].apply(update_column) cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes @@ -117,7 +117,7 @@ def move_percentage_to_rate(df): def move_large_numbers_to_flat_fee(df): def move_large_number(value): - if isinstance(value, str) and value.isdigit() and int(value) > 100: + if isinstance(value, str) and value.isdigit() and int(value) > 1000: return True return False diff --git a/src/prompts.py b/src/prompts.py index 00bdfe4..21b9dc2 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -72,8 +72,8 @@ Read and analyze the page, then populate a JSON dictionary with the following fi 'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. 'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. 'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence - only include the half of the lesser/greater of statement that is NOT Original_Methodology. For example, if the reimbursement is the lesser of X or Original_Methodology, return only X. -'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. If not, write N/A. -'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. If not, write N/A. +'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. Only include percentages. If not, write N/A. +'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. Only include dollar values. If not, write N/A. For REIMBURSEMENT_RATE and REIMBURSEMENT_FLAT_FEE, do NOT include the Rate or Flat Fee from Original_Methodology. @@ -120,7 +120,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour +Allowed Amount, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. diff --git a/src/test.py b/src/test.py index f5b7790..c37fae6 100644 --- a/src/test.py +++ b/src/test.py @@ -13,97 +13,47 @@ import prompt_funcs import claude_funcs -if config.WRITE_OUTPUT: - utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) - - - -# input_dict = utils.read_input() - -# filename = list(input_dict.keys())[0] -# contract_text = input_dict[filename] - -# def convert_to_dict(table_text): -# table_text = table_text.strip('{}') -# table_text_list = table_text.split(']') -# table_text_list = [item for item in table_text_list if len(item) > 0] - -# final_dict = {} -# num_elements = 0 -# for key_value_text in table_text_list: -# key = key_value_text.split(':')[0].strip(', \'') -# value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') - -# f = StringIO(value) -# reader = csv.reader(f, delimiter=',') - -# value_list = [] -# for row in reader: -# for r in row: -# value_list.append(r) - -# final_dict[key] = value_list -# num_elements = len(value_list) -# return final_dict, num_elements - - -# def format_table(table_json, table_size): -# table_text = "" -# for i in range(table_size): -# for key in table_json.keys(): -# if table_json[key][i]: -# #table_text += key + ': ' + table_json[key][i] + ', ' -# table_text += table_json[key][i] + ' : ' -# table_text += r'\n' -# return table_text - - -# def align_and_format_tables(text_dict): -# aligned_text_dict = {} -# for key, text in text_dict.items(): -# if 'Table Start' in text: -# table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) -# for table_text in table_texts: -# try: -# # Extract pretable text -# pretable = table_text.split('{')[0].strip() - -# # Extract and format table text -# #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" -# table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - -# table, table_size = convert_to_dict(table_only) -# table_formatted = format_table(table, table_size) - -# # Align -# if text.count(pretable) == 2: -# # Remove original table -# aligned_text = text.replace(table_text, "") -# # Align new table -# aligned_text = aligned_text.replace(pretable, pretable + table_formatted) -# else: -# aligned_text = text.replace(table_text, pretable + table_formatted) -# except: -# aligned_text = text -# aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") -# else: -# aligned_text_dict[key] = text - -# return aligned_text_dict - -# contract_text = preprocess.clean_newlines(contract_text) -# text_dict = preprocess.split_text(contract_text) -# text_dict = align_and_format_tables(text_dict) -# text_dict = preprocess.highlight_rates(text_dict) - -# #print(text_dict['13']) - -# # d = {'SERVICE' : 'Personal Care Services', 'FULL_METHODOLOGY' : 'Per 15 min (group), Health Plan fee schedule: $3.00 per unit (1 unit = 15 minutes)'} -# # prompt = BOTTOM_UP_CODES(d, text_dict['3']) - -# # answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - -# # print(answer) +def BOTTOM_UP_PRIMARY(page, payer): + return f""" +### PAGE START ### {page} ### PAGE END + +The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. + +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). + +If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. + +Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. + +Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: +'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. +'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. +'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. +'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. + +Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. + +Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + +input_dict = utils.read_input() + +filename = list(input_dict.keys())[0] +contract_text = input_dict[filename] +print(filename) + +contract_text = preprocess.clean_newlines(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) + +print(text_dict['2']) + +# prompt = BOTTOM_UP_PRIMARY(text_dict['2'], '') + +# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + +# print(answer) From be11899de199ebb7113335a9be4fc9a0312ba4a4 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 13:04:05 -0700 Subject: [PATCH 45/78] Updated table preprocessing - fixed multiple table issue --- src/table_funcs.py | 32 ++++++++----------- src/test.py | 78 +++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/src/table_funcs.py b/src/table_funcs.py index 0d62314..a438cac 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -22,7 +22,6 @@ def extract_all_tables(input_dict): tables_dict[filename] = extract_tables(filetext) return tables_dict - def convert_to_dict(table_text): table_text = table_text.strip('{}') table_text_list = table_text.split(']') @@ -32,15 +31,9 @@ def convert_to_dict(table_text): num_elements = 0 for key_value_text in table_text_list: key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip(' [] \'') + value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' - f = StringIO(value) - reader = csv.reader(f, delimiter=',') - - value_list = [] - for row in reader: - for r in row: - value_list.append(r) + value_list = ast.literal_eval(value) final_dict[key] = value_list num_elements = len(value_list) @@ -62,6 +55,7 @@ def format_table(table_json, table_size): def align_and_format_tables(text_dict): aligned_text_dict = {} for key, text in text_dict.items(): + aligned_text = text if 'Table Start' in text: table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) for table_text in table_texts: @@ -75,17 +69,17 @@ def align_and_format_tables(text_dict): table, table_size = convert_to_dict(table_only) table_formatted = format_table(table, table_size) - - # Align - if text.count(pretable) == 2: - # Remove original table - aligned_text = text.replace(table_text, "") - # Align new table - aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - else: - aligned_text = text.replace(table_text, pretable + table_formatted) except: - aligned_text = text + table_formatted = table_text + + # Align + if text.count(pretable) == 2: # One table + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: + aligned_text = aligned_text.replace(table_text, pretable + table_formatted) aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") else: aligned_text_dict[key] = text diff --git a/src/test.py b/src/test.py index c37fae6..6288640 100644 --- a/src/test.py +++ b/src/test.py @@ -4,6 +4,7 @@ import json import csv from io import StringIO import config +import ast import utils import preprocess @@ -42,18 +43,79 @@ filename = list(input_dict.keys())[0] contract_text = input_dict[filename] print(filename) +def convert_to_dict(table_text): + table_text = table_text.strip('{}') + table_text_list = table_text.split(']') + table_text_list = [item for item in table_text_list if len(item) > 0] + + final_dict = {} + num_elements = 0 + for key_value_text in table_text_list: + key = key_value_text.split(':')[0].strip(', \'') + value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' + + value_list = ast.literal_eval(value) + + final_dict[key] = value_list + num_elements = len(value_list) + return final_dict, num_elements + + +def format_table(table_json, table_size): + table_text = "" + keys = [key for key in table_json.keys()] + table_text += ' : '.join(keys) + "\n" + for i in range(table_size): + for key in keys: + if table_json[key][i]: + #table_text += key + ': ' + table_json[key][i] + ', ' + table_text += table_json[key][i] + ' : ' + table_text += r'\n' + return table_text + +def align_and_format_tables(text_dict): + aligned_text_dict = {} + for key, text in text_dict.items(): + aligned_text = text + if 'Table Start' in text: + table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) + for table_text in table_texts: + try: + # Extract pretable text + pretable = table_text.split('{')[0].strip() + + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + except: + table_formatted = table_text + + # Align + if text.count(pretable) == 2: # One table + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: + aligned_text = aligned_text.replace(table_text, pretable + table_formatted) + aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") + else: + aligned_text_dict[key] = text + + return aligned_text_dict + + contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = align_and_format_tables({'25' : text_dict['25']}) text_dict = preprocess.highlight_rates(text_dict) -print(text_dict['2']) - -# prompt = BOTTOM_UP_PRIMARY(text_dict['2'], '') - -# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - -# print(answer) +prompt = BOTTOM_UP_PRIMARY(text_dict['25'], '') +answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +print(answer) From 4f531cf8bdb51e587f0ded1ed465f1d8730355f8 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 13:26:28 -0700 Subject: [PATCH 46/78] Bottom Up Primary optimization --- src/prompts.py | 2 +- src/test.py | 70 +++----------------------------------------------- 2 files changed, 4 insertions(+), 68 deletions(-) diff --git a/src/prompts.py b/src/prompts.py index 21b9dc2..9fd5bbe 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -45,7 +45,7 @@ If any of the attributes are not found, return N/A. For all attributes, only wri Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: -'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. +'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A. 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. diff --git a/src/test.py b/src/test.py index 6288640..30586ed 100644 --- a/src/test.py +++ b/src/test.py @@ -27,7 +27,7 @@ If any of the attributes are not found, return N/A. For all attributes, only wri Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: -'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Do not leave this field N/A. +'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A. 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. @@ -43,77 +43,13 @@ filename = list(input_dict.keys())[0] contract_text = input_dict[filename] print(filename) -def convert_to_dict(table_text): - table_text = table_text.strip('{}') - table_text_list = table_text.split(']') - table_text_list = [item for item in table_text_list if len(item) > 0] - - final_dict = {} - num_elements = 0 - for key_value_text in table_text_list: - key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' - - value_list = ast.literal_eval(value) - - final_dict[key] = value_list - num_elements = len(value_list) - return final_dict, num_elements - - -def format_table(table_json, table_size): - table_text = "" - keys = [key for key in table_json.keys()] - table_text += ' : '.join(keys) + "\n" - for i in range(table_size): - for key in keys: - if table_json[key][i]: - #table_text += key + ': ' + table_json[key][i] + ', ' - table_text += table_json[key][i] + ' : ' - table_text += r'\n' - return table_text - -def align_and_format_tables(text_dict): - aligned_text_dict = {} - for key, text in text_dict.items(): - aligned_text = text - if 'Table Start' in text: - table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) - for table_text in table_texts: - try: - # Extract pretable text - pretable = table_text.split('{')[0].strip() - - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) - except: - table_formatted = table_text - - # Align - if text.count(pretable) == 2: # One table - # Remove original table - aligned_text = text.replace(table_text, "") - # Align new table - aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - else: - aligned_text = aligned_text.replace(table_text, pretable + table_formatted) - aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") - else: - aligned_text_dict[key] = text - - return aligned_text_dict - contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) -text_dict = align_and_format_tables({'25' : text_dict['25']}) +text_dict = table_funcs.align_and_format_tables({'2' : text_dict['2']}) text_dict = preprocess.highlight_rates(text_dict) -prompt = BOTTOM_UP_PRIMARY(text_dict['25'], '') +prompt = BOTTOM_UP_PRIMARY(text_dict['2'], '') answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) print(answer) From c2e5f3536fd7c7ac8bfe5a8bb6f8f1a2239a4d07 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 13:47:27 -0700 Subject: [PATCH 47/78] Bottom Up Methodology fix --- src/prompts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prompts.py b/src/prompts.py index 9fd5bbe..a013866 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -120,7 +120,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. From d767f55a0c7634a1b6c6dbc7579c0d2d538ca4ba Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 14:17:31 -0700 Subject: [PATCH 48/78] Updated Bottom Up Methodology --- src/prompts.py | 2 +- src/test.py | 27 ++++++++++++++++++++++----- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/prompts.py b/src/prompts.py index a013866..8c77aca 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -120,7 +120,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. diff --git a/src/test.py b/src/test.py index 30586ed..463f032 100644 --- a/src/test.py +++ b/src/test.py @@ -37,6 +37,23 @@ Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be pop Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ +def BOTTOM_UP_METHODOLOGY(d): + return f"""Analyze the full methodology text listed below: +{d['FULL_METHODOLOGY']} + +Your job is to identify what the concise methodology is based on the full methodology. + +Choose the closest option from the following: +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour + +For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. +For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. + +Return only the answer, without using complete sentences. If there is nothing even close to the above options in the full methodology text, simply return 'N/A' +""" + + + input_dict = utils.read_input() filename = list(input_dict.keys())[0] @@ -44,12 +61,12 @@ contract_text = input_dict[filename] print(filename) -contract_text = preprocess.clean_newlines(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables({'2' : text_dict['2']}) -text_dict = preprocess.highlight_rates(text_dict) +# contract_text = preprocess.clean_newlines(contract_text) +# text_dict = preprocess.split_text(contract_text) +# text_dict = table_funcs.align_and_format_tables({'7' : text_dict['5']}) +# text_dict = preprocess.highlight_rates(text_dict) -prompt = BOTTOM_UP_PRIMARY(text_dict['2'], '') +prompt = BOTTOM_UP_METHODOLOGY({'FULL_METHODOLOGY' : 'In addition, Plan will reimburse the Provider 100% of the IME/DME amount.'}) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) print(answer) From 160dfb71a1bc967bb6585c14f27372df3f0adabe Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 16:03:25 -0700 Subject: [PATCH 49/78] Fixed Post processing issues -faizan --- src/config.py | 8 +-- src/main.py | 2 +- src/postprocess.py | 11 ++++ src/postprocessingfuncs.py | 23 ++++++- src/prompts.py | 6 +- src/table_funcs.py | 15 ----- src/test.py | 124 ++++++++++++++++++++++++++++++++++--- 7 files changed, 156 insertions(+), 33 deletions(-) diff --git a/src/config.py b/src/config.py index 5cf3485..6774683 100644 --- a/src/config.py +++ b/src/config.py @@ -22,7 +22,7 @@ READ_MODE = '_LOCAL_' # OR '_S3_' # Output Settings WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written OUTPUT_DIRECTORY = 'output' -CONSOLIDATED_OUTPUT_DIRECTORY = 'consolidated_output' +CONSOLIDATED_OUTPUT_DIRECTORY = 'output_consolidated' OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.csv' TD_RESULTS_NAME = 'td_results.csv' BU_RESULTS_NAME = 'bu_results.csv' @@ -45,9 +45,9 @@ RUN_CODES = True FUZZY_MATCH_THRESHOLD = 0.8 # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXAI7RYNWW" -AWS_SECRET_ACCESS_KEY="fPm3Rn3cde8AOVj0hP/n3u1MwSrQU17qxDPRK7ar" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGYaCXVzLWVhc3QtMiJHMEUCIEFjYLb1fysSczLo2Ae7zUOD61H0YhfYPtrzlwXGw6TxAiEAnPVt3UX0UGGxJrAS3T14mVud26zn8i2qIbgjUZAkQmIqjAMI7///////////ARAAGgw2NjAxMzEwNjg3ODIiDH2L5zRJcBcGAXDxMSrgAhoB9QUZFydQiXFS7YUNrYBwSRX3fxPM6dml/WETf9qesJ7tkuOlXJIa7ZZLjJreuMCRfS4eUis0mMDZg+8pb1lLYQAAwuCe6JHN757NH7lwOXSdLY/pzqHc+w4atcGfKNsexj6BIGCyOKbqjbA0wcMgz1y5DqEY8lBeU+E/FTMBa6r9SKYH2hyD41KZQrtUAbKeOWrw2lMjxsr4TzlkdHXw8uQJC4WY7uC6iAzZoR+WDQvfOFu1ltzeJlMiJZA74V1pStK0s/f+QtlwmT6KLu8gVafepohAFs4tuWlz5Ohmn0T/hGQURD8Em1dU22Ljj0qfcGejEUFw8HQDI5kYtwFBDU4XLnDh5Z01Dyxidted8xV3DNI6Ok79cAT950jXXSipBuHKxNv9qMAbrgrAir5GUPhs04XnocjIw7FpOoYzglURU+Y/R+wHeOtXRou4VcDLdoOtlrnrKX9/WR5eOI8woZ+MswY6pgG+tl5kEnjwJJDMCErNLFBnAmBHaC+0oBFRPOaX7ZYHY7ryIfo21kqLEkgXD54XY6xRIyfpEMjBIled+AUOJC17BEjZUq3tUUMi1FEo4KFwANQntvQVDHQ+nwmtNQ4NWeCBfBOqqEArsXjG3SROTQOBKr75NHlPFYtsTr2vK03xCe6dVF/Uq0giyXvoEMyT9eCJTFW1Bzs9HUpUhG9h0lgO2q4LKaLd" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXEBLXGWPZ" +AWS_SECRET_ACCESS_KEY="kysOCZMQMY6LhAaG8yOqePxipD7E/xCgQ65PAyru" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEG4aCXVzLWVhc3QtMiJIMEYCIQDTt+AU18DgfdW0xumQTZD1hiyQcGBidA1w6LbxO2plhwIhAO5H86tzKKhVyVM8TJBjhyNL4VLysGd4A5wXkXb42AXdKowDCPj//////////wEQABoMNjYwMTMxMDY4NzgyIgy5MpnRnarWqYZ+d4Yq4AKKqJCRvzV6qgoUnCtNahiK7oiSHd6TpZghh4g87TVkIRUdzAirtYH86gfzlUfZsBgHRlq2w5DyRlEe88dCuUVoUd8XFRREqO2/DkPEMJzfs87EOc4wu6wu8IJ1BRIl7ALxtVUVgR0sXIpiHhS+0NDF6i/kebau7chpAZiHottOl2VO4h/+rKaAI2HahwzbgY5nuckb0kYLA1H7Er5OfX7UrbON4nC9RN7LRySJZc8YZU+TLkLRU0wzpoVxt+WTDkIARM8bSAf0aM6jYHzETQG3VtGHidWZROCj597sseSRc+Z2fypY/I/hzpXMFNF4LQvsMFtAvCnfbZDysWzAfimYb78zapwlAptPdhVtcqkEgKUYKBKozbX0JWZIFur/8gYUQALaRRK9A+vViUw1ihQGNAhxBkj2n+/oO6/dA1z3kwgQg01mNJHjUFOSi0Vobx9Iu8HfahhJeaJ3XjvFyzT3MLqRjrMGOqUBj9J316RGsZ3UfiHqGl72V81Bg/kGms5o/wz+pkec7vNFTRPbvgTtnNMX9aJ/HD73O5UvBUqLwYylJKzH0vkE+2KEgOR4SMyNuOEjHD4BcK0aZBY8/rpb+LkvxLfo8SJ3nEfStyyoLm7I6xQuhnWfMhzLvffQRWAjJlJMtck4w/tIYYf1QhFTPFndfTYIQVyN28zF17oQpBdF0BjGWbdTg7dECr9H" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/main.py b/src/main.py index 4912bfa..3226704 100644 --- a/src/main.py +++ b/src/main.py @@ -43,7 +43,7 @@ def main(): # Write Consolidated Output if config.WRITE_OUTPUT: - utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) + utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) if __name__ == "__main__": diff --git a/src/postprocess.py b/src/postprocess.py index e822a0f..2ca1070 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -124,12 +124,23 @@ def post_process_combined(df): except Exception as e: print(f"Error: {e}") + try: + df = postprocessingfuncs.set_rate_to_zero_if_not_covered(df) + except Exception as e: + print(f"Error: {e}") + + try: + df = postprocessingfuncs.adjust_reimbursement_rate(df) + except Exception as e: + print(f"Error: {e}") + try: clean_df = postprocessingfuncs.clean_pagenumbers(df) except Exception as e: print(f"Error: {e}") clean_df = df + return clean_df def postprocess_results(combined_df): diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 1ee41b4..e9caab8 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -32,7 +32,7 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): if match: return match return None - + df[new_column_name] = df[column_name].apply(update_column) cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes @@ -117,10 +117,29 @@ def move_percentage_to_rate(df): def move_large_numbers_to_flat_fee(df): def move_large_number(value): - if isinstance(value, str) and value.isdigit() and int(value) > 1000: + if isinstance(value, str) and value.isdigit() and int(value) > 100: return True return False df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: row['REIMBURSEMENT_RATE'] if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) df['REIMBURSEMENT_RATE'] = df.apply(lambda row: None if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_RATE'], axis=1) return df + +def set_rate_to_zero_if_not_covered(df): + def check_and_set_rate(row): + if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE): + return 0 + return row['REIMBURSEMENT_RATE'] + + df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1) + return df + +def adjust_reimbursement_rate(df): + def adjust_rate(row): + if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE): + if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)): + return 100 - row['REIMBURSEMENT_RATE'] + return row['REIMBURSEMENT_RATE'] + + df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1) + return df diff --git a/src/prompts.py b/src/prompts.py index 8c77aca..7b0934f 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -120,7 +120,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. @@ -165,8 +165,8 @@ The preceding text is one page of a contract between a Payer and a provider in t Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: 'CONTRACT_LOB' : List any Line of Business mentioned on the page. Choose ONLY from the following: Commercial, Group, Marketplace, Marketplace/Exchange, Medicaid, Medicaid-Medicare, Medicare, Medicare Advantage, Medicare Supplemental, Veterans Affairs. If none of these are found on the page, simply write 'N/A' for this answer. -'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS -'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer. +'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS. Do your best to infer the program from this page. +'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. Do your best to infer the network from this page. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer. 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list. diff --git a/src/table_funcs.py b/src/table_funcs.py index a438cac..8d0fbc4 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -6,21 +6,6 @@ import ast import csv from io import StringIO -def extract_tables(text): - pattern = re.compile(r"-------Table Start--------(.*?)-------Table End--------", re.DOTALL) - tables = pattern.findall(text) - table_list = [table.strip() for table in tables] - table_list = ['{'+table.split('{')[1] for table in table_list] - df_list = [pd.DataFrame(ast.literal_eval(table)) for table in table_list] - for df in df_list: - print(df.columns) - return df_list - -def extract_all_tables(input_dict): - tables_dict = {} - for filename, filetext in input_dict.items(): - tables_dict[filename] = extract_tables(filetext) - return tables_dict def convert_to_dict(table_text): table_text = table_text.strip('{}') diff --git a/src/test.py b/src/test.py index 463f032..7c3df14 100644 --- a/src/test.py +++ b/src/test.py @@ -37,6 +37,34 @@ Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be pop Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ +def BOTTOM_UP_CODES(d, page): + return f"""### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Service: {d['SERVICE']} +Methodology: {d['FULL_METHODOLOGY']} + +Your job is to identify codes associated with this specific term. + +Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided): +'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility. +'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes. +'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar. +'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options. +'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service. +'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures. +'REIMBURSEMENT_PROC_CODE_MODIFIER' : 2-digit alphanumeric codes accompanying a PROC code. +'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes. +'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status. + +Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. + +Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. + +Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + def BOTTOM_UP_METHODOLOGY(d): return f"""Analyze the full methodology text listed below: {d['FULL_METHODOLOGY']} @@ -61,14 +89,94 @@ contract_text = input_dict[filename] print(filename) -# contract_text = preprocess.clean_newlines(contract_text) -# text_dict = preprocess.split_text(contract_text) -# text_dict = table_funcs.align_and_format_tables({'7' : text_dict['5']}) -# text_dict = preprocess.highlight_rates(text_dict) - -prompt = BOTTOM_UP_METHODOLOGY({'FULL_METHODOLOGY' : 'In addition, Plan will reimburse the Provider 100% of the IME/DME amount.'}) -answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) -print(answer) + + + +def convert_to_dict(table_text): + table_text = table_text.strip('{}') + table_text_list = table_text.split(']') + table_text_list = [item for item in table_text_list if len(item) > 0] + + final_dict = {} + num_elements = 0 + for key_value_text in table_text_list: + key = key_value_text.split(':')[0].strip(', \'') + value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' + + value_list = ast.literal_eval(value) + + final_dict[key] = value_list + num_elements = len(value_list) + return final_dict, num_elements + + +def format_table(table_json, table_size): + table_text = "" + keys = [key for key in table_json.keys()] + table_text += ' : '.join(keys) + "\n" + for i in range(table_size): + for key in keys: + if table_json[key][i]: + #table_text += key + ': ' + table_json[key][i] + ', ' + table_text += table_json[key][i] + ' : ' + table_text += r'\n' + return table_text + +def align_and_format_tables(text_dict): + aligned_text_dict = {} + for key, text in text_dict.items(): + aligned_text = text + if 'Table Start' in text: + table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) + for table_text in table_texts: + try: + # Extract pretable text + pretable = table_text.split('{')[0].strip() + + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + print(table_formatted) + except: + table_formatted = table_text + + + # Align + if text.count(pretable) == 2: # One table + print('One Table') + # Remove original table + aligned_text = text.replace(table_text, "") + # Align new table + aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + print(aligned_text) + else: + aligned_text = aligned_text.replace(table_text, pretable + table_formatted) + aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") + else: + aligned_text_dict[key] = text + + return aligned_text_dict + + + + + +contract_text = preprocess.clean_newlines(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = align_and_format_tables({'5' : text_dict['5']}) +text_dict = preprocess.highlight_rates(text_dict) + +#print(text_dict['5']) + + +#prompt = BOTTOM_UP_CODES({'SERVICE' : 'Psychiatric Diagnostic Evaluation', 'FULL_METHODOLOGY' : 'N/A' }, text_dict['5']) +#print(prompt) + +# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +# print(answer) From 3208b4b8958125d7c914ec61b4557117a66da9df Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 7 Jun 2024 16:12:03 -0700 Subject: [PATCH 50/78] Table func bug fix --- src/table_funcs.py | 7 ++----- src/test.py | 24 +++++------------------- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/table_funcs.py b/src/table_funcs.py index 8d0fbc4..9ca3f31 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -6,7 +6,6 @@ import ast import csv from io import StringIO - def convert_to_dict(table_text): table_text = table_text.strip('{}') table_text_list = table_text.split(']') @@ -16,10 +15,8 @@ def convert_to_dict(table_text): num_elements = 0 for key_value_text in table_text_list: key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' - + value = ':'.join(key_value_text.split(':')[1:]).strip()+']' value_list = ast.literal_eval(value) - final_dict[key] = value_list num_elements = len(value_list) return final_dict, num_elements @@ -47,7 +44,6 @@ def align_and_format_tables(text_dict): try: # Extract pretable text pretable = table_text.split('{')[0].strip() - # Extract and format table text #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" @@ -63,6 +59,7 @@ def align_and_format_tables(text_dict): aligned_text = text.replace(table_text, "") # Align new table aligned_text = aligned_text.replace(pretable, pretable + table_formatted) + else: aligned_text = aligned_text.replace(table_text, pretable + table_formatted) aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") diff --git a/src/test.py b/src/test.py index 7c3df14..6215e48 100644 --- a/src/test.py +++ b/src/test.py @@ -101,10 +101,8 @@ def convert_to_dict(table_text): num_elements = 0 for key_value_text in table_text_list: key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip(' \'')+'\']' - + value = ':'.join(key_value_text.split(':')[1:]).strip()+']' value_list = ast.literal_eval(value) - final_dict[key] = value_list num_elements = len(value_list) return final_dict, num_elements @@ -132,26 +130,22 @@ def align_and_format_tables(text_dict): try: # Extract pretable text pretable = table_text.split('{')[0].strip() - # Extract and format table text #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" table, table_size = convert_to_dict(table_only) table_formatted = format_table(table, table_size) - print(table_formatted) except: table_formatted = table_text - # Align if text.count(pretable) == 2: # One table - print('One Table') # Remove original table aligned_text = text.replace(table_text, "") # Align new table aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - print(aligned_text) + else: aligned_text = aligned_text.replace(table_text, pretable + table_formatted) aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") @@ -160,23 +154,15 @@ def align_and_format_tables(text_dict): return aligned_text_dict - - - - contract_text = preprocess.clean_newlines(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = align_and_format_tables({'5' : text_dict['5']}) text_dict = preprocess.highlight_rates(text_dict) -#print(text_dict['5']) +prompt = BOTTOM_UP_CODES({'SERVICE' : 'Psychiatric Diagnostic Evaluation', 'FULL_METHODOLOGY' : 'N/A' }, text_dict['5']) - -#prompt = BOTTOM_UP_CODES({'SERVICE' : 'Psychiatric Diagnostic Evaluation', 'FULL_METHODOLOGY' : 'N/A' }, text_dict['5']) -#print(prompt) - -# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) -# print(answer) +answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +print(answer) From ef6c4ddbcae90536a16ded1875e5cc0ef0f11df0 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 10 Jun 2024 13:35:37 -0700 Subject: [PATCH 51/78] AWP handingling in Methodology prompt --- src/config.py | 12 ++++++------ src/prompts.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/config.py b/src/config.py index 6774683..8b211cd 100644 --- a/src/config.py +++ b/src/config.py @@ -8,7 +8,7 @@ from llama_index.llms.bedrock import Bedrock # General Settings TEST = False # True to run test prompt - just for testing model connection VERBOSE = True -CLIENT_NAME = 'Priority Health' +CLIENT_NAME = '' TODAY = datetime.now().strftime("%Y%m%d") # Valid values @@ -30,7 +30,7 @@ UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings -MAX_WORKERS = 20 +MAX_WORKERS = 5 # Prompt Debugging RUN_PRIMARY = True # Always True @@ -45,12 +45,12 @@ RUN_CODES = True FUZZY_MATCH_THRESHOLD = 0.8 # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXEBLXGWPZ" -AWS_SECRET_ACCESS_KEY="kysOCZMQMY6LhAaG8yOqePxipD7E/xCgQ65PAyru" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEG4aCXVzLWVhc3QtMiJIMEYCIQDTt+AU18DgfdW0xumQTZD1hiyQcGBidA1w6LbxO2plhwIhAO5H86tzKKhVyVM8TJBjhyNL4VLysGd4A5wXkXb42AXdKowDCPj//////////wEQABoMNjYwMTMxMDY4NzgyIgy5MpnRnarWqYZ+d4Yq4AKKqJCRvzV6qgoUnCtNahiK7oiSHd6TpZghh4g87TVkIRUdzAirtYH86gfzlUfZsBgHRlq2w5DyRlEe88dCuUVoUd8XFRREqO2/DkPEMJzfs87EOc4wu6wu8IJ1BRIl7ALxtVUVgR0sXIpiHhS+0NDF6i/kebau7chpAZiHottOl2VO4h/+rKaAI2HahwzbgY5nuckb0kYLA1H7Er5OfX7UrbON4nC9RN7LRySJZc8YZU+TLkLRU0wzpoVxt+WTDkIARM8bSAf0aM6jYHzETQG3VtGHidWZROCj597sseSRc+Z2fypY/I/hzpXMFNF4LQvsMFtAvCnfbZDysWzAfimYb78zapwlAptPdhVtcqkEgKUYKBKozbX0JWZIFur/8gYUQALaRRK9A+vViUw1ihQGNAhxBkj2n+/oO6/dA1z3kwgQg01mNJHjUFOSi0Vobx9Iu8HfahhJeaJ3XjvFyzT3MLqRjrMGOqUBj9J316RGsZ3UfiHqGl72V81Bg/kGms5o/wz+pkec7vNFTRPbvgTtnNMX9aJ/HD73O5UvBUqLwYylJKzH0vkE+2KEgOR4SMyNuOEjHD4BcK0aZBY8/rpb+LkvxLfo8SJ3nEfStyyoLm7I6xQuhnWfMhzLvffQRWAjJlJMtck4w/tIYYf1QhFTPFndfTYIQVyN28zF17oQpBdF0BjGWbdTg7dECr9H" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXGPEZJ5OW" +AWS_SECRET_ACCESS_KEY="PelB1A6KB6HnbBrdUeLgnf0Ii+dMpXCG5JLs4BUS" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEKz//////////wEaCXVzLWVhc3QtMiJIMEYCIQCxTs5XXRBybkfcptcaSQUN+2JvUa/dimXyxX31+CQeeAIhAL3NRuwIU+MxkVQBDon2BbGTZawkwF4NVvdcUEElGtxEKoMDCEUQABoMNjYwMTMxMDY4NzgyIgwLOdI9PpnuoefEZTMq4AJqQn9q4Uq3WQEK6Wg2iQNPaPTLbaBKLx8TtWwlS6wcIlhFNA+nPyaxew3boex97eYM4uzHiwU2MQ4OTbW+aOphdsWACU4jlaTM6Z0VWlJ+W+OKpMwajDIj7SQnfrDQSGwJ9Ewth3+uxBKqPtXXZmUe5WRCz7G4cIjYxEqdjtj0ZO6ll6zZozPEGVik1+DZoPMbh4Y4qczMDdpW3xXUlalYJBu6CaNqGIb6O1ocl4nhLSmamsKoiXLqGU853Loq0A3b+u+J7w48iPLhXWnuN0GXuN7gBc2C1lzPs/aABM2bU3LBlDqGxKGUDVPRHttg8HRNGwIkEr9LieGjheKvcvLQroi6DhTRkKnV6WEQ1wGzN9wCteXSL7DIktSsddnwX3ffvg7G5WhPN0xYc4F78Sw9gan50wmmebiHrc5lP+yQDJ7J6gvo4a3NsxCE5opV/hgJUjPUgJ7tCPZ+bf+kqKJqMNLem7MGOqUBjmMAY7Vvqp4pBdORFWTZkOHcmZ1UsmJmNYrS69btqyYQbwg79sCIaVR/wiJDn9T5v9vziyv0n98XD7dV6UEuyW97EKv/OWwnqZ7zJwHAa4vwJ09zCruIgT4hCctpzDAAfTYtkacb0PRrxa3pjQZy7pXQC0BvliocPY5fvR5f5QOa/mZGwhjUcih9LfvNWj0BsRW65pPcJoy6Wy3G0EgwsH+VW1h5" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/all_txt_updated/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/prompts.py b/src/prompts.py index 7b0934f..63dcb93 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -107,7 +107,7 @@ Methodology: {d['FULL_METHODOLOGY']} Your job is to identify if the reimbursement is based on a Fee Schedule. Read and analyze the Methodology, then populate a JSON dictionary with the following fields (descriptions provided): -REIMBURSEMENT_FEE_SCHEDULE : If the Methodology is based on a Fee Schedule, what is the name of the fee schedule on which it is based? It could be something like Medicaid, Medicare, CMS, DRG, or similar. Do not answer with a complete sentence. If the Methodology is not based on a Fee Schedule, just return 'N/A' +REIMBURSEMENT_FEE_SCHEDULE : If the Methodology is based on a Fee Schedule, what is the name of the fee schedule on which it is based? It could be something like Medicaid, Medicare, CMS, DRG, AWP, ASP, or similar. Do not answer with a complete sentence. If the Methodology is not based on a Fee Schedule, just return 'N/A' REIMBURSEMENT_FEE_SCHEDULE_VERSION : If the Methodology is based on a Fee Schedule, what specific version of the Fee Schedule is it based on. It could be something like 'Current', 'Prevailing', 'at the date of service', a specific year, or similar. If there is no answer, return 'N/A' Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. @@ -120,7 +120,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. From 01d171fe6667b9a8656b5bd1d7205795186663d7 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 11 Jun 2024 13:07:32 -0700 Subject: [PATCH 52/78] Updated requirements and imports --- src/carveouts.py | 1 + src/config.py | 16 ++-- src/main.py | 2 - src/postprocess.py | 3 +- src/postprocessingfuncs.py | 1 + src/prompt_funcs.py | 10 --- src/table_funcs.py | 4 - src/test.py | 164 +++---------------------------------- 8 files changed, 24 insertions(+), 177 deletions(-) diff --git a/src/carveouts.py b/src/carveouts.py index 18a2bc0..13843ad 100644 --- a/src/carveouts.py +++ b/src/carveouts.py @@ -2,6 +2,7 @@ import pandas as pd from difflib import SequenceMatcher import re import time + import prompts import claude_funcs diff --git a/src/config.py b/src/config.py index 8b211cd..5db760c 100644 --- a/src/config.py +++ b/src/config.py @@ -2,7 +2,7 @@ from datetime import datetime import boto3 -from llama_index.llms.bedrock import Bedrock +#from llama_index.llms.bedrock import Bedrock #pip install llama-index-llms-bedrock # General Settings @@ -74,13 +74,13 @@ MODEL_ID_CLAUDE3_SONNET = 'anthropic.claude-3-sonnet-20240229-v1:0' MODEL_ID_CLAUDE2 = 'anthropic.claude-instant-v1' # LLama Model -LLAMA_MODEL = Bedrock( - model=MODEL_ID_CLAUDE3_HAIKU, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - aws_session_token=AWS_SESSION_TOKEN, - region_name="us-east-1" - ) +# LLAMA_MODEL = Bedrock( + # model=MODEL_ID_CLAUDE3_HAIKU, + # aws_access_key_id=AWS_ACCESS_KEY_ID, + # aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + # aws_session_token=AWS_SESSION_TOKEN, + # region_name="us-east-1" + # ) diff --git a/src/main.py b/src/main.py index 3226704..5eca37c 100644 --- a/src/main.py +++ b/src/main.py @@ -1,12 +1,10 @@ # Imports -import os import concurrent.futures import traceback import utils import config -import prompt_funcs import claude_funcs import file_processing diff --git a/src/postprocess.py b/src/postprocess.py index 2ca1070..a0677a1 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,7 +1,8 @@ import pandas as pd -import config import re + import postprocessingfuncs +import config def sanitize_combined(df): for column in df.columns: diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index e9caab8..ce3ab00 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -1,3 +1,4 @@ + import pandas as pd import re import difflib diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index f6b4890..7cd0150 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -1,20 +1,10 @@ - -import os -import re -import pandas as pd import json -from llama_index.core import VectorStoreIndex, SimpleDirectoryReader, ServiceContext, PromptTemplate -from llama_index.core import SummaryIndex, Document -from llama_index.llms.bedrock import Bedrock -#pip install llama-index-llms-bedrock import config import prompts import claude_funcs -import preprocess import dict_operations -import table_funcs import utils diff --git a/src/table_funcs.py b/src/table_funcs.py index 9ca3f31..e02f144 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -1,10 +1,6 @@ import re -import pandas as pd -import json import ast -import csv -from io import StringIO def convert_to_dict(table_text): table_text = table_text.strip('{}') diff --git a/src/test.py b/src/test.py index 6215e48..1a2205a 100644 --- a/src/test.py +++ b/src/test.py @@ -5,6 +5,7 @@ import csv from io import StringIO import config import ast +import os import utils import preprocess @@ -14,155 +15,14 @@ import prompt_funcs import claude_funcs -def BOTTOM_UP_PRIMARY(page, payer): - return f""" -### PAGE START ### {page} ### PAGE END - -The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. - -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). - -If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. - -Return at least one json object for each combination of attributes seen. It is possible that a page will not have any attributes. When this is the case, return an empty list. - -Here are the attributes to be included in each dictionary, and instructions on how to correctly answer: -'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A. -'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. -'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. -'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. - -Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. - -Only return the list, with no other commentary or explanation. Ensure you abide by proper JSON formatting. -""" - -def BOTTOM_UP_CODES(d, page): - return f"""### PAGE START ### {page} ### PAGE END - -The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: -Service: {d['SERVICE']} -Methodology: {d['FULL_METHODOLOGY']} - -Your job is to identify codes associated with this specific term. - -Read and analyze the page, then populate a JSON dictionary with the following code types if they are associated with the given Service and Methodology (descriptions provided): -'REIMBURSEMENT_ADMITTYPE_CODES' : Single-digit codes that indicate the type of admission to a facility. -'REIMBURSEMENT_DIAG_CODES' : Multi-digit alphanumeric codes related to diagnosis. They may contain a '.' after the first 3 characters. These may be labelled as ICD-10 or ICD-10-CM codes. -'REIMBURSEMENT_GROUPER_CODES' : 3- or 4-digit codes that may be labelled DRG or or similar. -'REIMBURSEMENT_GROUPER' : If there are grouper codes, what type are they? They could be MSDRG, APDRG, APRDRG, APC, APG, or EAPG. Choose only from these options. -'REIMBURSEMENT_PLACEOFSERVICE_CODES' : 2-digit codes related to the place of service. -'REIMBURSEMENT_PROC_CODES' : 5-digit numeric or alphanumeric codes. They may be labelled as CPT or HCPCS codes. These are related to specific procedures. -'REIMBURSEMENT_PROC_CODE_MODIFIER' : 2-digit alphanumeric codes accompanying a PROC code. -'REIMBURSEMENT_REVENUE_CODES' : 3- or 4-digit numeric or alphanumeric codes related to revenue. They may be labelled as 'Rev' codes. -'REIMBURSEMENT_STATUS_INDICATOR_CODES' : 1- or 2-digit alphanumeric codes indicating the payment status. - -Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. - -Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. - -Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. -""" - - -def BOTTOM_UP_METHODOLOGY(d): - return f"""Analyze the full methodology text listed below: -{d['FULL_METHODOLOGY']} - -Your job is to identify what the concise methodology is based on the full methodology. - -Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour - -For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. -For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. - -Return only the answer, without using complete sentences. If there is nothing even close to the above options in the full methodology text, simply return 'N/A' -""" - - - -input_dict = utils.read_input() - -filename = list(input_dict.keys())[0] -contract_text = input_dict[filename] -print(filename) - - - - - -def convert_to_dict(table_text): - table_text = table_text.strip('{}') - table_text_list = table_text.split(']') - table_text_list = [item for item in table_text_list if len(item) > 0] - - final_dict = {} - num_elements = 0 - for key_value_text in table_text_list: - key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip()+']' - value_list = ast.literal_eval(value) - final_dict[key] = value_list - num_elements = len(value_list) - return final_dict, num_elements - - -def format_table(table_json, table_size): - table_text = "" - keys = [key for key in table_json.keys()] - table_text += ' : '.join(keys) + "\n" - for i in range(table_size): - for key in keys: - if table_json[key][i]: - #table_text += key + ': ' + table_json[key][i] + ', ' - table_text += table_json[key][i] + ' : ' - table_text += r'\n' - return table_text - -def align_and_format_tables(text_dict): - aligned_text_dict = {} - for key, text in text_dict.items(): - aligned_text = text - if 'Table Start' in text: - table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) - for table_text in table_texts: - try: - # Extract pretable text - pretable = table_text.split('{')[0].strip() - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) - except: - table_formatted = table_text - - # Align - if text.count(pretable) == 2: # One table - # Remove original table - aligned_text = text.replace(table_text, "") - # Align new table - aligned_text = aligned_text.replace(pretable, pretable + table_formatted) - - else: - aligned_text = aligned_text.replace(table_text, pretable + table_formatted) - aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") - else: - aligned_text_dict[key] = text - - return aligned_text_dict - -contract_text = preprocess.clean_newlines(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = align_and_format_tables({'5' : text_dict['5']}) -text_dict = preprocess.highlight_rates(text_dict) - -prompt = BOTTOM_UP_CODES({'SERVICE' : 'Psychiatric Diagnostic Evaluation', 'FULL_METHODOLOGY' : 'N/A' }, text_dict['5']) - -answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) -print(answer) - - - +all_dfs = [] +for file in os.listdir('output'): + if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): + try: + df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') + all_dfs.append(df) + except: + continue + +final_df = pd.concat(all_dfs, ignore_index=True) +final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') \ No newline at end of file From 305c43f4ffa1fcd0765132e05fdf18f33c9f20c4 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 12 Jun 2024 11:51:23 -0700 Subject: [PATCH 53/78] Added docstrings --- src/claude_funcs.py | 28 ++++++ src/dict_operations.py | 44 +++++++++ src/file_processing.py | 80 ++++++++++++++++ src/main.py | 12 +++ src/postprocess.py | 109 +++++++++++++++++++--- src/postprocessingfuncs.py | 185 ++++++++++++++++++++++++++++++++++++- src/preprocess.py | 70 ++++++++++++++ src/prompt_funcs.py | 130 ++++++++++++++++++++++++++ src/table_funcs.py | 40 ++++++++ 9 files changed, 682 insertions(+), 16 deletions(-) diff --git a/src/claude_funcs.py b/src/claude_funcs.py index b4a656e..9d1e9f8 100644 --- a/src/claude_funcs.py +++ b/src/claude_funcs.py @@ -6,6 +6,20 @@ import config # Claude calls def invoke_claude_2(prompt, max_tokens): + """ + Invokes the Claude 2 language model with specified parameters to generate a response based on the input prompt. + + This function constructs a JSON request with the prompt, modified to include pre-defined introductory and concluding + sections for consistency. It specifies the token limit and other parameters to control the generation process. The function + sends the request to the model's runtime environment and processes the response to extract and return the generated text. + + Parameters: + prompt (str): The text prompt to be processed by the Claude 2 model, describing the information or question addressed. + max_tokens (int): The maximum number of tokens that the model is allowed to generate in its response. + + Returns: + str: The text generated by the Claude 2 model in response to the input prompt. + """ body = json.dumps( {"prompt": anthropic.HUMAN_PROMPT + prompt + anthropic.AI_PROMPT, "max_tokens_to_sample": max_tokens, @@ -29,6 +43,20 @@ def invoke_claude_2(prompt, max_tokens): return response_text def invoke_claude_3(prompt, model_id=config.MODEL_ID_CLAUDE3_SONNET, max_tokens = 0): + """ + Invokes the Claude 3 language model with specified parameters to generate a response based on the input prompt. + + This function constructs a JSON request with the prompt, modified to include pre-defined introductory and concluding + sections for consistency. It specifies the token limit and other parameters to control the generation process. The function + sends the request to the model's runtime environment and processes the response to extract and return the generated text. + + Parameters: + prompt (str): The text prompt to be processed by the Claude 2 model, describing the information or question addressed. + max_tokens (int): The maximum number of tokens that the model is allowed to generate in its response. + + Returns: + str: The text generated by the Claude 2 model in response to the input prompt. + """ prompt = prompt body = json.dumps({ "anthropic_version": "bedrock-2023-05-31", diff --git a/src/dict_operations.py b/src/dict_operations.py index 3a41713..78ee6fa 100644 --- a/src/dict_operations.py +++ b/src/dict_operations.py @@ -4,6 +4,19 @@ import json from collections import defaultdict def secondary_string_to_dict(dict_string): + """ + Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues. + + This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing. + It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common + parsing errors by replacing problematic single quotes with double quotes before re-parsing. + + Parameters: + dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters. + + Returns: + dict: The dictionary obtained from parsing the cleaned and corrected string. + """ dict_string = dict_string.replace('<<<', '') dict_string = dict_string.replace('>>>', '') dict_string = dict_string.replace('\n', '') # Remove new line @@ -19,6 +32,22 @@ def secondary_string_to_dict(dict_string): return result_dict def primary_string_to_dict(string_dict, filename): + """ + Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries, + augmenting each with metadata such as page number and filename. + + This function iterates over each page number's string data, extracting and converting string representations of + dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries. + All dictionaries are then augmented with their respective page number and the filename before being compiled into a list. + + Parameters: + string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries. + filename (str): The filename associated with these entries, used to tag each resulting dictionary. + + Returns: + list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with + their page number and filename. + """ data = [] pattern = r'\{.*?\}' for page_num in string_dict.keys(): @@ -38,6 +67,21 @@ def primary_string_to_dict(string_dict, filename): return data def append_secondary(result_dicts): + """ + Aggregates and restructures a list of dictionaries by grouping based on specific fields and appending secondary and tertiary + entries for specific keys. + + This function organizes dictionaries by creating a unique key from non-'REIMBURSEMENT' fields and groups them accordingly. + It then processes each group to combine entries, designating the first three as primary, secondary, and tertiary respectively + for any 'REIMBURSEMENT' related fields. The combined results are stored in new dictionaries reflecting this hierarchical structure. + + Parameters: + result_dicts (list of dict): A list of dictionaries containing data with fields that may include several related to 'REIMBURSEMENT'. + + Returns: + list of dict: A list of new dictionaries where each dictionary represents combined information from the original list, + with 'REIMBURSEMENT' fields clearly distinguished as primary, secondary, or tertiary. + """ # Helper function to create a key from dict excluding certain keys def create_key(d): return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT')) diff --git a/src/file_processing.py b/src/file_processing.py index cc3caa6..755b1bc 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -11,7 +11,24 @@ import claude_funcs import postprocessingfuncs import postprocess + def clean_td(td): + """ + Cleans the data structure returned from a 'Top Down' analysis by normalizing and formatting its values. + + This function iterates through each entry in the list of dictionaries returned from the top down analysis, + modifying data based on specific criteria: + - Dates are converted to lists if not already in list form. + - Comma-separated strings are split into lists. + - 'N/A' values are replaced with empty lists. + - Preserves specific keys as they are (e.g., 'page_num', 'Filename'). + + Parameters: + td (list of dict): The list of dictionaries containing the results from top down analysis. + + Returns: + list of dict: A new list of dictionaries with the cleaned and normalized data. + """ td_clean = [] for d in td: new_d = {} @@ -32,12 +49,38 @@ def clean_td(td): def get_unique_keys(dicts): + """ + Collects and returns a set of unique keys from a list of dictionaries. + + This function aggregates all the keys from each dictionary in the provided list, + ensuring that each key is only listed once, regardless of how often it appears across + the dictionaries. + + Parameters: + dicts (list of dict): A list of dictionaries from which to extract unique keys. + + Returns: + set: A set containing all unique keys found in the input list of dictionaries. + """ keys = set() for d in dicts: keys.update(d.keys()) return keys def get_unique_date_fields(td_results): + """ + Extracts and aggregates unique date values from a list of dictionaries, specifically focusing on keys that include 'DATE'. + + This function iterates through each dictionary in the provided list, identifies keys that contain the substring 'DATE', + and collects unique values associated with these date keys. The collected values are stored in a set to avoid duplicates, + and then converted to a list for each date field before returning. + + Parameters: + td_results (list of dict): A list of dictionaries, each potentially containing multiple date-related fields. + + Returns: + dict: A dictionary where each key is a date field and the value is a list of unique dates found in that field. + """ unique_date_fields = {} for td in td_results: for key, value in td.items(): @@ -50,6 +93,22 @@ def get_unique_date_fields(td_results): return unique_date_fields def merge_results(td_results, bu_results, text_dict): + """ + Merges results from Top Down (TD) and Bottom Up (BU) processing streams based on page numbers and keys. + + This function combines results from two different analyses by aligning them using their page numbers and + merges values under the same keys, giving precedence to non-empty and list values. It also resolves conflicts + and fills missing values for date fields using a unique set of date values extracted from TD results. + + Parameters: + td_results (list of dict): Results from the Top Down processing, containing page numbers and various attributes. + bu_results (list of dict): Results from the Bottom Up processing, similarly structured as TD results. + text_dict (dict): Dictionary of page texts keyed by page numbers used for generating prompts if necessary. + + Returns: + list of dict: A list of dictionaries, each representing merged results from both TD and BU for a page. + """ + all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) date_fields = get_unique_date_fields(td_results) @@ -93,6 +152,27 @@ def merge_results(td_results, bu_results, text_dict): def process_file(file_object): + """ + Processes a single file object containing text data from a contract. + + This function orchestrates the full processing workflow for a single document, including: + - Directory setup for outputs based on the filename. + - Preprocessing text to clean and split it into manageable parts. + - Running 'Top Down' and 'Bottom Up' analysis using pre-defined models. + - Merging and consolidating results from both analysis methods. + - Postprocessing the consolidated results for final output. + + Parameters: + file_object (tuple): A tuple containing the filename and the text content of the file. + + Outputs: + - Several CSV files are generated in the designated output directory: + 1. Results from the Top Down analysis. + 2. Results from the Bottom Up analysis. + 3. Combined results of both analyses before and after postprocessing. + + Each step of the process is logged verbosely if the configuration is set to verbose mode. + """ ################## INITIATE PROCESSING ################## filename, contract_text = file_object diff --git a/src/main.py b/src/main.py index 5eca37c..87c08b5 100644 --- a/src/main.py +++ b/src/main.py @@ -1,4 +1,16 @@ +""" main.py +Doczy AI - Pricing Before Carveouts: Main Execution Module + +This module is the main entry point for the Priority B: Pricing Before Carveouts section of Doczy AI. + +Functional Overview: +- In test mode, a simple Claude 3 model invocation is demonstrated. +- In normal mode, the module reads input documents, processes each document concurrently, + and manages exceptions and errors during processing. +- The processed results can be optionally consolidated into a single CSV file. +""" + # Imports import concurrent.futures import traceback diff --git a/src/postprocess.py b/src/postprocess.py index a0677a1..02a8125 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -5,11 +5,41 @@ import postprocessingfuncs import config def sanitize_combined(df): + """ + Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value. + + This function iterates over each column in the DataFrame, applying a sanitization function to each element. + The sanitization function is designed to clean or adjust the data in a way that makes it more consistent or appropriate + for further processing or analysis. This could include operations like trimming whitespace, correcting formats, + or removing unwanted characters. + + Parameters: + df (pandas.DataFrame): The DataFrame to be sanitized. + + Returns: + pandas.DataFrame: The sanitized DataFrame, with the same structure but cleaned data. + """ for column in df.columns: df[column] = df[column].apply(postprocessingfuncs.sanitize_value) return df def extract_codes_CPT(value): + """ + Extracts and formats CPT codes from a given input value. + + This function searches for patterns that match CPT codes within the provided input. CPT codes typically consist + of a letter followed by four digits, or just five digits, and may include a two-letter state code as a suffix. + The function handles cases where the range of codes is specified using 'through' by converting it to a hyphen. + If any codes are found, they are returned as a comma-separated string. If no codes are found or the input is null, + an appropriate value (empty string or null) is returned. + + Parameters: + value (str or NaN): The value from which to extract CPT codes, which could potentially be a string or NaN. + + Returns: + str: A comma-separated string of the extracted CPT codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ if pd.isna(value): return value @@ -23,6 +53,21 @@ def extract_codes_CPT(value): return "" def extract_codes_Diagnosis(value): + """ + Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats. + + This function identifies patterns consistent with ICD diagnosis codes, which are generally a letter followed by numbers + and may include additional alphanumeric characters and a period for further specification. The function handles null inputs + gracefully, returning the input itself if null. If valid diagnosis codes are found, they are returned as a comma-separated + string. If no codes are found, the function returns an empty string. + + Parameters: + value (str or NaN): The input value from which to extract diagnosis codes, which could be a string or NaN. + + Returns: + str: A comma-separated string of the extracted diagnosis codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ if pd.isna(value): return value @@ -34,6 +79,20 @@ def extract_codes_Diagnosis(value): return "" def extract_codes_Revenue(value): + """ + Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits. + + This function searches for patterns that match revenue codes in the input provided. It handles cases where the input might be null (NaN), + returning the input itself in such cases. If any valid revenue codes are found, they are formatted into a comma-separated string. + If no codes are found, the function returns an empty string, indicating the absence of recognizable codes in the input. + + Parameters: + value (str or NaN): The input value from which to extract revenue codes, which could be a string or NaN. + + Returns: + str: A comma-separated string of the extracted revenue codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ if pd.isna(value): return value @@ -45,10 +104,44 @@ def extract_codes_Revenue(value): return "" def clean_code_column(df, column_name, extract_function): + """ + Applies a specified function to clean and extract codes from a specific column in a DataFrame. + + This function takes a DataFrame and applies a given extraction function to each element of a specified column. + The extraction function is expected to format or extract codes (e.g., CPT, ICD, Revenue codes) from the column's values. + This cleaned and extracted data replaces the original values in the column. The function is flexible and can be used + with any function designed to process and return cleaned values from a single input. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the data to be cleaned. + column_name (str): The name of the column to be cleaned. + extract_function (function): The function to apply to each value in the specified column, which processes and returns the cleaned value. + + Returns: + pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes. + """ df[column_name] = df[column_name].apply(extract_function) return df def post_process_combined(df): + """ + Performs comprehensive post-processing on a DataFrame to sanitize, correct, and clean data fields related to healthcare contracts. + + This function includes steps to: + - Sanitize the entire DataFrame. + - Ensure that LOB, Program, and Network fields contain valid values. + - Use fuzzy matching to correct near-miss entries in specified fields. + - Correct misplaced values across similar columns. + - Clean code columns for CPT, Diagnosis, and Revenue codes. + - Filter and adjust specific data columns to align with expected formats and values. + + Parameters: + df (pandas.DataFrame): The DataFrame containing combined data that needs post-processing. + + Returns: + pandas.DataFrame: The DataFrame after applying all specified cleaning, correction, and filtering processes. + """ + VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] @@ -75,19 +168,6 @@ def post_process_combined(df): except: pass - # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_LOB', VALID_LOBS, 'Corrected_LOB') - - # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 'Corrected_PROGRAM') - # postprocessingfuncs.clean_columns_combined(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 'Corrected_NETWORK') - - # # Apply fuzzy matching to the columns - # try: - # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_LOB', VALID_LOBS, 0.8) - # except: - # pass - # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_PROGRAM', VALID_PROGRAMS, 0.8) - # postprocessingfuncs.clean_columns_combined_fuzzy(df, 'CONTRACT_NETWORK', VALID_NETWORKS, 0.8) - # Correct misplaced values across columns try: df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) @@ -140,10 +220,9 @@ def post_process_combined(df): except Exception as e: print(f"Error: {e}") clean_df = df - - return clean_df + def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index ce3ab00..7a82d20 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -4,6 +4,20 @@ import re import difflib def sanitize_value(value): + """ + Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting. + + This function performs several cleanliness checks and transformations: + - It leaves null (NaN) values unchanged. + - Converts lists into comma-separated strings. + - Strips unnecessary characters from strings and handles strings that represent lists, ensuring individual elements are properly formatted. + + Parameters: + value (varied types): The value to be sanitized, which can be of any type including nulls, lists, or strings. + + Returns: + varied types: The sanitized value, which can be a string or the original type if no changes were needed. + """ if pd.isna(value): return value if isinstance(value, list): @@ -14,6 +28,20 @@ def sanitize_value(value): return value def exact_match(val, valid_values): + """ + Compares a given string value against a list of valid values to determine if there is an exact match, case-insensitively. + + This function processes the input value by trimming whitespace and converting to uppercase for a case-insensitive comparison. + It then checks each item in the list of valid values (also treated case-insensitively) to find an exact match. If a match is found, + the function returns the valid value in its original form as specified in the list. If no match is found, it returns None. + + Parameters: + val (str): The value to be checked for an exact match. + valid_values (list of str): A list of valid values against which the input value is compared. + + Returns: + str or None: The matching valid value if found, or None if no exact match is found. + """ val = val.strip().upper() for valid_val in valid_values: if val == valid_val.upper(): @@ -21,6 +49,22 @@ def exact_match(val, valid_values): return None def clean_columns_combined(df, column_name, valid_values, new_column_name): + """ + Cleans and standardizes entries in a specified column of a DataFrame based on a list of valid values, creating a new column with standardized values. + + This function first sanitizes the values in the specified column. It then examines each entry in that column, + looking for any sub-values (separated by commas) that exactly match any of the provided valid values. If a match is found, + it is recorded in a new column. The function also tracks changes and compares the original and cleaned unique values. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the data to be cleaned. + column_name (str): The name of the column to be cleaned. + valid_values (list of str): A list of valid values to be used for checking the entries in the specified column. + new_column_name (str): The name of the new column where cleaned, standardized values will be stored. + + Returns: + tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes from original to new values. + """ changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -38,14 +82,48 @@ def clean_columns_combined(df, column_name, valid_values, new_column_name): cleaned_values = df[new_column_name].unique() return original_values, cleaned_values, changes + def get_closest_match(val, valid_values, similarity_threshold=0.7): + """ + Finds the closest match for a given string from a list of valid values, using a similarity threshold. + + This function cleans the input value and compares it to each cleaned value in the valid values list using a fuzzy + matching technique. It returns the closest match that meets or exceeds the specified similarity threshold. If no + matches meet the threshold, the function returns None. + + Parameters: + val (str): The value to be matched. + valid_yvalues (list of str): A list of valid values against which the input value is compared. + similarity_threshold (float): The cutoff similarity score (between 0 and 1) below which matches are not considered. + + Returns: + str or None: The closest valid value if a match is found; otherwise, None. + """ if pd.isna(val): return None val = val.strip().upper() matches = difflib.get_close_matches(val, [v.upper() for v in valid_values], n=1, cutoff=similarity_threshold) return matches[0] if matches else None + def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): + """ + Applies fuzzy matching to entries in a specified column of a DataFrame to standardize them according to a list of valid values, + using a defined similarity threshold to determine matches. + + The function sanitizes the column values and then processes each entry, splitting it into sub-values if multiple are present. + Each sub-value is compared against the valid values list using a fuzzy matching algorithm. If a close match is found (above the + threshold), it replaces the original sub-value. The function logs any changes made for tracking and verification purposes. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the column to be cleaned. + column_name (str): The name of the column to be cleaned. + valid_values (list of str): A list of valid values used for fuzzy matching. + threshold (float): The similarity threshold for accepting matches. + + Returns: + tuple: Contains arrays of original unique values, cleaned unique values, and a dictionary of changes (original to new values). + """ changes = {} df[column_name] = df[column_name].apply(sanitize_value) original_values = df[column_name].unique() @@ -66,7 +144,21 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): cleaned_values = df[column_name].unique() return original_values, cleaned_values, changes + def extract_page_number(page_text): + """ + Extracts the page number from a given text string that includes a range of pages formatted as "Pages X-Y". + + This function uses a regular expression to find and extract the first page number from a text string that includes a page range. + If the specific pattern "Pages X-Y" is found in the text, it captures and returns the number corresponding to 'X'. If the pattern + is not found, the entire input text is returned, indicating that no page number could be extracted based on the expected format. + + Parameters: + page_text (str): The text containing the page number information, typically including a page range. + + Returns: + str: The extracted first page number if the pattern is found; otherwise, the original text. + """ match = re.search(r'Pages\s+(\d+)-\d+', page_text) if match: return match.group(1) @@ -78,6 +170,22 @@ def clean_pagenumbers(df): return df def correct_misplaced_values(df, columns, valid_values_dict): + """ + Corrects misplaced values within specified columns of a DataFrame based on a dictionary of valid values for each column. + + This function iterates through each row of the DataFrame and checks the values of specified columns. If a value + is found that matches valid entries for a different column (based on the valid_values_dict), it moves the value to + the appropriate column. If the target column already contains a value, it uses fuzzy matching to determine if the + misplaced value should replace the existing one. If a replacement isn't suitable, it logs the correction attempt. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the data to be corrected. + columns (list of str): The columns to check for misplaced values. + valid_values_dict (dict): A dictionary mapping column names to lists of valid values for those columns. + + Returns: + pandas.DataFrame: The DataFrame with corrected placements of values across the specified columns. + """ for index, row in df.iterrows(): for col in columns: if pd.notna(row[col]): @@ -98,7 +206,22 @@ def correct_misplaced_values(df, columns, valid_values_dict): df.at[index, col] = None return df + def filter_service_column(df): + """ + Filters out rows in a DataFrame based on specified keywords within the 'SERVICE' column. + + This function uses a list of predefined keywords associated with non-service related terms like 'LIABILITY' or 'RISK'. + It constructs a regular expression pattern from these keywords and removes any rows where the 'SERVICE' column contains + any of these keywords, effectively filtering the DataFrame to exclude rows that are likely to be irrelevant or misclassified + as services when they pertain to other contractual obligations or conditions. + + Parameters: + df (pandas.DataFrame): The DataFrame from which rows will be filtered. + + Returns: + pandas.DataFrame: A new DataFrame with rows containing specified keywords in the 'SERVICE' column removed. + """ keywords = [ 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' ] @@ -106,7 +229,21 @@ def filter_service_column(df): df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] return df + def move_percentage_to_rate(df): + """ + Moves percentage values from the 'REIMBURSEMENT_FLAT_FEE' column to the 'REIMBURSEMENT_RATE' column in a DataFrame. + + This function identifies entries in the 'REIMBURSEMENT_FLAT_FEE' column that contain a percentage sign, indicating they are + incorrectly placed. It then transfers these percentage values to the 'REIMBURSEMENT_RATE' column. The original 'REIMBURSEMENT_FLAT_FEE' + entries from which percentages are moved are set to None, correcting the placement of data within the DataFrame. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the reimbursement data to be adjusted. + + Returns: + pandas.DataFrame: The DataFrame with percentage values moved from the 'REIMBURSEMENT_FLAT_FEE' to the 'REIMBURSEMENT_RATE' column. + """ def move_percentage(value): if isinstance(value, str) and '%' in value: return True @@ -116,9 +253,24 @@ def move_percentage_to_rate(df): df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: None if move_percentage(row['REIMBURSEMENT_FLAT_FEE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) return df + def move_large_numbers_to_flat_fee(df): + """ + Transfers large numerical values from the 'REIMBURSEMENT_RATE' column to the 'REIMBURSEMENT_FLAT_FEE' column in a DataFrame. + + This function examines entries in the 'REIMBURSEMENT_RATE' column to identify numeric values exceeding 1000, + which are more indicative of flat fees rather than rates. It moves these values to the 'REIMBURSEMENT_FLAT_FEE' column. + The original 'REIMBURSEMENT_RATE' entries from which these numbers are moved are set to None, ensuring the data reflects + the correct financial parameters. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the reimbursement data to be adjusted. + + Returns: + pandas.DataFrame: The DataFrame with large numbers moved from the 'REIMBURSEMENT_RATE' to the 'REIMBURSEMENT_FLAT_FEE' column. + """ def move_large_number(value): - if isinstance(value, str) and value.isdigit() and int(value) > 100: + if isinstance(value, str) and value.isdigit() and int(value) > 1000: return True return False @@ -126,7 +278,23 @@ def move_large_numbers_to_flat_fee(df): df['REIMBURSEMENT_RATE'] = df.apply(lambda row: None if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_RATE'], axis=1) return df + def set_rate_to_zero_if_not_covered(df): + """ + Sets the 'REIMBURSEMENT_RATE' to zero for entries in the DataFrame where the service is explicitly not covered, + as indicated by the content of the 'FULL_METHODOLOGY' column. + + This function checks each row's 'FULL_METHODOLOGY' column for phrases indicating that the service is not covered + (e.g., "not covered"). If such a phrase is found, the 'REIMBURSEMENT_RATE' for that row is set to zero to reflect that + there is no financial reimbursement for these services. This helps ensure that the data accurately represents the terms + of service coverage. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the reimbursement and methodology data. + + Returns: + pandas.DataFrame: The updated DataFrame with adjusted 'REIMURSEMENT_RATE' values where applicable. + """ def check_and_set_rate(row): if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'not covered', row['FULL_METHODOLOGY'], re.IGNORECASE): return 0 @@ -135,7 +303,22 @@ def set_rate_to_zero_if_not_covered(df): df['REIMBURSEMENT_RATE'] = df.apply(check_and_set_rate, axis=1) return df + def adjust_reimbursement_rate(df): + """ + Adjusts the 'REIMBURSEMENT_RATE' in the DataFrame based on specific conditions mentioned in the 'FULL_METHODOLOGY' column. + + This function reviews each row's 'FULL_METHODOLOGY' column for mentions of being "case insensitive" as a part of the methodology. + If this condition is noted and the 'REIMBURSEMENT_RATE' is a numerical value, it adjusts the rate by subtracting it from 100, + effectively transforming the rate into a complementary percentage. This modification is intended to correct or recalibrate + reimbursement rates under specific contractual terms. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the methodology and reimbursement rate data. + + Returns: + pandas.DataFrame: The updated DataFrame with recalibrated 'REIMBURSEMENT_RATE' values based on the specified methodology condition. + """ def adjust_rate(row): if pd.notna(row['FULL_METHODOLOGY']) and re.search(r'case insensitive', row['FULL_METHODOLOGY'], re.IGNORECASE): if pd.notna(row['REIMBURSEMENT_RATE']) and isinstance(row['REIMBURSEMENT_RATE'], (int, float)): diff --git a/src/preprocess.py b/src/preprocess.py index 7accf76..89354c7 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -3,15 +3,57 @@ from itertools import groupby, count import re def clean_newlines(contract_text): + """ + Cleans up isolated newlines in a contract text, converting them into spaces to ensure text continuity. + + This function uses a regular expression to identify standalone newlines (those not part of a paragraph break, which + typically consists of consecutive newlines) and replaces them with a single space. This process helps in preserving + the flow of text and making it more readable and processable, particularly for text extraction and analysis tasks that + might be sensitive to abrupt breaks in the text. + + Parameters: + contract_text (str): The text of the contract that may contain sporadic newlines. + + Returns: + str: The cleaned contract text with isolated newlines replaced by spaces. + """ cleaned_text = re.sub(r'(?>>' and '<<<' to highlight them. The modified text for each page + is then updated in the dictionary, allowing for easier identification and processing of rate-related terms in subsequent analyses. + + Parameters: + text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages. + + Returns: + dict: The updated dictionary with rate-related terms highlighted in the text of each page. + """ for page, text in text_dict.items(): words = text.split() highlighted_words = [] @@ -22,7 +64,21 @@ def highlight_rates(text_dict): text_dict[page] = ' '.join(highlighted_words) return text_dict + def chunk_text(text_dict): + """ + Creates text chunks from a dictionary of page texts, focusing on pages with special characters (percentages and dollar amounts). + + This function identifies pages that contain '%' or '$' signs and includes those pages along with their immediate neighbors + (previous and next pages) to form chunks. The chunks are then grouped and concatenated into single text blocks for easier + processing. Each chunk is stored in a new dictionary where the keys represent the range of pages included in the chunk. + + Parameters: + text_dict (dict): A dictionary where keys are page numbers (as strings) and values are the text content of those pages. + + Returns: + dict: A new dictionary where each key is a string representing the range of pages in a chunk, and each value is the concatenated text of those pages. + """ special_pages = {int(page): text for page, text in text_dict.items() if '%' in text or '$' in text} page_numbers = sorted(special_pages.keys()) chunk_page_numbers = [] @@ -40,7 +96,21 @@ def chunk_text(text_dict): chunk_dict[dict_key] = text return chunk_dict + def clean_billed_charges(text): + """ + Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage. + + This function uses a regular expression to find all instances of the phrase 'billed charges'. For each match, it checks the preceding + text (up to 15 characters before the match) for a percentage sign ('%'). If a percentage is found, it leaves the match unchanged. + Otherwise, it replaces 'billed charges' with '100% of billed charges'. + + Parameters: + text (str): The input text to be cleaned. + + Returns: + str: The cleaned text with appropriate replacements made for 'billed charges'. + """ def replace(match): start_pos = match.start() pre_text = text[max(0, start_pos-15):start_pos] diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 7cd0150..0719d92 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -9,6 +9,25 @@ import utils def run_bottom_up_lesser(d, text_dict, tokens=8000): + """ + Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of', + using a language model to analyze and extract relevant data from the identified text segment. + + This function takes a dictionary representing initial results and a text dictionary, uses the page number from the initial + results to locate the relevant text, and generates a prompt for language model analysis. Based on the response, it + determines whether 'lesser of' or 'greater of' language is indicated, updates the results, and returns a new dictionary + for the updated data or retains the original if no such language is present. + + Parameters: + d (dict): A dictionary containing initial processing results from previous analyses, including a 'page_num' key. + text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed. + tokens (int): Token limit for language model invocations, default set to 8000. + + Returns: + tuple: A tuple containing two dictionaries, the first for any identified 'lesser of' or 'greater of' data and the second + containing the original or updated data depending on the presence of such language. + """ + prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) lesser_of_dict = dict_operations.secondary_string_to_dict(lesser_of_answer) @@ -23,6 +42,19 @@ def run_bottom_up_lesser(d, text_dict, tokens=8000): def run_bottom_up_primary(text_dict, tokens): + """ + Executes the primary Bottom Up processing for pages that contain reimbursement terms. + + This function scans through a dictionary of page texts, identifies pages containing reimbursement terms, + and processes those pages using Claude to extract relevant information. + + Parameters: + text_dict (dict): A dictionary containing text content of documents keyed by page numbers. + tokens (int): The maximum number of tokens to use in the language model invocation for generating the response. + + Returns: + dict: A dictionary where each key is a page number and the value is the response from the language model. + """ #chunk_dict = preprocess.chunk_text(text_dict) answer_dict = {} #for page_number in chunk_dict.keys(): @@ -38,6 +70,24 @@ def run_bottom_up_primary(text_dict, tokens): def run_bottom_up_secondary(answer_dicts, text_dict, tokens): + """ + Executes the secondary Bottom Up processing phase on the results obtained from the primary Bottom Up analysis. + + This function enhances the primary results with additional analyses based on configured conditions. Invokes + Claude 3 with tailored prompts to generate structured information that complements the initial results. + + Each piece of data processed possibly undergoes several rounds of checks and transformations, ensuring detailed and + comprehensive output. + + Parameters: + answer_dicts (list of dict): Initial processed data from the primary Bottom Up analysis. + text_dict (dict): Dictionary containing the original document text keyed by page numbers. + tokens (int): Token limit for language model invocations. + + Returns: + list of dict: A list of dictionaries containing enriched and finalized structured data from both the primary and + secondary analyses. + """ # New rows for lesser of temp_dicts = [] for d in answer_dicts: @@ -90,6 +140,20 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts def run_bottom_up(filename, text_dict): + """ + Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information. + + This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further + refine and structure them into a dictionary form. + + Parameters: + filename (str): The name of the file being processed, used to tag output data. + text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed. + + Returns: + list of dict: A list of dictionaries with each dictionary containing refined and structured information + from both primary and secondary Bottom Up analyses, all tagged with the filename. + """ # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) @@ -102,7 +166,23 @@ def run_bottom_up(filename, text_dict): d['Filename'] = filename return results_dicts # List of dictionaries + def run_top_down_metal_level(d, page): + """ + Identifies and extracts the metal level of a contract within a specific line of business (LOB) from the provided page text. + + This function determines whether the contract's LOB is associated with marketplace or commercial sectors by examining the + 'CONTRACT_LOB' key in the dictionary. If relevant, it constructs and sends a specific prompt to a language model to extract the + metal level (e.g., Bronze, Silver, Gold, Platinum). If the LOB is not relevant, it directly assigns 'N/A'. + + Parameters: + d (dict): A dictionary containing extracted information from primary Top Down analysis, specifically the 'CONTRACT_LOB' key. + page (str): The text content of the page that is being analyzed for metal level information. + + Returns: + str: The metal level of the contract as determined by the analysis, or 'N/A' if the contract LOB is not applicable. + """ + if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper(): prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) @@ -110,13 +190,46 @@ def run_top_down_metal_level(d, page): answer = 'N/A' return answer + def run_top_down_date(type_, d, page): + """ + Extracts specific date-related information from a page of text using a Top Down processing approach. + + This function tailors the extraction to focus on either 'EFFECTIVE' or 'TERMINATION' dates by constructing + a prompt that directs a language model to search for and interpret date information relevant to the provided type. + It formats the input data to fit the processing needs, sends it along with the page text to the model, and captures + the model's response. + + Parameters: + type_ (str): Specifies the type of date to extract, 'EFFECTIVE' or 'TERMINATION'. + d (dict): A dictionary containing preliminary data extracted from the text, which may include metadata like filename or page number. + page (str): The text content of the page from which to extract the date. + + Returns: + str: The extracted date as a string, based on the model's interpretation of the input prompt and text context. + """ formatted_d = utils.format_td_check([d], ['Filename', 'page_num']) prompt = prompts.TOP_DOWN_DATE('EFFECTIVE', formatted_d, page) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) return answer + def top_down_secondary(td_results, text_dict): + """ + Conducts secondary processing on results obtained from the primary Top Down analysis of document text, enhancing detail and accuracy. + + This function iteratively enhances each dictionary result from the initial analysis by adding or refining information related to: + - Market metal levels by running a specific function to determine the contract marketplace metal level. + - Effective dates by analyzing the context around specific keywords in the text and extracting these dates. + - Termination dates, similarly extracted based on the presence and context of specific keywords. + + Parameters: + td_results (list of dict): Initial dictionaries from the primary Top Down processing containing basic extracted data. + text_dict (dict): Dictionary keyed by page numbers, providing the full text for corresponding analysis and extraction tasks. + + Returns: + list of dict: Updated list of dictionaries, each enhanced with additional detailed attributes like metal level and date specifics. + """ updated_dicts = [] for d in td_results: # Run Metal Level @@ -132,7 +245,24 @@ def top_down_secondary(td_results, text_dict): return updated_dicts + def run_top_down(filename, text_dict): + """ + Executes the Top Down processing strategy on a provided dictionary of text pages, extracting structured data based on specified prompts. + + The function first filters and processes each page of text that is correctly numbered, using a primary analysis prompt. The results + from this analysis are then parsed into dictionaries, tagged with their respective page numbers and filename, and collected. + After the initial processing, a secondary Top Down analysis is conducted to refine and possibly expand the extracted data. + + Parameters: + filename (str): The filename associated with the text, used to tag the output data. + text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages. + + Returns: + list of dict: A list of dictionaries where each dictionary contains structured results from both primary and secondary + Top Down analyses, associated with a specific page and the overall document. + """ + all_results = [] # Primary diff --git a/src/table_funcs.py b/src/table_funcs.py index e02f144..741a4a8 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -3,6 +3,18 @@ import re import ast def convert_to_dict(table_text): + """ + Converts a formatted table text into a dictionary where each key maps to a list of values. + + This function processes a string representation of a table, extracting key-value pairs and converting them into a dictionary. + Each key maps to a list of values parsed from the input string. The function also returns the number of elements in each list. + + Parameters: + table_text (str): The input string representing the table, formatted with keys and values. + + Returns: + tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list. + """ table_text = table_text.strip('{}') table_text_list = table_text.split(']') table_text_list = [item for item in table_text_list if len(item) > 0] @@ -19,6 +31,20 @@ def convert_to_dict(table_text): def format_table(table_json, table_size): + """ + Formats a dictionary of table data into a string representation of the table. + + This function takes a dictionary where each key maps to a list of values and converts it into a formatted string. + The keys are used as headers, and each row is constructed by iterating through the lists up to the specified table size. + The resulting string separates headers and values with ' : ' and includes newline characters to delineate rows. + + Parameters: + table_json (dict): The dictionary containing the table data, with keys as column headers and lists as column values. + table_size (int): The number of rows in the table. + + Returns: + str: The formatted string representation of the table. + """ table_text = "" keys = [key for key in table_json.keys()] table_text += ' : '.join(keys) + "\n" @@ -30,7 +56,21 @@ def format_table(table_json, table_size): table_text += r'\n' return table_text + def align_and_format_tables(text_dict): + """ + Aligns and formats tables within the text of each page in a dictionary. + + This function searches for table text within the input text of each page, marked by 'Table Start' and 'Table End' tags. + It extracts, converts, and formats these tables, then replaces the original table text with the formatted version while + maintaining the alignment with surrounding text. The cleaned and formatted text for each page is then stored in a new dictionary. + + Parameters: + text_dict (dict): A dictionary where keys are page numbers and values are the text content of those pages, potentially containing tables. + + Returns: + dict: A new dictionary with aligned and formatted tables in the text for each page. + """ aligned_text_dict = {} for key, text in text_dict.items(): aligned_text = text From 732a89c9613ed74cfe75db6e3a393cecbe2be582 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 17 Jun 2024 17:07:27 -0700 Subject: [PATCH 54/78] update postprocessing --- src/config.py | 8 +-- src/file_processing.py | 144 +------------------------------------ src/merge_funcs.py | 107 +++++++++++++++++++++++++++ src/postprocessingfuncs.py | 39 ++++++++++ 4 files changed, 152 insertions(+), 146 deletions(-) create mode 100644 src/merge_funcs.py diff --git a/src/config.py b/src/config.py index 5db760c..045d9fa 100644 --- a/src/config.py +++ b/src/config.py @@ -6,7 +6,7 @@ import boto3 #pip install llama-index-llms-bedrock # General Settings -TEST = False # True to run test prompt - just for testing model connection +TEST = True # True to run test prompt - just for testing model connection VERBOSE = True CLIENT_NAME = '' TODAY = datetime.now().strftime("%Y%m%d") @@ -45,9 +45,9 @@ RUN_CODES = True FUZZY_MATCH_THRESHOLD = 0.8 # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXGPEZJ5OW" -AWS_SECRET_ACCESS_KEY="PelB1A6KB6HnbBrdUeLgnf0Ii+dMpXCG5JLs4BUS" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEKz//////////wEaCXVzLWVhc3QtMiJIMEYCIQCxTs5XXRBybkfcptcaSQUN+2JvUa/dimXyxX31+CQeeAIhAL3NRuwIU+MxkVQBDon2BbGTZawkwF4NVvdcUEElGtxEKoMDCEUQABoMNjYwMTMxMDY4NzgyIgwLOdI9PpnuoefEZTMq4AJqQn9q4Uq3WQEK6Wg2iQNPaPTLbaBKLx8TtWwlS6wcIlhFNA+nPyaxew3boex97eYM4uzHiwU2MQ4OTbW+aOphdsWACU4jlaTM6Z0VWlJ+W+OKpMwajDIj7SQnfrDQSGwJ9Ewth3+uxBKqPtXXZmUe5WRCz7G4cIjYxEqdjtj0ZO6ll6zZozPEGVik1+DZoPMbh4Y4qczMDdpW3xXUlalYJBu6CaNqGIb6O1ocl4nhLSmamsKoiXLqGU853Loq0A3b+u+J7w48iPLhXWnuN0GXuN7gBc2C1lzPs/aABM2bU3LBlDqGxKGUDVPRHttg8HRNGwIkEr9LieGjheKvcvLQroi6DhTRkKnV6WEQ1wGzN9wCteXSL7DIktSsddnwX3ffvg7G5WhPN0xYc4F78Sw9gan50wmmebiHrc5lP+yQDJ7J6gvo4a3NsxCE5opV/hgJUjPUgJ7tCPZ+bf+kqKJqMNLem7MGOqUBjmMAY7Vvqp4pBdORFWTZkOHcmZ1UsmJmNYrS69btqyYQbwg79sCIaVR/wiJDn9T5v9vziyv0n98XD7dV6UEuyW97EKv/OWwnqZ7zJwHAa4vwJ09zCruIgT4hCctpzDAAfTYtkacb0PRrxa3pjQZy7pXQC0BvliocPY5fvR5f5QOa/mZGwhjUcih9LfvNWj0BsRW65pPcJoy6Wy3G0EgwsH+VW1h5" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMRIB6IY4" +AWS_SECRET_ACCESS_KEY="o+w4z8WyJeQi+l9O9LKAuWNpPogXAJZoS1VSgUJv" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGAaCXVzLWVhc3QtMiJHMEUCIQCt/8UZmc37xOqSNWy1mhtFzBRR4aJ2jjSI+DZXsL+dgQIgIKSurMIGok1CIpwkWzZetBEFZJuFrChMsnPq1GMl6tEqjAMI+f//////////ARAAGgw2NjAxMzEwNjg3ODIiDG/YpM3sZ+xicr8GcCrgAp5GZeUYm818DYciItJblOTIk0oNaPR73NMXCXHHeHORGR1KJiXTLinqR2OpI/GGX/AsSE6yIpuu2oEQXPUyM0RN9wSpPyVSWDtHmrtj19qsDMTDwLQeTQITBEEKpohhpWNX6y9kSgDF/2QOXxq1uI7mZlXBobdp0XfN7mEvQWBz6UH5EKvc3clkCWiE3GhlhvNsBFnIZSklPyy8jpkxQ1SL0xDeNUWHO9YzWlkrg3dM5T3ZUXk1IW7YiVEUQs/kbG6rVpMSSuINApd3AUNnMY/hBvc32CBOyK/9VgMMI+5hvqd+nUEDUE2zNa/e9YBHqn8JbsQG4cDIQasdD2G1CgDYLvGwJt2x3l+O7w61yfvmknoVOYROKDA2mmA9/XWvhPLUjTDR8Dl3iIcW8YIROVNb9GHMM/t1CX6p5VtiLYbKhPqn7YS44ulnbqbKdRW1MBKTmzzeLQquxZK19JZtBd0wnpzDswY6pgFSmdwIcQBYWdVBtaQNycF5p8041L0kZ4mLe3oqa4DQCHwZMeLA5LZP4c/VSMVHCnNM2rovqyO2A12pLgWvlsXIzitIuL+r405Sk/tJT3BXDuyxKGIS6wvP6eLteSrCCCyNXT+qipoRugVStkObsvBBGT/I535cDlb2DiG6eoPBkwnPvcgJbEJMK2UqNoTJAiuQhP1cmFIV5OratlRC7GjPqnmNxZOQ" # File Paths LOCAL_PATH = 'data/all_txt_updated/' # Replace with local diff --git a/src/file_processing.py b/src/file_processing.py index 755b1bc..ecafcfb 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -6,149 +6,9 @@ import config import preprocess import table_funcs import prompt_funcs -import prompts -import claude_funcs import postprocessingfuncs import postprocess - - -def clean_td(td): - """ - Cleans the data structure returned from a 'Top Down' analysis by normalizing and formatting its values. - - This function iterates through each entry in the list of dictionaries returned from the top down analysis, - modifying data based on specific criteria: - - Dates are converted to lists if not already in list form. - - Comma-separated strings are split into lists. - - 'N/A' values are replaced with empty lists. - - Preserves specific keys as they are (e.g., 'page_num', 'Filename'). - - Parameters: - td (list of dict): The list of dictionaries containing the results from top down analysis. - - Returns: - list of dict: A new list of dictionaries with the cleaned and normalized data. - """ - td_clean = [] - for d in td: - new_d = {} - for k, v in d.items(): - if 'DATE' in k: - new_d[k] = v if isinstance(v, list) else [v] - elif k not in ['page_num', 'Filename']: - if isinstance(v, str) and ',' in v: - new_d[k] = [item.strip() for item in v.split(',')] - elif v == 'N/A': - new_d[k] = [] - else: - new_d[k] = [v] if isinstance(v, str) else v - else: - new_d[k] = v - td_clean.append(new_d) - return td_clean - - -def get_unique_keys(dicts): - """ - Collects and returns a set of unique keys from a list of dictionaries. - - This function aggregates all the keys from each dictionary in the provided list, - ensuring that each key is only listed once, regardless of how often it appears across - the dictionaries. - - Parameters: - dicts (list of dict): A list of dictionaries from which to extract unique keys. - - Returns: - set: A set containing all unique keys found in the input list of dictionaries. - """ - keys = set() - for d in dicts: - keys.update(d.keys()) - return keys - -def get_unique_date_fields(td_results): - """ - Extracts and aggregates unique date values from a list of dictionaries, specifically focusing on keys that include 'DATE'. - - This function iterates through each dictionary in the provided list, identifies keys that contain the substring 'DATE', - and collects unique values associated with these date keys. The collected values are stored in a set to avoid duplicates, - and then converted to a list for each date field before returning. - - Parameters: - td_results (list of dict): A list of dictionaries, each potentially containing multiple date-related fields. - - Returns: - dict: A dictionary where each key is a date field and the value is a list of unique dates found in that field. - """ - unique_date_fields = {} - for td in td_results: - for key, value in td.items(): - if 'DATE' in key: - if key not in unique_date_fields: - unique_date_fields[key] = set() - unique_date_fields[key].update(value if isinstance(value, list) else [value]) - for key in unique_date_fields: - unique_date_fields[key] = list(unique_date_fields[key]) - return unique_date_fields - -def merge_results(td_results, bu_results, text_dict): - """ - Merges results from Top Down (TD) and Bottom Up (BU) processing streams based on page numbers and keys. - - This function combines results from two different analyses by aligning them using their page numbers and - merges values under the same keys, giving precedence to non-empty and list values. It also resolves conflicts - and fills missing values for date fields using a unique set of date values extracted from TD results. - - Parameters: - td_results (list of dict): Results from the Top Down processing, containing page numbers and various attributes. - bu_results (list of dict): Results from the Bottom Up processing, similarly structured as TD results. - text_dict (dict): Dictionary of page texts keyed by page numbers used for generating prompts if necessary. - - Returns: - list of dict: A list of dictionaries, each representing merged results from both TD and BU for a page. - """ - - all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) - - date_fields = get_unique_date_fields(td_results) - - merged_results = [] - for bu in bu_results: - merged = {key: bu.get(key, "") for key in all_keys} - td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']] - - for td in td_on_page: - for key, value in td.items(): - if not merged[key]: - merged[key] = value - elif isinstance(value, list) and value and not isinstance(merged[key], list): - merged[key] = value - elif isinstance(value, list) and value: - merged[key].extend(value) - - for date_key, date_values in date_fields.items(): - if date_key not in merged or not merged[date_key]: - merged[date_key] = date_values - else: - merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) - - for key in all_keys: - if isinstance(merged[key], list) and len(merged[key]) > 1: - page_num = bu['page_num'] - page_text = text_dict.get(page_num, "") - prompt = prompts.GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key]) - response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - try: - selected_value = response.strip() - merged[key] = selected_value - except Exception as e: - print(f"Error processing LLM response for key {key}: {e}") - merged[key] = ', '.join(merged[key]) - - merged_results.append(merged) - - return merged_results +import merge_funcs def process_file(file_object): @@ -200,7 +60,7 @@ def process_file(file_object): print(f"Bottom Up Complete - {filename}") ################## COMBINE TOP DOWN AND BOTTOM UP ################## - combined_results = merge_results(clean_td(td_results), bu_results, text_dict) + combined_results = merge_funcs.merge_results(postprocessingfuncs.clean_td(td_results), bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") ################## WRITE UNPROCESSED OUTPUT ################## diff --git a/src/merge_funcs.py b/src/merge_funcs.py new file mode 100644 index 0000000..a3307f5 --- /dev/null +++ b/src/merge_funcs.py @@ -0,0 +1,107 @@ + +import prompts +import claude_funcs + +def get_unique_keys(dicts): + """ + Collects and returns a set of unique keys from a list of dictionaries. + + This function aggregates all the keys from each dictionary in the provided list, + ensuring that each key is only listed once, regardless of how often it appears across + the dictionaries. + + Parameters: + dicts (list of dict): A list of dictionaries from which to extract unique keys. + + Returns: + set: A set containing all unique keys found in the input list of dictionaries. + """ + keys = set() + for d in dicts: + keys.update(d.keys()) + return keys + + +def get_unique_date_fields(td_results): + """ + Extracts and aggregates unique date values from a list of dictionaries, specifically focusing on keys that include 'DATE'. + + This function iterates through each dictionary in the provided list, identifies keys that contain the substring 'DATE', + and collects unique values associated with these date keys. The collected values are stored in a set to avoid duplicates, + and then converted to a list for each date field before returning. + + Parameters: + td_results (list of dict): A list of dictionaries, each potentially containing multiple date-related fields. + + Returns: + dict: A dictionary where each key is a date field and the value is a list of unique dates found in that field. + """ + unique_date_fields = {} + for td in td_results: + for key, value in td.items(): + if 'DATE' in key: + if key not in unique_date_fields: + unique_date_fields[key] = set() + unique_date_fields[key].update(value if isinstance(value, list) else [value]) + for key in unique_date_fields: + unique_date_fields[key] = list(unique_date_fields[key]) + return unique_date_fields + +def merge_results(td_results, bu_results, text_dict): + """ + Merges results from Top Down (TD) and Bottom Up (BU) processing streams based on page numbers and keys. + + This function combines results from two different analyses by aligning them using their page numbers and + merges values under the same keys, giving precedence to non-empty and list values. It also resolves conflicts + and fills missing values for date fields using a unique set of date values extracted from TD results. + + Parameters: + td_results (list of dict): Results from the Top Down processing, containing page numbers and various attributes. + bu_results (list of dict): Results from the Bottom Up processing, similarly structured as TD results. + text_dict (dict): Dictionary of page texts keyed by page numbers used for generating prompts if necessary. + + Returns: + list of dict: A list of dictionaries, each representing merged results from both TD and BU for a page. + """ + + all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) + + date_fields = get_unique_date_fields(td_results) + + merged_results = [] + for bu in bu_results: + merged = {key: bu.get(key, "") for key in all_keys} + td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']] + + for td in td_on_page: + for key, value in td.items(): + if not merged[key]: + merged[key] = value + elif isinstance(value, list) and value and not isinstance(merged[key], list): + merged[key] = value + elif isinstance(value, list) and value: + merged[key].extend(value) + + for date_key, date_values in date_fields.items(): + if date_key not in merged or not merged[date_key]: + merged[date_key] = date_values + else: + merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) + + for key in all_keys: + if isinstance(merged[key], list) and len(merged[key]) > 1: + page_num = bu['page_num'] + page_text = text_dict.get(page_num, "") + prompt = prompts.GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key]) + response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + try: + selected_value = response.strip() + merged[key] = selected_value + except Exception as e: + print(f"Error processing LLM response for key {key}: {e}") + merged[key] = ', '.join(merged[key]) + + merged_results.append(merged) + + return merged_results + diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 7a82d20..380a367 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -327,3 +327,42 @@ def adjust_reimbursement_rate(df): df['REIMBURSEMENT_RATE'] = df.apply(adjust_rate, axis=1) return df + + +def clean_td(td): + """ + Cleans the data structure returned from a 'Top Down' analysis by normalizing and formatting its values. + + This function iterates through each entry in the list of dictionaries returned from the top down analysis, + modifying data based on specific criteria: + - Dates are converted to lists if not already in list form. + - Comma-separated strings are split into lists. + - 'N/A' values are replaced with empty lists. + - Preserves specific keys as they are (e.g., 'page_num', 'Filename'). + + Parameters: + td (list of dict): The list of dictionaries containing the results from top down analysis. + + Returns: + list of dict: A new list of dictionaries with the cleaned and normalized data. + """ + td_clean = [] + for d in td: + new_d = {} + for k, v in d.items(): + if 'DATE' in k: + new_d[k] = v if isinstance(v, list) else [v] + elif k not in ['page_num', 'Filename']: + if isinstance(v, str) and ',' in v: + new_d[k] = [item.strip() for item in v.split(',')] + elif v == 'N/A': + new_d[k] = [] + else: + new_d[k] = [v] if isinstance(v, str) else v + else: + new_d[k] = v + td_clean.append(new_d) + return td_clean + + + From c2590719b26c7452628717baef06a9b5ca86e96f Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 17 Jun 2024 19:10:44 -0700 Subject: [PATCH 55/78] Updated postprocessing --- src/config.py | 13 +++++++++++-- src/file_processing.py | 4 +++- src/postprocess.py | 27 +++++++++++++++------------ src/postprocessingfuncs.py | 4 +++- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/config.py b/src/config.py index 045d9fa..ce64d2c 100644 --- a/src/config.py +++ b/src/config.py @@ -6,7 +6,7 @@ import boto3 #pip install llama-index-llms-bedrock # General Settings -TEST = True # True to run test prompt - just for testing model connection +TEST = False # True to run test prompt - just for testing model connection VERBOSE = True CLIENT_NAME = '' TODAY = datetime.now().strftime("%Y%m%d") @@ -43,6 +43,15 @@ RUN_CODES = True # Postprocessing Settings FUZZY_MATCH_THRESHOLD = 0.8 +VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'REIMBURSEMENT_RATE', 'FULL_METHODOLOGY', +'REIMBURSEMENT_METHODOLOGY', 'REIMBURSEMENT_FEE_SCHEDULE', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION', +'LESSER_OF_LANGUAGE_IND', 'GREATER_OF_LANGUAGE_IND', +'CONTRACT_LOB', 'CONTRACT_NETWORK', 'PRODUCT', 'CONTRACT_PROGRAM', 'CONTRACT_MARKETPLACE_METAL_LEVEL', +'REIMBURSEMENT_PROC_CODES', 'REIMBURSEMENT_PROC_CODE_MODIFIERS', 'REIMBURSEMENT_REVENUE_CODES', 'REIMBURSEMENT_STATUS_INDICATOR_CODES', +'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', +'REIMBURSEMENT_ADMITTYPE_CODES', 'REIMBURSEMENT_EXCEPTION_IND', 'REIMBURSEMENT_DESCRIBE_EXCEPTION', +'RATE_ESCALATOR_IND', 'RATE_ESCALATOR_DESCRIBE', 'LOB_PRICING_TERMS_EFFECTIVE_DATE', 'LOB_PRICING_TERMS_TERMINATION_DATE', +'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMRIB6IY4" @@ -50,7 +59,7 @@ AWS_SECRET_ACCESS_KEY="o+w4z8WyJeQi+l9O9LKAuWNpPogXAJZoS1VSgUJv" AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGAaCXVzLWVhc3QtMiJHMEUCIQCt/8UZmc37xOqSNWy1mhtFzBRR4aJ2jjSI+DZXsL+dgQIgIKSurMIGok1CIpwkWzZetBEFZJuFrChMsnPq1GMl6tEqjAMI+f//////////ARAAGgw2NjAxMzEwNjg3ODIiDG/YpM3sZ+xicr8GcCrgAp5GZeUYm818DYciItJblOTIk0oNaPR73NMXCXHHeHORGR1KJiXTLinqR2OpI/GGX/AsSE6yIpuu2oEQXPUyM0RN9wSpPyVSWDtHmrtj19qsDMTDwLQeTQITBEEKpohhpWNX6y9kSgDF/2QOXxq1uI7mZlXBobdp0XfN7mEvQWBz6UH5EKvc3clkCWiE3GhlhvNsBFnIZSklPyy8jpkxQ1SL0xDeNUWHO9YzWlkrg3dM5T3ZUXk1IW7YiVEUQs/kbG6rVpMSSuINApd3AUNnMY/hBvc32CBOyK/9VgMMI+5hvqd+nUEDUE2zNa/e9YBHqn8JbsQG4cDIQasdD2G1CgDYLvGwJt2x3l+O7w61yfvmknoVOYROKDA2mmA9/XWvhPLUjTDR8Dl3iIcW8YIROVNb9GHMM/t1CX6p5VtiLYbKhPqn7YS44ulnbqbKdRW1MBKTmzzeLQquxZK19JZtBd0wnpzDswY6pgFSmdwIcQBYWdVBtaQNycF5p8041L0kZ4mLe3oqa4DQCHwZMeLA5LZP4c/VSMVHCnNM2rovqyO2A12pLgWvlsXIzitIuL+r405Sk/tJT3BXDuyxKGIS6wvP6eLteSrCCCyNXT+qipoRugVStkObsvBBGT/I535cDlb2DiG6eoPBkwnPvcgJbEJMK2UqNoTJAiuQhP1cmFIV5OratlRC7GjPqnmNxZOQ" # File Paths -LOCAL_PATH = 'data/all_txt_updated/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index ecafcfb..d6d7438 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -68,7 +68,9 @@ def process_file(file_object): combined_df.to_csv(os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False) ################## RUN POSTPROCESSING ################## - combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) + # combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated + combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) + post_processed_combined_df = postprocess.postprocess_results(combined_df) post_processed_combined_df.to_csv(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) print(f"Postprocessing Complete - {filename} ") diff --git a/src/postprocess.py b/src/postprocess.py index 02a8125..6834f5a 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -175,18 +175,18 @@ def post_process_combined(df): pass # Clean the specified code columns - try: - df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) - except Exception as e: - print(f"Error: {e}") - try: - df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) - except Exception as e: - print(f"Error: {e}") - try: - df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) - except Exception as e: - print(f"Error: {e}") + # try: + # df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) + # except Exception as e: + # print(f"Error: {e}") + # try: + # df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) + # except Exception as e: + # print(f"Error: {e}") + # try: + # df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) + # except Exception as e: + # print(f"Error: {e}") # Filter out specific SERVICE rows try: @@ -225,4 +225,7 @@ def post_process_combined(df): def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) + # Order columns + cols = [col for col in config.VALID_COLUMNS if col in combined_df.columns] + [col for col in combined_df.columns if col not in config.VALID_COLUMNS] + combined_df = combined_df[cols] return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 380a367..278a88e 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -3,6 +3,8 @@ import pandas as pd import re import difflib +import config + def sanitize_value(value): """ Sanitizes a given value by handling nulls, lists, and strings to ensure consistency in formatting. @@ -201,7 +203,7 @@ def correct_misplaced_values(df, columns, valid_values_dict): df.at[index, col] = None else: current_value = row[target_col].strip() - if get_closest_match(match, [current_value], 0.8) is None: + if get_closest_match(match, [current_value], config.FUZZY_MATCH_THRESHOLD) is None: df.at[index, 'Corrected_' + target_col] = f"Found {term} in {col} cell" df.at[index, col] = None return df From 4576ad4c63736c50aa4ae1b93106df7fbfa71ff3 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 17 Jun 2024 20:36:15 -0700 Subject: [PATCH 56/78] Update postprocessing --- src/file_processing.py | 2 +- src/postprocess.py | 28 +++++++++++++++------------- src/postprocessingfuncs.py | 2 ++ src/prompts.py | 4 ++-- src/test.py | 29 +++++++++++++++++++---------- 5 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/file_processing.py b/src/file_processing.py index d6d7438..79f2c23 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -68,7 +68,7 @@ def process_file(file_object): combined_df.to_csv(os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False) ################## RUN POSTPROCESSING ################## - # combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated + #combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) post_processed_combined_df = postprocess.postprocess_results(combined_df) diff --git a/src/postprocess.py b/src/postprocess.py index 6834f5a..927b6e4 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -160,20 +160,21 @@ def post_process_combined(df): for field_list in [['CONTRACT_LOB', 'Corrected_LOB', VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', VALID_NETWORKS]]: try: postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) - except: - pass + except Exception as e: + print(f"Postprocessing Error - clean_columns_combined : {e}") try: postprocessingfuncs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) - except: - pass - + except Exception as e: + print(f"Postprocessing Error - clean_columns_combined_fuzzy : {e}") + # Correct misplaced values across columns try: df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) - except: - pass + except Exception as e: + print(f"Postprocessing Error - correct_misplaced_values : {e}") + # Clean the specified code columns # try: # df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) @@ -192,33 +193,33 @@ def post_process_combined(df): try: df = postprocessingfuncs.filter_service_column(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - filter_service_column : {e}") # Move percentages and large numbers to correct columns try: df = postprocessingfuncs.move_percentage_to_rate(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - move_percentage_to_rate : {e}") try: df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - move_large_numbers_to_flat_fee : {e}") try: df = postprocessingfuncs.set_rate_to_zero_if_not_covered(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - set_rate_to_zero_if_not_covered : {e}") try: df = postprocessingfuncs.adjust_reimbursement_rate(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - adjust_reimbursement_rate : {e}") try: clean_df = postprocessingfuncs.clean_pagenumbers(df) except Exception as e: - print(f"Error: {e}") + print(f"Postprocessing Error - clean_pagenumbers : {e}") clean_df = df return clean_df @@ -227,5 +228,6 @@ def postprocess_results(combined_df): combined_df = post_process_combined(combined_df) # Order columns cols = [col for col in config.VALID_COLUMNS if col in combined_df.columns] + [col for col in combined_df.columns if col not in config.VALID_COLUMNS] + #combined_df = combined_df.loc[:, cols] combined_df = combined_df[cols] return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 278a88e..f2a4931 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -224,6 +224,8 @@ def filter_service_column(df): Returns: pandas.DataFrame: A new DataFrame with rows containing specified keywords in the 'SERVICE' column removed. """ + df = df.dropna(subset=['SERVICE']) + keywords = [ 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' ] diff --git a/src/prompts.py b/src/prompts.py index 63dcb93..94a19f0 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -120,12 +120,12 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. -Return only the answer, without using complete sentences. If there is nothing even close to the above options in the full methodology text, simply return 'N/A' +Return only the answer, without using complete sentences. If there is nothing close to the above options in the full methodology text, simply return 'N/A' """ diff --git a/src/test.py b/src/test.py index 1a2205a..8a770c3 100644 --- a/src/test.py +++ b/src/test.py @@ -8,6 +8,8 @@ import ast import os import utils +import postprocess +import postprocessingfuncs import preprocess import table_funcs import prompts @@ -15,14 +17,21 @@ import prompt_funcs import claude_funcs -all_dfs = [] -for file in os.listdir('output'): - if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): - try: - df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') - all_dfs.append(df) - except: - continue +# all_dfs = [] +# for file in os.listdir('output'): +# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): +# try: +# df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') +# all_dfs.append(df) +# except: +# continue -final_df = pd.concat(all_dfs, ignore_index=True) -final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') \ No newline at end of file +# final_df = pd.concat(all_dfs, ignore_index=True) +# final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') + +combined_df = pd.read_csv('output/2017-07-01 Cedars-Sinai Medical Center CSMF CDM AMD MU/combined_results_unprocessed.csv') + +combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) + +post_processed_combined_df = postprocess.postprocess_results(combined_df) + From 6deea0071e82a227889766956eda7db502ba0628 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 17 Jun 2024 21:36:48 -0700 Subject: [PATCH 57/78] Updated primary prompt for examples --- src/file_processing.py | 2 +- src/prompts.py | 2 +- src/test.py | 13 ++++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/file_processing.py b/src/file_processing.py index 79f2c23..d6d7438 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -68,7 +68,7 @@ def process_file(file_object): combined_df.to_csv(os.path.join(output_dir, config.UNPROCESSED_RESULTS_NAME), index=False) ################## RUN POSTPROCESSING ################## - #combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated + # combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) post_processed_combined_df = postprocess.postprocess_results(combined_df) diff --git a/src/prompts.py b/src/prompts.py index 94a19f0..2ada485 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -38,7 +38,7 @@ def BOTTOM_UP_PRIMARY(page, payer): The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections identified as examples - these must be omitted from output. If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. diff --git a/src/test.py b/src/test.py index 8a770c3..0d04e64 100644 --- a/src/test.py +++ b/src/test.py @@ -29,9 +29,16 @@ import claude_funcs # final_df = pd.concat(all_dfs, ignore_index=True) # final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') -combined_df = pd.read_csv('output/2017-07-01 Cedars-Sinai Medical Center CSMF CDM AMD MU/combined_results_unprocessed.csv') -combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) +input_dict = utils.read_input() -post_processed_combined_df = postprocess.postprocess_results(combined_df) +(filename, contract_text) = list(input_dict.items())[0] +contract_text = preprocess.clean_newlines(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) + +bu_results = prompt_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000) # Returns list of dictionaries +print(bu_results) + From 4db7260edc462fd70db665e2a6a61f1734a4da38 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Tue, 18 Jun 2024 12:38:40 -0700 Subject: [PATCH 58/78] Updated exception/escalator --- src/config.py | 6 +++--- src/postprocessingfuncs.py | 2 +- src/preprocess.py | 2 +- src/prompt_funcs.py | 2 +- src/prompts.py | 8 +++++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/config.py b/src/config.py index ce64d2c..0e3dd90 100644 --- a/src/config.py +++ b/src/config.py @@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMRIB6IY4" -AWS_SECRET_ACCESS_KEY="o+w4z8WyJeQi+l9O9LKAuWNpPogXAJZoS1VSgUJv" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEGAaCXVzLWVhc3QtMiJHMEUCIQCt/8UZmc37xOqSNWy1mhtFzBRR4aJ2jjSI+DZXsL+dgQIgIKSurMIGok1CIpwkWzZetBEFZJuFrChMsnPq1GMl6tEqjAMI+f//////////ARAAGgw2NjAxMzEwNjg3ODIiDG/YpM3sZ+xicr8GcCrgAp5GZeUYm818DYciItJblOTIk0oNaPR73NMXCXHHeHORGR1KJiXTLinqR2OpI/GGX/AsSE6yIpuu2oEQXPUyM0RN9wSpPyVSWDtHmrtj19qsDMTDwLQeTQITBEEKpohhpWNX6y9kSgDF/2QOXxq1uI7mZlXBobdp0XfN7mEvQWBz6UH5EKvc3clkCWiE3GhlhvNsBFnIZSklPyy8jpkxQ1SL0xDeNUWHO9YzWlkrg3dM5T3ZUXk1IW7YiVEUQs/kbG6rVpMSSuINApd3AUNnMY/hBvc32CBOyK/9VgMMI+5hvqd+nUEDUE2zNa/e9YBHqn8JbsQG4cDIQasdD2G1CgDYLvGwJt2x3l+O7w61yfvmknoVOYROKDA2mmA9/XWvhPLUjTDR8Dl3iIcW8YIROVNb9GHMM/t1CX6p5VtiLYbKhPqn7YS44ulnbqbKdRW1MBKTmzzeLQquxZK19JZtBd0wnpzDswY6pgFSmdwIcQBYWdVBtaQNycF5p8041L0kZ4mLe3oqa4DQCHwZMeLA5LZP4c/VSMVHCnNM2rovqyO2A12pLgWvlsXIzitIuL+r405Sk/tJT3BXDuyxKGIS6wvP6eLteSrCCCyNXT+qipoRugVStkObsvBBGT/I535cDlb2DiG6eoPBkwnPvcgJbEJMK2UqNoTJAiuQhP1cmFIV5OratlRC7GjPqnmNxZOQ" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFY5DMLIK" +AWS_SECRET_ACCESS_KEY="IN2w1JrXU4f7rKR2IbiOwCZNUrgls3ZK/65w8cCS" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHMaCXVzLWVhc3QtMiJHMEUCIEhqvUNWd6r+bPwIIFkg242wxJGTn9k7qWiWVZSXn4dLAiEAr01immSRrcDW1oqW0QfN3gqivPNQV8JkTgLipIW0rpsqgwMIHBAAGgw2NjAxMzEwNjg3ODIiDJCiH+aROTYN8KdY2CrgAmCbjJwVL63njz5qxkOtexudA6CZ7Kz5BGpaA2WbdyHHc8j/UB/44MfOAxrTeLiP6w12qpYMc7U5GdZIULvNwWxJhJmnOHwLmLU0XuOtc52l3VCQmXLLMj1Q/dVPm+MVFgTM4RLTTNRGRE9boX1MP92pZlUZ/sjSKBl2ELfiJkrgzOkS3kYADmICJ0wWU1Ndqpi8sGTs8JHCyHCj3TSkJVaeCcwyvWssZHrgOv7/MUdgeHkiJx3m/MJNdOmxyZzr7mWtAFgJMODr8zM/vAPWogzcD0MEoMpa98ag9fcC6nE9us4KpiIIGAJNzXX6YpjkxoU+/tdXwMLsbkqmnqblz3zvB6jEyqTLMeEMYfexlzA7bBrq6Ooh2+nvTpFwF3E4RTdXsOdjgiw2nTwfjBYFT4cAGz9rQl2hZCLdhEhwZu5dmF/oKn6Nd5eec+KV0+Ep/SPor9p/uQh597zDDWt1BMwwurvHswY6pgFAChD2esP7NSzOQ+dIKA0CTZuNwdK7mjfkjtGEPsjDJBF3ueD4CybkHrqM2Ud6xhI8LwZU6DiIdUeqFXQHawdF1wDegBgy1Tymv1nCOZYCyTbW6ihJ+L4oJsr8XJwKpUhRgQQ8ECscAWFGRMN3zPt6T+OAcxJvjnKtdQO+vOCJNMzkr1yb7cuEoUcWPSwFrVkZg5iKP4T9DLLDCfGZUiV7AND6trJ+" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index f2a4931..8dd2d64 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -227,7 +227,7 @@ def filter_service_column(df): df = df.dropna(subset=['SERVICE']) keywords = [ - 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST' + 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program' ] pattern = '|'.join(keywords) df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] diff --git a/src/preprocess.py b/src/preprocess.py index 89354c7..54a6b90 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -79,7 +79,7 @@ def chunk_text(text_dict): Returns: dict: A new dictionary where each key is a string representing the range of pages in a chunk, and each value is the concatenated text of those pages. """ - special_pages = {int(page): text for page, text in text_dict.items() if '%' in text or '$' in text} + special_pages = {int(page): text for page, text in text_dict.items() if (('%' in text) or ('$' in text) or ('percent' in text))} page_numbers = sorted(special_pages.keys()) chunk_page_numbers = [] for page in page_numbers: diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 0719d92..686b52d 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -61,7 +61,7 @@ def run_bottom_up_primary(text_dict, tokens): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): - if page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number]): + if page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number] or 'percent' in text_dict[page_number].lower()): # Run Primary prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) diff --git a/src/prompts.py b/src/prompts.py index 2ada485..c7b4b14 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -38,7 +38,7 @@ def BOTTOM_UP_PRIMARY(page, payer): The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections identified as examples - these must be omitted from output. +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections explicitly identified as examples - these must be omitted from output. If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. @@ -48,7 +48,7 @@ Here are the attributes to be included in each dictionary, and instructions on h 'SERVICE' : What is the Service that is being reimbursed? Use your best judgement and knowledge of the healthcare industry to answer. Use only exact text from the document. Do not leave this field N/A. 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. -'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methology. +'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. @@ -94,7 +94,9 @@ Your job is to identify if the listed reimbursement contains some sort of except Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): REIMBURSEMENT_EXCEPTION_IND: If the listed reimbursement contains some sort of exception (like for certain codes, hospitals, or other situations), return 'Y'. Otherwise, 'N' REIMBURSEMENT_DESCRIBE_EXCEPTION: If the listed reimbursement contains some sort of exception, describe it here. Try to use exact words from the text only. If there is none, just write 'N/A' -RATE_ESCALATOR_IND: If the listed reimbursement represents a year-over-year increase, return 'Y'. Otherwise, return 'N'. +RATE_ESCALATOR_IND: Does the listed reimbursement include annual rate escalators for year-over-year rate increases? If yes, return 'Y'. Otherwise, 'N'. +RATE_ESCALATOR_BASIS: If contract does include annual rate escalators, on what are they based? Do not be overly verbose in your answer. +RATE_ESCALATOR_YEARLY_PERCENT_INCREASE: What is the percent value of increase or decrease year-over-year? If a decrease, return a negative percentage. Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ From a416da1a4895509fded5fc565e41b0241f6dda2a Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 19 Jun 2024 13:06:50 -0700 Subject: [PATCH 59/78] Updated billed charges implied % --- src/config.py | 6 +- src/postprocessingfuncs.py | 3 +- src/preprocess.py | 65 +++++++--- src/prompt_funcs.py | 11 +- src/prompts.py | 6 +- src/table_funcs.py | 21 ++-- src/test.py | 54 ++++++++- src/textract_template.py | 236 +++++++++++++++++++++++++++++++++++++ 8 files changed, 354 insertions(+), 48 deletions(-) create mode 100644 src/textract_template.py diff --git a/src/config.py b/src/config.py index 0e3dd90..bf22093 100644 --- a/src/config.py +++ b/src/config.py @@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFY5DMLIK" -AWS_SECRET_ACCESS_KEY="IN2w1JrXU4f7rKR2IbiOwCZNUrgls3ZK/65w8cCS" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHMaCXVzLWVhc3QtMiJHMEUCIEhqvUNWd6r+bPwIIFkg242wxJGTn9k7qWiWVZSXn4dLAiEAr01immSRrcDW1oqW0QfN3gqivPNQV8JkTgLipIW0rpsqgwMIHBAAGgw2NjAxMzEwNjg3ODIiDJCiH+aROTYN8KdY2CrgAmCbjJwVL63njz5qxkOtexudA6CZ7Kz5BGpaA2WbdyHHc8j/UB/44MfOAxrTeLiP6w12qpYMc7U5GdZIULvNwWxJhJmnOHwLmLU0XuOtc52l3VCQmXLLMj1Q/dVPm+MVFgTM4RLTTNRGRE9boX1MP92pZlUZ/sjSKBl2ELfiJkrgzOkS3kYADmICJ0wWU1Ndqpi8sGTs8JHCyHCj3TSkJVaeCcwyvWssZHrgOv7/MUdgeHkiJx3m/MJNdOmxyZzr7mWtAFgJMODr8zM/vAPWogzcD0MEoMpa98ag9fcC6nE9us4KpiIIGAJNzXX6YpjkxoU+/tdXwMLsbkqmnqblz3zvB6jEyqTLMeEMYfexlzA7bBrq6Ooh2+nvTpFwF3E4RTdXsOdjgiw2nTwfjBYFT4cAGz9rQl2hZCLdhEhwZu5dmF/oKn6Nd5eec+KV0+Ep/SPor9p/uQh597zDDWt1BMwwurvHswY6pgFAChD2esP7NSzOQ+dIKA0CTZuNwdK7mjfkjtGEPsjDJBF3ueD4CybkHrqM2Ud6xhI8LwZU6DiIdUeqFXQHawdF1wDegBgy1Tymv1nCOZYCyTbW6ihJ+L4oJsr8XJwKpUhRgQQ8ECscAWFGRMN3zPt6T+OAcxJvjnKtdQO+vOCJNMzkr1yb7cuEoUcWPSwFrVkZg5iKP4T9DLLDCfGZUiV7AND6trJ+" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXPGBTIYHY" +AWS_SECRET_ACCESS_KEY="2zV8fp+u6LW8qpr75ZKEG/kWnvbrgQYo6Rg7MDln" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJIMEYCIQDZXOwwKnirwI7+o3716cEa6yVDLJiTu3Kzz7kPGooFuAIhAOGrTFPBigoNNA1x1OGJSMZpwGPLmiMxpJIpWbzYxyZbKoMDCDMQABoMNjYwMTMxMDY4NzgyIgwu4eVW0nZEUNi69Pcq4AJw9r1cQpF2vnIdXzmMp18dm9+oB7IZQMFQ9W3LR8R90PY3zk0HzY1UGaTIKnSPsXAdl5bIW2nb7a79RPoiQr4Ro70qM4G0WzHjaP8ottl/YCpiEhFzmNhBamT9lutBSQIyjumzzYa6LX76T2K9zSzumG9+2ubPKLtqLO91d9MV6EDCOt4hBsFM46WT4YBsVxEBQ3ABIsRAVkG2Ff8jgQ01nygd+IYqFjjQBndCxjSqcxM03+MQDBxsaRjNdwbJ/lRcrJpD7PeunPG1d+BIg9u4RfrHuTjJo8wVaZAKUlqjeRPWtS5woCiUEfGsSykWJp+N+x0cv+/tiVxzU1lPSwiLkwTQ5FotFr4dfA7q3NaqkKZ5YUA3GrKrATmsD1UlCRuCx7/jvMgCABFuCOERD7nrY57rgLY167PvKY7/8Fi6mpBgH8xHHtYoICfBQRMz2dLTwHVFC5R4pdA+f7ysF9MZMP+tzLMGOqUBCkhJSij99lLWdoOQQTn67bJHGMFcbXFHkeGnY7mk0jaLnACuodUMgvHPPuHDR6YsIEKk7Yw70pjMDoiXcbDgjbKdYquX86iXKMgv8RP6blXJJSgxsrabyBHTMWITMZDe530iQCoRkwiAo++RQanljFMah0LPCZpfqq6lpeLtws7Ya6SmvBUT6BBrhVDn0FZhKxnuXGJ58tnqaGA2eSYddht5lcDs" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 8dd2d64..eae72e7 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -227,7 +227,8 @@ def filter_service_column(df): df = df.dropna(subset=['SERVICE']) keywords = [ - 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program' + 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED', + 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES', ] pattern = '|'.join(keywords) df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] diff --git a/src/preprocess.py b/src/preprocess.py index 54a6b90..caddb66 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -40,7 +40,7 @@ def split_text(text): return text_dict -def highlight_rates(text_dict): +def highlight_rates(text): """ Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary. @@ -54,15 +54,13 @@ def highlight_rates(text_dict): Returns: dict: The updated dictionary with rate-related terms highlighted in the text of each page. """ - for page, text in text_dict.items(): - words = text.split() - highlighted_words = [] - for word in words: - if '%' in word or '$' in word: - word = f'>>>{word}<<<' - highlighted_words.append(word) - text_dict[page] = ' '.join(highlighted_words) - return text_dict + words = text.split() + highlighted_words = [] + for word in words: + if '%' in word or '$' in word: + word = f'>>>{word}<<<' + highlighted_words.append(word) + return ' '.join(highlighted_words) def chunk_text(text_dict): @@ -97,7 +95,7 @@ def chunk_text(text_dict): return chunk_dict -def clean_billed_charges(text): +def clean_billed_charges(contract_text): """ Cleans occurrences of 'billed charges' in a text by replacing them with '100% of billed charges' unless preceded by a percentage. @@ -111,11 +109,40 @@ def clean_billed_charges(text): Returns: str: The cleaned text with appropriate replacements made for 'billed charges'. """ - def replace(match): - start_pos = match.start() - pre_text = text[max(0, start_pos-15):start_pos] - if '%' in pre_text: - return match.group() - else: - return r'100% of billed charges' - return re.sub(r'\bbilled charges\b', replace, text, flags=re.IGNORECASE) + + substrings = [ + "Physician's Billed Charges", + "Provider's Billed Charges", + "Allowable Billed Charges", + "Hospital's Billed Charges", + "Physician's Charges", + "Provider's Charges", + "Allowable Charges", + "Hospital's Charges", + "Billed Charges" + ] + max_substring_length = max([len(s) for s in substrings]) + + def find_substring_indices(contract_text, s): + indices = [] + lower_contract_text = contract_text.lower() + lower_s = s.lower() + index = lower_contract_text.find(lower_s) + while index != -1: + indices.append(index) + index = lower_contract_text.find(lower_s, index + 1) + return indices + + for s in substrings: + indices = find_substring_indices(contract_text, s) + index_adder = 0 + if indices: + for i in indices: + index = i+index_adder + end_index = index + max_substring_length + match_part = contract_text[index:end_index] + previous = contract_text[max(0, index-30):index] + if '%' not in previous: + contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:] + index_adder += 9 + return contract_text.replace(' ', ' ') diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 686b52d..6831c42 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -107,13 +107,10 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): # Bottom Up Methodology if config.RUN_METHODOLOGY: - if d['REIMBURSEMENT_FLAT_FEE'] == 'N/A': - prompt = prompts.BOTTOM_UP_METHODOLOGY(d) - methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) - d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer - else: - d['REIMBURSEMENT_METHODOLOGY'] = 'Flat Fee' - + prompt = prompts.BOTTOM_UP_METHODOLOGY(d) + methodology_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=100) + d['REIMBURSEMENT_METHODOLOGY'] = methodology_answer + # # Bottom Up FS if config.RUN_FS: prompt = prompts.BOTTOM_UP_FS(d) diff --git a/src/prompts.py b/src/prompts.py index c7b4b14..ae8698d 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -38,7 +38,9 @@ def BOTTOM_UP_PRIMARY(page, payer): The preceding text is one page of a contract between Payer {payer} and a provider in their network. Your job is to extract attributes related to the reimbursement of different services and specialties. -The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). Some values are found in sections explicitly identified as examples - these must be omitted from output. +The reimbursement values will be identified with >>> <<< indicators (e.g. >>>105%<<<). Make sure there is at least one dictionary object for EVERY reimbursement value seen (either % or $). + +Some values are found in sections explicitly identified as examples - these must be omitted from output. Other values might be related to liability - these must also be excluded. If any of the attributes are not found, return N/A. For all attributes, only write what is written on the page. Do not make up new phrases or words. @@ -122,7 +124,7 @@ def BOTTOM_UP_METHODOLOGY(d): Your job is to identify what the concise methodology is based on the full methodology. Choose the closest option from the following: -Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, IME/DME Amount, Cost to Charge Ratio, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate +Allowed Amount, Allowable Charges, Fee Schedule, Medicare Fee Schedule, Medicare Allowed Amount, Medicaid Fee Schedule, Medicaid Allowed Amount, Provider's Charges, Billed Charges, Normal fee for such service, IME/DME Amount, Cost to Charge Ratio, Cost plus %, MSRP, Per Unit, Per Diem, Per Visit, Per Member, Per Member Per Month, Per Hour, Case Rate, Contracted Rate For 'Allowed Amount' and 'Fee Schedule', use the more specific answer (Medicare/Medicaid) if applicable. For the 'Per ___' values, note that this list is not necessarily exhaustive. It could be anything like 'Per ___'. diff --git a/src/table_funcs.py b/src/table_funcs.py index 741a4a8..d6e80e1 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -73,21 +73,22 @@ def align_and_format_tables(text_dict): """ aligned_text_dict = {} for key, text in text_dict.items(): + print(key) aligned_text = text if 'Table Start' in text: table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) for table_text in table_texts: - try: - # Extract pretable text - pretable = table_text.split('{')[0].strip() - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + # Extract pretable text + pretable = table_text.split('{')[0].strip() + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) - except: - table_formatted = table_text + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + + table_formatted = table_text # Align if text.count(pretable) == 2: # One table diff --git a/src/test.py b/src/test.py index 0d04e64..981b967 100644 --- a/src/test.py +++ b/src/test.py @@ -1,4 +1,5 @@ import pandas as pd +import numpy as np import re import json import csv @@ -31,14 +32,55 @@ import claude_funcs input_dict = utils.read_input() - (filename, contract_text) = list(input_dict.items())[0] -contract_text = preprocess.clean_newlines(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) -text_dict = preprocess.highlight_rates(text_dict) +def clean_billed_charges(contract_text): + substrings = [ + "Physician's Billed Charges", + "Provider's Billed Charges", + "Allowable Billed Charges", + "Hospital's Billed Charges", + "Physician's Charges", + "Provider's Charges", + "Allowable Charges", + "Hospital's Charges", + "Billed Charges" + ] + max_substring_length = np.max([len(s) for s in substrings]) -bu_results = prompt_funcs.run_bottom_up_primary({'5' : text_dict['5']}, 4000) # Returns list of dictionaries + def find_substring_indices(contract_text, s): + indices = [] + lower_contract_text = contract_text.lower() + lower_s = s.lower() + index = lower_contract_text.find(lower_s) + while index != -1: + indices.append(index) + index = lower_contract_text.find(lower_s, index + 1) + return indices + + for s in substrings: + indices = find_substring_indices(contract_text, s) + index_adder = 0 + if indices: + for i in indices: + index = i+index_adder + end_index = index + max_substring_length + match_part = contract_text[index:end_index] + previous = contract_text[max(0, index-30):index] + if '%' not in previous: + contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:] + index_adder += 9 + return contract_text.replace(' ', ' ') + +contract_text = preprocess.clean_newlines(contract_text) +contract_text = clean_billed_charges(contract_text) +contract_text = preprocess.highlight_rates(contract_text) +text_dict = preprocess.split_text(contract_text) +# text_dict = table_funcs.align_and_format_tables(text_dict) + + +print(text_dict['23']) + +bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries print(bu_results) diff --git a/src/textract_template.py b/src/textract_template.py new file mode 100644 index 0000000..510d8e1 --- /dev/null +++ b/src/textract_template.py @@ -0,0 +1,236 @@ + +""" +This script was written to extract .txt files from pdf for adhoc client runs. +Ensure you have permissions for API gateway, S3, Lambda function and SQS to execute this (DEVELOPER & above roles in DEV & UAT for Doczy should suffice) +If you have Analyst / Test roles and need the permissions to be elevated or policies to be updated then please ask Sannan Iqbal to make the changes. + +The flow to execute the pipeline is: +Create batch using api req +Add files to the newly created batch along with the batch id as the tag +Construct payload to trigger pipeline +Post request to trigger pipeline with payload +The output text file can be found in the client_bucket/contract_text_file/batch_123456 +And the final LLM parsed outputs can be found in client_bucket/final_output/batch_123456 +""" + + + + +import os +import boto3 +import requests +import json +import boto3 +from datetime import datetime +import os +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import get_credentials +from botocore.session import Session +import time + +# Function to upload files to S3 +def upload_files_to_s3(directory, bucket, batch_id): + contract_list = [] + for filename in os.listdir(directory): + if filename.endswith('.pdf'): + file_path = os.path.join(directory, filename) + s3_key = f'contracts-landing-zone/{batch_id}/{filename}' + + # Upload file to S3 + s3_client.upload_file(file_path, bucket, s3_key) + print(f'Uploaded {filename} to S3 bucket\n') + + # Add tags to the uploaded file + s3_client.put_object_tagging( + Bucket=bucket, + Key=s3_key, + Tagging={ + 'TagSet': [ + { + 'Key': 'BatchId', + 'Value': batch_id + } + ] + } + ) + print(f'Added tags to {filename}\n') + # Add file details to contract list + # For now we can leave it as is since only A, C have been operationalized + contract_list.append({ + "contract_name": filename, + "groups": [ + "A", # This needs to be dynamic in UI 1 based on what group has been selected + "C" + ], + "contract_source_path": s3_key + }) + return contract_list + + + + +def create_batch(client_bucket, create_batch_url): + myobj = { "client-bucket-name": client_bucket } + + # Call create batch API endpoint + response = requests.post(create_batch_url, json = myobj) + if response.status_code >= 200 and response.status_code < 300: + try: + new_batch_id = json.loads(json.loads(response.text)['body'])['batch_id'] + landing_zone = json.loads(json.loads(response.text)['body'])['landing_zone'] + except: + print(myobj) + print(response.text) + new_batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + else: + print(response.text) + new_batch_id = 'failed_cases' + landing_zone = 'contracts_landing_zone' + return new_batch_id, landing_zone + + + + +def list_filtered_files(bucket_name, prefix, start_date, end_date, profile_name): + """ + List all files in an S3 bucket filtered by date range. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - prefix: str, the prefix (folder) in the S3 bucket + - start_date: str, the start date in ISO 8601 format + - end_date: str, the end date in ISO 8601 format + - profile_name: str, the AWS profile name + + Returns: + - list of str: the keys of the filtered files + """ + # Parse the dates + start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00')) + end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00')) + + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client('s3') + + # List objects in the bucket with the specified prefix + paginator = s3_client.get_paginator('list_objects_v2') + page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix) + + # Filter files by date + filtered_files = [] + for page in page_iterator: + if 'Contents' in page: + for obj in page['Contents']: + last_modified = obj['LastModified'] + if start_date <= last_modified <= end_date: + filtered_files.append(obj['Key']) + + return filtered_files + + + +def download_files(bucket_name, file_keys, profile_name, local_directory): + """ + Download files from an S3 bucket. + + Parameters: + - bucket_name: str, the name of the S3 bucket + - file_keys: list of str, the keys of the files to download + - profile_name: str, the AWS profile name + - local_directory: str, the local directory to download files to + """ + # Initialize a session using the specified profile + session = boto3.Session(profile_name=profile_name) + s3_client = session.client('s3') + + # Ensure the local directory exists + if not os.path.exists(local_directory): + os.makedirs(local_directory) + + # Download each file + for key in file_keys: + file_name = os.path.basename(key) # Get only the file name from the key + local_file_path = os.path.join(local_directory, file_name) + print(f"Downloading {key} to {local_file_path}") + s3_client.download_file(bucket_name, key, local_file_path) + + print("Download completed.") + + + +if __name__ == "__main__": + # Define your variables + s3_bucket = 'doczyai-use2-u-cn1-s3-textract-processing-001' + # batch_id = 'batch_100524101551' + client_name = 'Priority Health' + username = 'ADHOC USER' + + # These endpoints are in UAT, please change them to DEV if there are access issues with UAT + api_endpoint = 'https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/trigger-pipeline' + create_batch_url = "https://29gm8cek03.execute-api.us-east-2.amazonaws.com/dev/create-batch" + + + + session = boto3.Session(profile_name='temp_cred') # Change the profile name to the one you have in your .aws/credentials file + s3_client = session.client('s3') + + + + # Use current directory as PDF directory + pdf_directory = "C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\UAT- test East Paris Surgical Center copy" + + + # Create batch + batch_id, landing_zone = create_batch(s3_bucket, create_batch_url) + print(f"Batch ID: {batch_id}") + print(f"Landing Zone: {landing_zone}") + # batch_id = 'batch_110624213433' + # landing_zone = 'contracts_landing_zone/batch_110624213433/' + + if batch_id == 'failed_cases': + print("Batch creation failed. Exiting...") + exit() + # Upload files and get contract list + contract_list = upload_files_to_s3(pdf_directory, s3_bucket, batch_id) + + # Create JSON object + data = { + "s3_bucket": s3_bucket, + "batch_id": batch_id, + "client_name": client_name, + "username": username, + "contract_list": contract_list + } + + # Make POST request to API + response = requests.post(api_endpoint, json=data) + + + + + + # Print response + print(response.status_code) + print(response.json()) + + ################## + # Get text files + + prefix = f'contract-text-file/{batch_id}/' # if you have a specific prefix (folder) in your bucket + + # These dates are to filter the contracts in case there are older contracts in the same batch + start_date = '2024-06-04T00:00:00Z' # ISO 8601 format + end_date = '2024-06-14T23:59:59Z' # ISO 8601 format + profile_name = 'temp_cred' + local_directory = 'C:\\Doczy\\Priority Health\\For_Textract\\For_Textract\\text otuput' # local directory to save files + + # List filtered files + time.sleep(30) # Waiting for the text files to be generated, this may take longer and files may not be available after 30 seconds sometimes + filtered_files = list_filtered_files(s3_bucket, prefix, start_date, end_date, profile_name) + print(f"Filtered files: {len(filtered_files)}") + + # Download files + download_files(s3_bucket, filtered_files, profile_name, local_directory) \ No newline at end of file From 518e3aa82ecec771d661ffaec7cd22ebb219ed74 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 19 Jun 2024 15:39:27 -0700 Subject: [PATCH 60/78] Updated table preprocessing --- src/config.py | 8 ++-- src/file_processing.py | 1 + src/preprocess.py | 18 +++++---- src/table_funcs.py | 84 +++++++++++++++++++++++++++++++++--------- src/test.py | 62 +++++++------------------------ 5 files changed, 95 insertions(+), 78 deletions(-) diff --git a/src/config.py b/src/config.py index bf22093..f244c2d 100644 --- a/src/config.py +++ b/src/config.py @@ -54,12 +54,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXPGBTIYHY" -AWS_SECRET_ACCESS_KEY="2zV8fp+u6LW8qpr75ZKEG/kWnvbrgQYo6Rg7MDln" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJIMEYCIQDZXOwwKnirwI7+o3716cEa6yVDLJiTu3Kzz7kPGooFuAIhAOGrTFPBigoNNA1x1OGJSMZpwGPLmiMxpJIpWbzYxyZbKoMDCDMQABoMNjYwMTMxMDY4NzgyIgwu4eVW0nZEUNi69Pcq4AJw9r1cQpF2vnIdXzmMp18dm9+oB7IZQMFQ9W3LR8R90PY3zk0HzY1UGaTIKnSPsXAdl5bIW2nb7a79RPoiQr4Ro70qM4G0WzHjaP8ottl/YCpiEhFzmNhBamT9lutBSQIyjumzzYa6LX76T2K9zSzumG9+2ubPKLtqLO91d9MV6EDCOt4hBsFM46WT4YBsVxEBQ3ABIsRAVkG2Ff8jgQ01nygd+IYqFjjQBndCxjSqcxM03+MQDBxsaRjNdwbJ/lRcrJpD7PeunPG1d+BIg9u4RfrHuTjJo8wVaZAKUlqjeRPWtS5woCiUEfGsSykWJp+N+x0cv+/tiVxzU1lPSwiLkwTQ5FotFr4dfA7q3NaqkKZ5YUA3GrKrATmsD1UlCRuCx7/jvMgCABFuCOERD7nrY57rgLY167PvKY7/8Fi6mpBgH8xHHtYoICfBQRMz2dLTwHVFC5R4pdA+f7ysF9MZMP+tzLMGOqUBCkhJSij99lLWdoOQQTn67bJHGMFcbXFHkeGnY7mk0jaLnACuodUMgvHPPuHDR6YsIEKk7Yw70pjMDoiXcbDgjbKdYquX86iXKMgv8RP6blXJJSgxsrabyBHTMWITMZDe530iQCoRkwiAo++RQanljFMah0LPCZpfqq6lpeLtws7Ya6SmvBUT6BBrhVDn0FZhKxnuXGJ58tnqaGA2eSYddht5lcDs" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXKP6R66NS" +AWS_SECRET_ACCESS_KEY="O4McuRggESDNl7omTvxVSiRjwECumUk8NhZpVFL3" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEI///////////wEaCXVzLWVhc3QtMiJIMEYCIQCMXW8IJ73YApb3JRLD09atRBInARf7L2bh5AU5HzD/GQIhALOzv8YiTkGXStNjWJ6DGnosjCMa2sRKFSs7oN3Gfp8eKoMDCDgQABoMNjYwMTMxMDY4NzgyIgxCEZzpAN+sOqLAj5Iq4AIrLejZWlEcl0D/35ryiNBifF4PgYNXPogUsgk7fjYNDnrG2xzVVdjALoOo+OZm3vLF+7X5HB5Foc3DTfuu8dTCGnGO2p4TPRXJ2Ftsh9ichNELsLVTgY39tBOOf7YMpQPaaNWHgJRZlAJ7g5+Xy8FzOL8VuiikxSI4UMKYbUMml4ZdQMNXsjqF+RbBDdWzodRnmirdhXhVBBUhpRIkRIioYizWjS+A51Y9Irmq4SnevG6/yLmuk2BV5Bx9uFeuvkOpUM/AJ8VUOUEff7cIKPhO6YG0sNFsJG4V00JVE9ORA8RqL5O46rvDKX8Ce3iuXAOr8TrQEjJ1FT8epzoy2EH0OhxTfpFTgmlo+X929sl3BeA6ZsHye8T8s42Y76MVQvAE5yEg6TtmLzfsDfecUe2z0SHilcWof9o9tswlWOUCIPXMbmERK7yFTTPax3JvyFA3lWRXCPlD/XpZgt26jdjjMJm6zbMGOqUBc+DbSEICCsra29nlXSHtA0YqQNwJ/bxTpBnAYJBb/1/0su2FVbIBYNfAcDty4ox4wB/HkUhpv2AEGGQIko78Y70ws6C4qD0zum74EPHG5VB1K2SU5npO5J8a6p5Jckj7i3n8Gb8nnB0UBvMyKUF4RkSbQJVB7ctgpoROW5IDu+7S7r/edt0yKZ8hpu2ixwKZvg7J6KYegCsbPT6xDOwb1+bAqPPl" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/all_txt_updated_batch_1_numbered/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index d6d7438..73e0cbb 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -45,6 +45,7 @@ def process_file(file_object): ################## PREPROCESS ################## contract_text = preprocess.clean_newlines(contract_text) + contract_text = preprocess.clean_billed_charges(contract_text) text_dict = preprocess.split_text(contract_text) text_dict = table_funcs.align_and_format_tables(text_dict) text_dict = preprocess.highlight_rates(text_dict) diff --git a/src/preprocess.py b/src/preprocess.py index caddb66..5298feb 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -40,7 +40,7 @@ def split_text(text): return text_dict -def highlight_rates(text): +def highlight_rates(text_dict): """ Highlights rate-related terms (percentages and dollar amounts) in the text of each page within a dictionary. @@ -54,13 +54,15 @@ def highlight_rates(text): Returns: dict: The updated dictionary with rate-related terms highlighted in the text of each page. """ - words = text.split() - highlighted_words = [] - for word in words: - if '%' in word or '$' in word: - word = f'>>>{word}<<<' - highlighted_words.append(word) - return ' '.join(highlighted_words) + for page, text in text_dict.items(): + words = text.split() + highlighted_words = [] + for word in words: + if '%' in word or '$' in word: + word = f'>>>{word}<<<' + highlighted_words.append(word) + text_dict[page] = ' '.join(highlighted_words) + return text_dict def chunk_text(text_dict): diff --git a/src/table_funcs.py b/src/table_funcs.py index d6e80e1..411873f 100644 --- a/src/table_funcs.py +++ b/src/table_funcs.py @@ -2,6 +2,43 @@ import re import ast +def clean_tables(text): + def replace_colon(text): + # Define a function to use in re.sub to check the context of the match + def replacer(match): + # Extract the character after ':' to check if it's '[' + following_text = match.group(1) + if following_text.strip().startswith('['): + return match.group(0) # Return the original match (':' and whatever follows) + else: + return '' + match.group(1) # Replace ':' with ';' and return the rest + + # Use a regular expression to find ':' and the text that follows + pattern = r':(\s*.)' + replaced_text = re.sub(pattern, replacer, text) + return replaced_text + + def remove_unquoted_brackets(text): + # Define a function to check the content between brackets + def replacer(match): + content = match.group(1) + if "'" not in content and (len(content) > 0): + return content # Return just the content without brackets + else: + return match.group(0) # Return the original match if it contains a quote + + + # Use a regular expression to find brackets and the text within + pattern = r'\[([^\[\]]*?)\]' + cleaned_text = re.sub(pattern, replacer, text) + return cleaned_text + + # Clean tables + text = text.strip('{}') # Remove brackets + text = replace_colon(text) + text = remove_unquoted_brackets(text) + return text + def convert_to_dict(table_text): """ Converts a formatted table text into a dictionary where each key maps to a list of values. @@ -15,7 +52,8 @@ def convert_to_dict(table_text): Returns: tuple: A dictionary with keys mapping to lists of values, and an integer representing the number of elements in each list. """ - table_text = table_text.strip('{}') + table_text = clean_tables(table_text) + # print(table_text, '\n') table_text_list = table_text.split(']') table_text_list = [item for item in table_text_list if len(item) > 0] @@ -23,10 +61,21 @@ def convert_to_dict(table_text): num_elements = 0 for key_value_text in table_text_list: key = key_value_text.split(':')[0].strip(', \'') - value = ':'.join(key_value_text.split(':')[1:]).strip()+']' - value_list = ast.literal_eval(value) - final_dict[key] = value_list - num_elements = len(value_list) + try: + value = ':'.join(key_value_text.split(':')[1:]).strip()+']' + # print(value, '\n') + if len(value) > 1: + value_list = ast.literal_eval(value) + final_dict[key] = value_list + num_elements = len(value_list) + except: + value = ':'.join(key_value_text.split(':')[1:]).strip()+"']" + # print(value, '\n') + if len(value) > 1: + value_list = ast.literal_eval(value) + final_dict[key] = value_list + num_elements = len(value_list) + return final_dict, num_elements @@ -73,23 +122,24 @@ def align_and_format_tables(text_dict): """ aligned_text_dict = {} for key, text in text_dict.items(): - print(key) aligned_text = text if 'Table Start' in text: table_texts = re.findall(r'-------Table Start--------(.*?)-------Table End--------', text, re.DOTALL) for table_text in table_texts: + # print(table_text, '\n') - # Extract pretable text - pretable = table_text.split('{')[0].strip() - # Extract and format table text - #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" - table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + try: + # Extract pretable text + pretable = table_text.split('{')[0].strip() + # Extract and format table text + #table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0].replace("'", '"') + "}" + table_only = "{" + table_text.split('{', 1)[1].rsplit('}', 1)[0] + "}" + + table, table_size = convert_to_dict(table_only) + table_formatted = format_table(table, table_size) + except: + table_formatted = table_text - table, table_size = convert_to_dict(table_only) - table_formatted = format_table(table, table_size) - - table_formatted = table_text - # Align if text.count(pretable) == 2: # One table # Remove original table @@ -98,7 +148,7 @@ def align_and_format_tables(text_dict): aligned_text = aligned_text.replace(pretable, pretable + table_formatted) else: - aligned_text = aligned_text.replace(table_text, pretable + table_formatted) + aligned_text = aligned_text.replace(table_text, pretable + ' ' + table_formatted) aligned_text_dict[key] = aligned_text.replace("-------Table Start--------", "").replace("-------Table End--------", "") else: aligned_text_dict[key] = text diff --git a/src/test.py b/src/test.py index 981b967..0207686 100644 --- a/src/test.py +++ b/src/test.py @@ -31,56 +31,20 @@ import claude_funcs # final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') + input_dict = utils.read_input() -(filename, contract_text) = list(input_dict.items())[0] +for i in range(len(input_dict.items())): + print(i) + if i not in [10, 36, 44]: + (filename, contract_text) = list(input_dict.items())[i] + print(f"Testing preprocessing for {filename}") + contract_text = preprocess.clean_newlines(contract_text) + contract_text = preprocess.clean_billed_charges(contract_text) + text_dict = preprocess.split_text(contract_text) + text_dict = table_funcs.align_and_format_tables(text_dict) + text_dict = preprocess.highlight_rates(text_dict) -def clean_billed_charges(contract_text): - substrings = [ - "Physician's Billed Charges", - "Provider's Billed Charges", - "Allowable Billed Charges", - "Hospital's Billed Charges", - "Physician's Charges", - "Provider's Charges", - "Allowable Charges", - "Hospital's Charges", - "Billed Charges" - ] - max_substring_length = np.max([len(s) for s in substrings]) - def find_substring_indices(contract_text, s): - indices = [] - lower_contract_text = contract_text.lower() - lower_s = s.lower() - index = lower_contract_text.find(lower_s) - while index != -1: - indices.append(index) - index = lower_contract_text.find(lower_s, index + 1) - return indices - - for s in substrings: - indices = find_substring_indices(contract_text, s) - index_adder = 0 - if indices: - for i in indices: - index = i+index_adder - end_index = index + max_substring_length - match_part = contract_text[index:end_index] - previous = contract_text[max(0, index-30):index] - if '%' not in previous: - contract_text = contract_text[0:index] + f" 100% of {match_part}" + contract_text[end_index:] - index_adder += 9 - return contract_text.replace(' ', ' ') - -contract_text = preprocess.clean_newlines(contract_text) -contract_text = clean_billed_charges(contract_text) -contract_text = preprocess.highlight_rates(contract_text) -text_dict = preprocess.split_text(contract_text) -# text_dict = table_funcs.align_and_format_tables(text_dict) - - -print(text_dict['23']) - -bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries -print(bu_results) +# bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries +# print(bu_results) From a7a43e21e8f23da3e1a05d96948b4ad6a06f6ce0 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 20 Jun 2024 09:08:17 -0700 Subject: [PATCH 61/78] Updated consolidated output --- src/config.py | 2 +- src/consolidate_output.py | 18 +++++++++++++++++ src/test.py | 42 +++++++++++++++++++-------------------- 3 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 src/consolidate_output.py diff --git a/src/config.py b/src/config.py index f244c2d..bf67bb8 100644 --- a/src/config.py +++ b/src/config.py @@ -30,7 +30,7 @@ UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings -MAX_WORKERS = 5 +MAX_WORKERS = 3 # Prompt Debugging RUN_PRIMARY = True # Always True diff --git a/src/consolidate_output.py b/src/consolidate_output.py new file mode 100644 index 0000000..3a7b3bf --- /dev/null +++ b/src/consolidate_output.py @@ -0,0 +1,18 @@ + +import os +import pandas as pd + +import config + +all_dfs = [] +for file in os.listdir(config.OUTPUT_DIRECTORY): + if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, file)): + try: + df = pd.read_csv(f'{config.OUTPUT_DIRECTORY}/{file}/{config.PROCESSED_RESULTS_NAME}') + all_dfs.append(df) + except: + continue + +final_df = pd.concat(all_dfs, ignore_index=True) + +final_df.to_excel(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) \ No newline at end of file diff --git a/src/test.py b/src/test.py index 0207686..75867ef 100644 --- a/src/test.py +++ b/src/test.py @@ -18,31 +18,31 @@ import prompt_funcs import claude_funcs -# all_dfs = [] -# for file in os.listdir('output'): -# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): -# try: -# df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') -# all_dfs.append(df) -# except: -# continue +all_dfs = [] +for file in os.listdir('output'): + if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): + try: + df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') + all_dfs.append(df) + except: + continue -# final_df = pd.concat(all_dfs, ignore_index=True) -# final_df.to_excel('output_consolidated/test_20240610_batch1.xlsx') +final_df = pd.concat(all_dfs, ignore_index=True) +final_df.to_excel('output_consolidated/test_20240620_batch1.xlsx') -input_dict = utils.read_input() -for i in range(len(input_dict.items())): - print(i) - if i not in [10, 36, 44]: - (filename, contract_text) = list(input_dict.items())[i] - print(f"Testing preprocessing for {filename}") - contract_text = preprocess.clean_newlines(contract_text) - contract_text = preprocess.clean_billed_charges(contract_text) - text_dict = preprocess.split_text(contract_text) - text_dict = table_funcs.align_and_format_tables(text_dict) - text_dict = preprocess.highlight_rates(text_dict) +# input_dict = utils.read_input() +# for i in range(len(input_dict.items())): +# print(i) +# if i not in [10, 36, 44]: +# (filename, contract_text) = list(input_dict.items())[i] +# print(f"Testing preprocessing for {filename}") +# contract_text = preprocess.clean_newlines(contract_text) +# contract_text = preprocess.clean_billed_charges(contract_text) +# text_dict = preprocess.split_text(contract_text) +# text_dict = table_funcs.align_and_format_tables(text_dict) +# text_dict = preprocess.highlight_rates(text_dict) # bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries From 5d217df36a7026424e53756379cbbe86f2e583c0 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 21 Jun 2024 08:49:07 -0700 Subject: [PATCH 62/78] Testing new TD BU merge --- src/config.py | 10 +-- src/file_processing.py | 4 +- src/merge_funcs.py | 133 +++++++++--------------------- src/postprocess.py | 6 -- src/postprocessingfuncs.py | 37 +++++---- src/prompt_funcs.py | 4 +- src/prompts.py | 19 ++++- src/test.py | 164 +++++++++++++++++++++++++++++++------ 8 files changed, 223 insertions(+), 154 deletions(-) diff --git a/src/config.py b/src/config.py index bf67bb8..faff867 100644 --- a/src/config.py +++ b/src/config.py @@ -23,7 +23,7 @@ READ_MODE = '_LOCAL_' # OR '_S3_' WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written OUTPUT_DIRECTORY = 'output' CONSOLIDATED_OUTPUT_DIRECTORY = 'output_consolidated' -OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.csv' +OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.xlsx' TD_RESULTS_NAME = 'td_results.csv' BU_RESULTS_NAME = 'bu_results.csv' UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' @@ -54,12 +54,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXKP6R66NS" -AWS_SECRET_ACCESS_KEY="O4McuRggESDNl7omTvxVSiRjwECumUk8NhZpVFL3" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEI///////////wEaCXVzLWVhc3QtMiJIMEYCIQCMXW8IJ73YApb3JRLD09atRBInARf7L2bh5AU5HzD/GQIhALOzv8YiTkGXStNjWJ6DGnosjCMa2sRKFSs7oN3Gfp8eKoMDCDgQABoMNjYwMTMxMDY4NzgyIgxCEZzpAN+sOqLAj5Iq4AIrLejZWlEcl0D/35ryiNBifF4PgYNXPogUsgk7fjYNDnrG2xzVVdjALoOo+OZm3vLF+7X5HB5Foc3DTfuu8dTCGnGO2p4TPRXJ2Ftsh9ichNELsLVTgY39tBOOf7YMpQPaaNWHgJRZlAJ7g5+Xy8FzOL8VuiikxSI4UMKYbUMml4ZdQMNXsjqF+RbBDdWzodRnmirdhXhVBBUhpRIkRIioYizWjS+A51Y9Irmq4SnevG6/yLmuk2BV5Bx9uFeuvkOpUM/AJ8VUOUEff7cIKPhO6YG0sNFsJG4V00JVE9ORA8RqL5O46rvDKX8Ce3iuXAOr8TrQEjJ1FT8epzoy2EH0OhxTfpFTgmlo+X929sl3BeA6ZsHye8T8s42Y76MVQvAE5yEg6TtmLzfsDfecUe2z0SHilcWof9o9tswlWOUCIPXMbmERK7yFTTPax3JvyFA3lWRXCPlD/XpZgt26jdjjMJm6zbMGOqUBc+DbSEICCsra29nlXSHtA0YqQNwJ/bxTpBnAYJBb/1/0su2FVbIBYNfAcDty4ox4wB/HkUhpv2AEGGQIko78Y70ws6C4qD0zum74EPHG5VB1K2SU5npO5J8a6p5Jckj7i3n8Gb8nnB0UBvMyKUF4RkSbQJVB7ctgpoROW5IDu+7S7r/edt0yKZ8hpu2ixwKZvg7J6KYegCsbPT6xDOwb1+bAqPPl" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFKDEEC5H" +AWS_SECRET_ACCESS_KEY="T1pKiCx9I4sN2X31l9MY14JX7PN9o2nujjquhqER" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELf//////////wEaCXVzLWVhc3QtMiJIMEYCIQDJEC9kUJpl9GatZyAGVs3gPioSsxInFuplep+B293IcQIhAIFh2IbYCFGL0JIFjSnJt3lRGGhci7kNZwz7qORA1iRCKoMDCGAQABoMNjYwMTMxMDY4NzgyIgzN96Wl+CMwk0MLRtsq4ALq4KyN8E6jUsJ5QR6THkpAR65ZN85WXZlWncCZlMatC8vquuHpQIuHgCHIuIM4W2KWmH12FlXepLcwcg0m2ZXm6bi74HXqk7RAK3afKlZRGjdIv+BUuu4Xm7kq1X9Eq8/5u8G80AZFdK33TzUBXoDu6UlawckSfB5zwy+hFKDwtRWGWgL/xr/bEUXoDcrx5XcklLAlxoyVNEwOQ/ZQEV6mOyGeB/VipZ/KlQc+sFXumtkPCydga/CtVs1/vcFzCGf5XOAoFfaSUrganjSSpUQutGkH0BlPLxmUyagNlbbEcBM410AviA4dGHujNnBKtYmNkrtj2xwMcx73PrKwxg53DyzIYCPB7jTLnkmea2WfozXpzcEP2D0k8JDlmxJNPlHCOwUPoqUppPQXQZviR34x/UpXtCbP65zGOd+za1gOcIAsQEwer9urYF8GiFFYWpQAT/Wed1Zv4QoY7rqDTL6PMN+n1rMGOqUBNlpi7WilJtJvxsuPrKz8gKwZEDIExcAcxWthTFRlsbavMTsxP8kInjcnBg2/r7ot0F6ddIp9iJXGDS9dROpopvSmBPnc2UnYbgpIqoebs+viawLUyXb/+b6KDmHhMQbCpeu4rczqMbBoUjG0plteugGEtYo5rNZHAvnJONBWOUxAxcAo1X88DyAWLpaHg44mnB2lSezviTnaOJvEBaqyrjF+Y9/J" # File Paths -LOCAL_PATH = 'data/all_txt_updated_batch_1_numbered/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index 73e0cbb..b48ef8e 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -54,14 +54,14 @@ def process_file(file_object): td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page pd.DataFrame(td_results).to_csv(os.path.join(output_dir, config.TD_RESULTS_NAME), index=False) print(f"Top Down Complete - {filename}") - + ################## RUN BOTTOM UP ################## bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, config.BU_RESULTS_NAME), index=False) print(f"Bottom Up Complete - {filename}") ################## COMBINE TOP DOWN AND BOTTOM UP ################## - combined_results = merge_funcs.merge_results(postprocessingfuncs.clean_td(td_results), bu_results, text_dict) + combined_results = merge_funcs.merge_results(td_results, bu_results, text_dict) print(f"TD/BU Merge Complete - {filename} ") ################## WRITE UNPROCESSED OUTPUT ################## diff --git a/src/merge_funcs.py b/src/merge_funcs.py index a3307f5..0b5bd75 100644 --- a/src/merge_funcs.py +++ b/src/merge_funcs.py @@ -2,106 +2,51 @@ import prompts import claude_funcs -def get_unique_keys(dicts): - """ - Collects and returns a set of unique keys from a list of dictionaries. - This function aggregates all the keys from each dictionary in the provided list, - ensuring that each key is only listed once, regardless of how often it appears across - the dictionaries. +def prompt_select_multiple(key, value, bu_dict, page): + prompt = prompts.MERGE_SELECT_PROMPT(key, value, bu_dict, page) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + return answer - Parameters: - dicts (list of dict): A list of dictionaries from which to extract unique keys. - - Returns: - set: A set containing all unique keys found in the input list of dictionaries. - """ - keys = set() - for d in dicts: - keys.update(d.keys()) - return keys - - -def get_unique_date_fields(td_results): - """ - Extracts and aggregates unique date values from a list of dictionaries, specifically focusing on keys that include 'DATE'. - - This function iterates through each dictionary in the provided list, identifies keys that contain the substring 'DATE', - and collects unique values associated with these date keys. The collected values are stored in a set to avoid duplicates, - and then converted to a list for each date field before returning. - - Parameters: - td_results (list of dict): A list of dictionaries, each potentially containing multiple date-related fields. - - Returns: - dict: A dictionary where each key is a date field and the value is a list of unique dates found in that field. - """ - unique_date_fields = {} - for td in td_results: - for key, value in td.items(): - if 'DATE' in key: - if key not in unique_date_fields: - unique_date_fields[key] = set() - unique_date_fields[key].update(value if isinstance(value, list) else [value]) - for key in unique_date_fields: - unique_date_fields[key] = list(unique_date_fields[key]) - return unique_date_fields def merge_results(td_results, bu_results, text_dict): - """ - Merges results from Top Down (TD) and Bottom Up (BU) processing streams based on page numbers and keys. - - This function combines results from two different analyses by aligning them using their page numbers and - merges values under the same keys, giving precedence to non-empty and list values. It also resolves conflicts - and fills missing values for date fields using a unique set of date values extracted from TD results. - - Parameters: - td_results (list of dict): Results from the Top Down processing, containing page numbers and various attributes. - bu_results (list of dict): Results from the Bottom Up processing, similarly structured as TD results. - text_dict (dict): Dictionary of page texts keyed by page numbers used for generating prompts if necessary. - - Returns: - list of dict: A list of dictionaries, each representing merged results from both TD and BU for a page. - """ - - all_keys = get_unique_keys(td_results) | get_unique_keys(bu_results) + def find_td_dict(page_num): + """ Helper function to find dictionary in td_results with matching page_num """ + for td in td_results: + if td['page_num'] == page_num: + return td + return None - date_fields = get_unique_date_fields(td_results) + def get_value(td_dict, key, page_num, text_dict): + """ Helper function to handle value selection based on the rules specified. """ + if td_dict is None: + # If there's no matching page, use recursion to look one page earlier + if int(page_num) > 1: + return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict) + else: + return "N/A" # Base case if no previous pages exist + else: + value = td_dict.get(key, "N/A") + if value == "N/A" or value == "": + # If value is 'N/A' or empty, use recursion to look one page earlier + return get_value(find_td_dict(int(page_num) - 1), key, int(page_num) - 1, text_dict) + elif ',' in value and 'DATE' not in key: + # Randomly select one of the comma-separated values + return prompt_select_multiple(key, value, bu_dict, text_dict[page_num]) + else: + return value merged_results = [] - for bu in bu_results: - merged = {key: bu.get(key, "") for key in all_keys} - td_on_page = [td for td in td_results if td['page_num'] == bu['page_num']] + for bu_dict in bu_results: + page_num = bu_dict['page_num'] + td_dict = find_td_dict(page_num) - for td in td_on_page: - for key, value in td.items(): - if not merged[key]: - merged[key] = value - elif isinstance(value, list) and value and not isinstance(merged[key], list): - merged[key] = value - elif isinstance(value, list) and value: - merged[key].extend(value) - - for date_key, date_values in date_fields.items(): - if date_key not in merged or not merged[date_key]: - merged[date_key] = date_values - else: - merged[date_key].extend([val for val in date_values if val not in merged[date_key]]) - - for key in all_keys: - if isinstance(merged[key], list) and len(merged[key]) > 1: - page_num = bu['page_num'] - page_text = text_dict.get(page_num, "") - prompt = prompts.GENERATE_PROMPT(bu, td_on_page, page_text, field_name=key, values=merged[key]) - response = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - try: - selected_value = response.strip() - merged[key] = selected_value - except Exception as e: - print(f"Error processing LLM response for key {key}: {e}") - merged[key] = ', '.join(merged[key]) + new_dict = bu_dict.copy() # Start with the bu_dict's data + # Add or overwrite keys from td_dict + if td_dict: + for key, value in td_dict.items(): + if key not in ['Filename', 'page_num']: + new_dict[key] = get_value(td_dict, key, page_num, text_dict) - merged_results.append(merged) - - return merged_results - + merged_results.append(new_dict) + return merged_results \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index 927b6e4..1367901 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -189,12 +189,6 @@ def post_process_combined(df): # except Exception as e: # print(f"Error: {e}") - # Filter out specific SERVICE rows - try: - df = postprocessingfuncs.filter_service_column(df) - except Exception as e: - print(f"Postprocessing Error - filter_service_column : {e}") - # Move percentages and large numbers to correct columns try: df = postprocessingfuncs.move_percentage_to_rate(df) diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index eae72e7..3a0fa8c 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -20,14 +20,16 @@ def sanitize_value(value): Returns: varied types: The sanitized value, which can be a string or the original type if no changes were needed. """ - if pd.isna(value): + try: + if isinstance(value, list): + return ', '.join(str(v) for v in value) + elif isinstance(value, str): + value = value.strip('[]') + return ', '.join([item.strip(" '") for item in value.split(',')]) + elif pd.isna(value): + return "N/A" + except: return value - if isinstance(value, list): - value = ', '.join(str(v) for v in value) - if isinstance(value, str): - value = value.strip('[]') - value = ', '.join([item.strip(" '") for item in value.split(',')]) - return value def exact_match(val, valid_values): """ @@ -209,30 +211,31 @@ def correct_misplaced_values(df, columns, valid_values_dict): return df -def filter_service_column(df): +def filter_service_column(d): """ - Filters out rows in a DataFrame based on specified keywords within the 'SERVICE' column. + Filters out items in a list of dictionaries based on specified keywords within the 'SERVICE' key. This function uses a list of predefined keywords associated with non-service related terms like 'LIABILITY' or 'RISK'. - It constructs a regular expression pattern from these keywords and removes any rows where the 'SERVICE' column contains - any of these keywords, effectively filtering the DataFrame to exclude rows that are likely to be irrelevant or misclassified + It constructs a regular expression pattern from these keywords and removes any dictionary items where the 'SERVICE' key contains + any of these keywords, effectively filtering the list to exclude items that are likely to be irrelevant or misclassified as services when they pertain to other contractual obligations or conditions. Parameters: - df (pandas.DataFrame): The DataFrame from which rows will be filtered. + d (list): A list of dictionaries, each containing a 'SERVICE' key among others. Returns: - pandas.DataFrame: A new DataFrame with rows containing specified keywords in the 'SERVICE' column removed. + list: A new list of dictionaries with items containing specified keywords in the 'SERVICE' key removed. """ - df = df.dropna(subset=['SERVICE']) - keywords = [ 'LIABILITY', 'RISK', 'LOBBYING', 'DAMAGES', 'CONFIDENTIALITY', 'ARBITRATION', 'FALSE CLAIMS ACT', 'UNSPECIFIED', 'AUDIT', 'INTEREST', 'N/A', 'BUSINESS', 'COMPLIANCE', 'Medical Assistance Program', 'MATERIAL SUBCONTRACT', 'GIFTS', 'GRATUITIES', ] pattern = '|'.join(keywords) - df = df[~df['SERVICE'].str.contains(pattern, case=False, na=False)] - return df + regex = re.compile(pattern, re.IGNORECASE) + + filtered_list = [item for item in d if not regex.search(item.get('SERVICE', ''))] + + return filtered_list def move_percentage_to_rate(df): diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 6831c42..db21445 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -6,6 +6,7 @@ import prompts import claude_funcs import dict_operations import utils +import postprocessingfuncs def run_bottom_up_lesser(d, text_dict, tokens=8000): @@ -155,9 +156,10 @@ def run_bottom_up(filename, text_dict): # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY answer_strings = run_bottom_up_primary(text_dict, 8000) answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) + answer_dicts_filtered = postprocessingfuncs.filter_service_column(answer_dicts) # Bottom Up Secondary - results_dicts = run_bottom_up_secondary(answer_dicts, text_dict, 8000) + results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000) for d in results_dicts: d['Filename'] = filename diff --git a/src/prompts.py b/src/prompts.py index ae8698d..bbd5729 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -21,8 +21,6 @@ Methodology: {bu_dict['FULL_METHODOLOGY']} Listed above is a page of a contract, as well as a Service and Methodology found on this page. Your job is to identify which of the following entities this specific Service is associated with. -----Change Below---- - Here are the entities, represented as dictionary objects: {td_dicts} @@ -31,6 +29,23 @@ Write ONLY the number of the dictionary that is most associated with the Service """ +def MERGE_SELECT_PROMPT(key, value, bu_dict, page): + return f"""### PAGE START ### {page} ### PAGE END + +Service: {bu_dict['SERVICE']} +Methodology: {bu_dict['FULL_METHODOLOGY']} + +Listed above is a page of a contract, as well as a specific Service and Methodology that may be found on this page. +Your job is to identify which of the following {key}s this specific Service is associated with. + +Here is a comma-separated list of options: + +{value} + +Write ONLY the value or values in the list that DIRECTLY applies to the Service and Methodology listed above. If none of the values apply, write 'N/A'. +If the Service and Methodology are not found on the page, write the LAST item in the list. Do not write any other commentary or explanation. +""" + def BOTTOM_UP_PRIMARY(page, payer): return f""" diff --git a/src/test.py b/src/test.py index 75867ef..83ec7a7 100644 --- a/src/test.py +++ b/src/test.py @@ -16,35 +16,145 @@ import table_funcs import prompts import prompt_funcs import claude_funcs +import merge_funcs -all_dfs = [] -for file in os.listdir('output'): - if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', file)): - try: - df = pd.read_csv(f'output/{file}/combined_results_post_processed.csv') - all_dfs.append(df) - except: - continue - -final_df = pd.concat(all_dfs, ignore_index=True) -final_df.to_excel('output_consolidated/test_20240620_batch1.xlsx') +input_dict = utils.read_input() + +(filename, contract_text) = list(input_dict.items())[0] + +contract_text = preprocess.clean_newlines(contract_text) +contract_text = preprocess.clean_billed_charges(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) + +# for page_num, page_text in text_dict.items(): +# if page_num == '31': +# prompt = prompts.TOP_DOWN_PRIMARY(page_text) +# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) +# answer_dict = json.loads(answer) + +# print(answer_dict) - -# input_dict = utils.read_input() -# for i in range(len(input_dict.items())): -# print(i) -# if i not in [10, 36, 44]: -# (filename, contract_text) = list(input_dict.items())[i] -# print(f"Testing preprocessing for {filename}") -# contract_text = preprocess.clean_newlines(contract_text) -# contract_text = preprocess.clean_billed_charges(contract_text) -# text_dict = preprocess.split_text(contract_text) -# text_dict = table_funcs.align_and_format_tables(text_dict) -# text_dict = preprocess.highlight_rates(text_dict) - - -# bu_results = prompt_funcs.run_bottom_up_primary({'23' : text_dict['23']}, 4000) # Returns list of dictionaries +# bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries # print(bu_results) - + + +bu_results = [ +{'SERVICE': 'Ancillary Provider clean claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '1.5%', +'FULL_METHODOLOGY': 'PCHP must pay Ancillary Provider interest on all claims that are not paid within 45 days at a rate of 1.5% per month (18% annual) for each month the claim remains unpaid.', +'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', +'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Monthly', +'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '18', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', +'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, +{'SERVICE': 'Ancillary Provider clean claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '1.5%', +'FULL_METHODOLOGY': 'PCHP must pay Physician interest on all claims that are not paid within 45 days at a rate of 1.5% per month (18% annual) for each month the claim remains unpaid.', +'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', +'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Annual', +'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '18', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', +'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, +{'SERVICE': 'claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '85%', +'FULL_METHODOLOGY': 'Notwithstanding the intent to audit the claim, PCHP must pay eighty-five percent (85%) of the claim at the rates set forth in this Agreement within such thirty (30) day period.', +'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Contracted Rate', +'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'Y', +'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'Notwithstanding the intent to audit the claim, PCHP must pay eighty-five percent (85%) of the claim at the rates set forth in this Agreement within such thirty (30) day period.', +'RATE_ESCALATOR_IND': 'N', 'RATE_ESCALATOR_BASIS': 'N/A', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', +'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, +{'SERVICE': 'PCHP MEDICAID STAR HMO SERVICES', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '100%', +'FULL_METHODOLOGY': "One hundred percent (100%) of the prevailing yearly and current Medicaid fee schedule for the State of Texas or the Participating Ancillary Services Provider's usual and customary charge, whichever is less.", +'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicaid Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicaid', +'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'prevailing yearly and current', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', +'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', +'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, +{'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': "the Participating Ancillary Services Provider's usual and customary charge", 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_FLAT_FEE': 'N/A', +'page_num': '31', 'SERVICE': 'PCHP MEDICAID STAR HMO SERVICES', 'REIMBURSEMENT_METHODOLOGY': "Provider's Charges", 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', +'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', +'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '100', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', +'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt'}, +{'SERVICE': 'PCHP CHIP HMO SERVICES', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '100%', +'FULL_METHODOLOGY': "One hundred percent (100%) of the prevailing yearly and current Medicaid fee schedule for the State of Texas or the Participating Ancillary Services Provider's usual and customary charge, whichever is less.", +'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicaid Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicaid', +'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'prevailing yearly and current', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', +'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', +'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, +{'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': "the Participating Ancillary Services Provider's usual and customary charge", 'REIMBURSEMENT_RATE': 'N/A', +'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '31', 'SERVICE': 'PCHP CHIP HMO SERVICES', 'REIMBURSEMENT_METHODOLOGY': "Provider's Charges", 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', +'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', +'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '100', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', +'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', +'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt'}] + + +td_results = [{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan Inc., OPTionah Care Care Options forkids Services', 'page_num': '1', +'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '2', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', +'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP PLAN', 'page_num': '3', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '4', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP PLAN', 'page_num': '5', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '6', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '7', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'N/A', 'page_num': '8', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '9', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP Plan, PCHP', 'page_num': '10', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '11', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan', 'page_num': '12', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '13', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'PCHP', 'page_num': '14', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '15', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '16', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '17', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP Plan', 'page_num': '18', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '19', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '20', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid,CHIP', 'CONTRACT_PROGRAM': 'STAR,CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '21', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '22', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid, Medicare', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '24', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '25', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '26', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '27', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'CHIP', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan, Inc.', 'page_num': '28', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan, Care opTions for kids', 'page_num': '29', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'MAY 01 2000', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'MAY 01 2000'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '30', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'HEALTHfirst, KIDSfirst', 'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, +{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '32', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', +'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}] + + + + + + + From 484c9433864659c05817521d34b360e728ecdc8d Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 21 Jun 2024 11:57:49 -0700 Subject: [PATCH 63/78] Updated Date postprocessing --- src/config.py | 6 +- src/postprocess.py | 210 +++++++------------------------------ src/postprocessingfuncs.py | 174 +++++++++++++++++++++--------- src/test.py | 132 ++--------------------- 4 files changed, 177 insertions(+), 345 deletions(-) diff --git a/src/config.py b/src/config.py index faff867..5ab60b8 100644 --- a/src/config.py +++ b/src/config.py @@ -12,10 +12,10 @@ CLIENT_NAME = '' TODAY = datetime.now().strftime("%Y%m%d") # Valid values +VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] +VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] -VALID_PROGRAMS = [] -VALID_LOB = [] - + # Input Settings READ_MODE = '_LOCAL_' # OR '_S3_' diff --git a/src/postprocess.py b/src/postprocess.py index 1367901..3d27cd2 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -4,160 +4,23 @@ import re import postprocessingfuncs import config -def sanitize_combined(df): - """ - Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value. - - This function iterates over each column in the DataFrame, applying a sanitization function to each element. - The sanitization function is designed to clean or adjust the data in a way that makes it more consistent or appropriate - for further processing or analysis. This could include operations like trimming whitespace, correcting formats, - or removing unwanted characters. - - Parameters: - df (pandas.DataFrame): The DataFrame to be sanitized. - - Returns: - pandas.DataFrame: The sanitized DataFrame, with the same structure but cleaned data. - """ - for column in df.columns: - df[column] = df[column].apply(postprocessingfuncs.sanitize_value) - return df - -def extract_codes_CPT(value): - """ - Extracts and formats CPT codes from a given input value. - - This function searches for patterns that match CPT codes within the provided input. CPT codes typically consist - of a letter followed by four digits, or just five digits, and may include a two-letter state code as a suffix. - The function handles cases where the range of codes is specified using 'through' by converting it to a hyphen. - If any codes are found, they are returned as a comma-separated string. If no codes are found or the input is null, - an appropriate value (empty string or null) is returned. - - Parameters: - value (str or NaN): The value from which to extract CPT codes, which could potentially be a string or NaN. - - Returns: - str: A comma-separated string of the extracted CPT codes, or an empty string if none are found. - If the input is NaN, the function returns the input as is. - """ - if pd.isna(value): - return value - value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE) - - pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b' - matches = re.findall(pattern, value) - if matches: - return ', '.join([''.join(match) for match in matches]) - else: - return "" -def extract_codes_Diagnosis(value): - """ - Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats. - This function identifies patterns consistent with ICD diagnosis codes, which are generally a letter followed by numbers - and may include additional alphanumeric characters and a period for further specification. The function handles null inputs - gracefully, returning the input itself if null. If valid diagnosis codes are found, they are returned as a comma-separated - string. If no codes are found, the function returns an empty string. +def postprocess_results(combined_df): - Parameters: - value (str or NaN): The input value from which to extract diagnosis codes, which could be a string or NaN. - - Returns: - str: A comma-separated string of the extracted diagnosis codes, or an empty string if none are found. - If the input is NaN, the function returns the input as is. - """ - if pd.isna(value): - return value - - pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b' - matches = re.findall(pattern, value) - if matches: - return ', '.join(matches) - else: - return "" - -def extract_codes_Revenue(value): - """ - Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits. - - This function searches for patterns that match revenue codes in the input provided. It handles cases where the input might be null (NaN), - returning the input itself in such cases. If any valid revenue codes are found, they are formatted into a comma-separated string. - If no codes are found, the function returns an empty string, indicating the absence of recognizable codes in the input. - - Parameters: - value (str or NaN): The input value from which to extract revenue codes, which could be a string or NaN. - - Returns: - str: A comma-separated string of the extracted revenue codes, or an empty string if none are found. - If the input is NaN, the function returns the input as is. - """ - if pd.isna(value): - return value - - pattern = r'\b\d{3,4}\b' - matches = re.findall(pattern, value) - if matches: - return ', '.join(matches) - else: - return "" - -def clean_code_column(df, column_name, extract_function): - """ - Applies a specified function to clean and extract codes from a specific column in a DataFrame. - - This function takes a DataFrame and applies a given extraction function to each element of a specified column. - The extraction function is expected to format or extract codes (e.g., CPT, ICD, Revenue codes) from the column's values. - This cleaned and extracted data replaces the original values in the column. The function is flexible and can be used - with any function designed to process and return cleaned values from a single input. - - Parameters: - df (pandas.DataFrame): The DataFrame containing the data to be cleaned. - column_name (str): The name of the column to be cleaned. - extract_function (function): The function to apply to each value in the specified column, which processes and returns the cleaned value. - - Returns: - pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes. - """ - df[column_name] = df[column_name].apply(extract_function) - return df - -def post_process_combined(df): - """ - Performs comprehensive post-processing on a DataFrame to sanitize, correct, and clean data fields related to healthcare contracts. - - This function includes steps to: - - Sanitize the entire DataFrame. - - Ensure that LOB, Program, and Network fields contain valid values. - - Use fuzzy matching to correct near-miss entries in specified fields. - - Correct misplaced values across similar columns. - - Clean code columns for CPT, Diagnosis, and Revenue codes. - - Filter and adjust specific data columns to align with expected formats and values. - - Parameters: - df (pandas.DataFrame): The DataFrame containing combined data that needs post-processing. - - Returns: - pandas.DataFrame: The DataFrame after applying all specified cleaning, correction, and filtering processes. - """ - - VALID_LOBS = ['MEDICARE', 'MEDICARE ADVANTAGE', 'MEDICAID', 'MARKETPLACE', 'COMMERCIAL', 'GROUP', 'MEDICARE-MEDICAID'] - VALID_PROGRAMS = ['CHIP', 'CHIP-P', 'CHIP-PERINATE', 'STAR', 'STAR+PLUS', 'MA', 'DUAL SPECIAL NEEDS PLAN', 'DSNP', 'DUAL'] - VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] - # Define valid values dictionary valid_values_dict = { - 'CONTRACT_LOB': VALID_LOBS, - 'CONTRACT_PROGRAM': VALID_PROGRAMS, - 'CONTRACT_NETWORK': VALID_NETWORKS + 'CONTRACT_LOB': config.VALID_LOBS, + 'CONTRACT_PROGRAM': config.VALID_PROGRAMS, + 'CONTRACT_NETWORK': config.VALID_NETWORKS } - + # Sanitize the combined data - df = sanitize_combined(df) - + df = postprocessingfuncs.sanitize_combined(df) + # Clean specific columns for exact matches - for field_list in [['CONTRACT_LOB', 'Corrected_LOB', VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', VALID_NETWORKS]]: + for field_list in [['CONTRACT_LOB', 'Corrected_LOB', config.VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', config.VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', config.VALID_NETWORKS]]: try: postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) except Exception as e: @@ -173,21 +36,6 @@ def post_process_combined(df): df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) except Exception as e: print(f"Postprocessing Error - correct_misplaced_values : {e}") - - - # Clean the specified code columns - # try: - # df = clean_code_column(df, 'REIMBURSEMENT_PROC_CODES', extract_codes_CPT) - # except Exception as e: - # print(f"Error: {e}") - # try: - # df = clean_code_column(df, 'REIMBURSEMENT_DIAG_CODES', extract_codes_Diagnosis) - # except Exception as e: - # print(f"Error: {e}") - # try: - # df = clean_code_column(df, 'REIMBURSEMENT_REVENUE_CODES', extract_codes_Revenue) - # except Exception as e: - # print(f"Error: {e}") # Move percentages and large numbers to correct columns try: @@ -195,31 +43,47 @@ def post_process_combined(df): except Exception as e: print(f"Postprocessing Error - move_percentage_to_rate : {e}") - try: - df = postprocessingfuncs.move_large_numbers_to_flat_fee(df) - except Exception as e: - print(f"Postprocessing Error - move_large_numbers_to_flat_fee : {e}") - + # Replace N/As in "Not Covered" with 0% try: df = postprocessingfuncs.set_rate_to_zero_if_not_covered(df) except Exception as e: print(f"Postprocessing Error - set_rate_to_zero_if_not_covered : {e}") + # Adjust for adding or subtracting try: df = postprocessingfuncs.adjust_reimbursement_rate(df) except Exception as e: print(f"Postprocessing Error - adjust_reimbursement_rate : {e}") - try: - clean_df = postprocessingfuncs.clean_pagenumbers(df) + # Clean dates so that Termination and Effective aren't the same + try: + df = postprocessingfuncs.clean_dates(df) except Exception as e: - print(f"Postprocessing Error - clean_pagenumbers : {e}") - clean_df = df - return clean_df + print(f"Postprocessing Error - clean_dates : {e}") - -def postprocess_results(combined_df): - combined_df = post_process_combined(combined_df) + return df + + + + + + + + + + + + + + + + + + + + + + # Order columns cols = [col for col in config.VALID_COLUMNS if col in combined_df.columns] + [col for col in combined_df.columns if col not in config.VALID_COLUMNS] #combined_df = combined_df.loc[:, cols] diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 3a0fa8c..344f4da 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -149,29 +149,6 @@ def clean_columns_combined_fuzzy(df, column_name, valid_values, threshold): return original_values, cleaned_values, changes -def extract_page_number(page_text): - """ - Extracts the page number from a given text string that includes a range of pages formatted as "Pages X-Y". - - This function uses a regular expression to find and extract the first page number from a text string that includes a page range. - If the specific pattern "Pages X-Y" is found in the text, it captures and returns the number corresponding to 'X'. If the pattern - is not found, the entire input text is returned, indicating that no page number could be extracted based on the expected format. - - Parameters: - page_text (str): The text containing the page number information, typically including a page range. - - Returns: - str: The extracted first page number if the pattern is found; otherwise, the original text. - """ - match = re.search(r'Pages\s+(\d+)-\d+', page_text) - if match: - return match.group(1) - else: - return page_text - -def clean_pagenumbers(df): - df['page_num'] = df['page_num'].apply(extract_page_number) - return df def correct_misplaced_values(df, columns, valid_values_dict): """ @@ -262,31 +239,6 @@ def move_percentage_to_rate(df): return df -def move_large_numbers_to_flat_fee(df): - """ - Transfers large numerical values from the 'REIMBURSEMENT_RATE' column to the 'REIMBURSEMENT_FLAT_FEE' column in a DataFrame. - - This function examines entries in the 'REIMBURSEMENT_RATE' column to identify numeric values exceeding 1000, - which are more indicative of flat fees rather than rates. It moves these values to the 'REIMBURSEMENT_FLAT_FEE' column. - The original 'REIMBURSEMENT_RATE' entries from which these numbers are moved are set to None, ensuring the data reflects - the correct financial parameters. - - Parameters: - df (pandas.DataFrame): The DataFrame containing the reimbursement data to be adjusted. - - Returns: - pandas.DataFrame: The DataFrame with large numbers moved from the 'REIMBURSEMENT_RATE' to the 'REIMBURSEMENT_FLAT_FEE' column. - """ - def move_large_number(value): - if isinstance(value, str) and value.isdigit() and int(value) > 1000: - return True - return False - - df['REIMBURSEMENT_FLAT_FEE'] = df.apply(lambda row: row['REIMBURSEMENT_RATE'] if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_FLAT_FEE'], axis=1) - df['REIMBURSEMENT_RATE'] = df.apply(lambda row: None if move_large_number(row['REIMBURSEMENT_RATE']) else row['REIMBURSEMENT_RATE'], axis=1) - return df - - def set_rate_to_zero_if_not_covered(df): """ Sets the 'REIMBURSEMENT_RATE' to zero for entries in the DataFrame where the service is explicitly not covered, @@ -374,3 +326,129 @@ def clean_td(td): +def sanitize_combined(df): + """ + Sanitizes all columns in a DataFrame by applying a predefined sanitization function to each value. + + This function iterates over each column in the DataFrame, applying a sanitization function to each element. + The sanitization function is designed to clean or adjust the data in a way that makes it more consistent or appropriate + for further processing or analysis. This could include operations like trimming whitespace, correcting formats, + or removing unwanted characters. + + Parameters: + df (pandas.DataFrame): The DataFrame to be sanitized. + + Returns: + pandas.DataFrame: The sanitized DataFrame, with the same structure but cleaned data. + """ + for column in df.columns: + df[column] = df[column].apply(sanitize_value) + return df + +def extract_codes_CPT(value): + """ + Extracts and formats CPT codes from a given input value. + + This function searches for patterns that match CPT codes within the provided input. CPT codes typically consist + of a letter followed by four digits, or just five digits, and may include a two-letter state code as a suffix. + The function handles cases where the range of codes is specified using 'through' by converting it to a hyphen. + If any codes are found, they are returned as a comma-separated string. If no codes are found or the input is null, + an appropriate value (empty string or null) is returned. + + Parameters: + value (str or NaN): The value from which to extract CPT codes, which could potentially be a string or NaN. + + Returns: + str: A comma-separated string of the extracted CPT codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ + if pd.isna(value): + return value + + value = re.sub(r'\bthrough\b', '-', str(value), flags=re.IGNORECASE) + + pattern = r'\b([A-Z]\d{4}|\d{5})(-[A-Z]{2})?\b' + matches = re.findall(pattern, value) + if matches: + return ', '.join([''.join(match) for match in matches]) + else: + return "" + +def extract_codes_Diagnosis(value): + """ + Extracts and formats diagnosis codes from a given input value, typically adhering to ICD (International Classification of Diseases) formats. + + This function identifies patterns consistent with ICD diagnosis codes, which are generally a letter followed by numbers + and may include additional alphanumeric characters and a period for further specification. The function handles null inputs + gracefully, returning the input itself if null. If valid diagnosis codes are found, they are returned as a comma-separated + string. If no codes are found, the function returns an empty string. + + Parameters: + value (str or NaN): The input value from which to extract diagnosis codes, which could be a string or NaN. + + Returns: + str: A comma-separated string of the extracted diagnosis codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ + if pd.isna(value): + return value + + pattern = r'\b[A-Z][0-9][A-Z0-9]{1,4}(\.[A-Z0-9]{1,4})?\b' + matches = re.findall(pattern, value) + if matches: + return ', '.join(matches) + else: + return "" + +def extract_codes_Revenue(value): + """ + Extracts revenue codes from a given input value. Revenue codes are typically numerical codes of three to four digits. + + This function searches for patterns that match revenue codes in the input provided. It handles cases where the input might be null (NaN), + returning the input itself in such cases. If any valid revenue codes are found, they are formatted into a comma-separated string. + If no codes are found, the function returns an empty string, indicating the absence of recognizable codes in the input. + + Parameters: + value (str or NaN): The input value from which to extract revenue codes, which could be a string or NaN. + + Returns: + str: A comma-separated string of the extracted revenue codes, or an empty string if none are found. + If the input is NaN, the function returns the input as is. + """ + if pd.isna(value): + return value + + pattern = r'\b\d{3,4}\b' + matches = re.findall(pattern, value) + if matches: + return ', '.join(matches) + else: + return "" + +def clean_code_column(df, column_name, extract_function): + """ + Applies a specified function to clean and extract codes from a specific column in a DataFrame. + + This function takes a DataFrame and applies a given extraction function to each element of a specified column. + The extraction function is expected to format or extract codes (e.g., CPT, ICD, Revenue codes) from the column's values. + This cleaned and extracted data replaces the original values in the column. The function is flexible and can be used + with any function designed to process and return cleaned values from a single input. + + Parameters: + df (pandas.DataFrame): The DataFrame containing the data to be cleaned. + column_name (str): The name of the column to be cleaned. + extract_function (function): The function to apply to each value in the specified column, which processes and returns the cleaned value. + + Returns: + pandas.DataFrame: The DataFrame with the specified column updated with cleaned and processed codes. + """ + df[column_name] = df[column_name].apply(extract_function) + return df + + +def clean_dates(df): + for idx, row in df.iterrows(): + if row['LOB_PRICING_TERMS_EFFECTIVE_DATE'] == row['LOB_PRICING_TERMS_TERMINATION_DATE']: + df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A' + + return df diff --git a/src/test.py b/src/test.py index 83ec7a7..994e0d1 100644 --- a/src/test.py +++ b/src/test.py @@ -19,15 +19,15 @@ import claude_funcs import merge_funcs -input_dict = utils.read_input() +# input_dict = utils.read_input() -(filename, contract_text) = list(input_dict.items())[0] +# (filename, contract_text) = list(input_dict.items())[0] -contract_text = preprocess.clean_newlines(contract_text) -contract_text = preprocess.clean_billed_charges(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) -text_dict = preprocess.highlight_rates(text_dict) +# contract_text = preprocess.clean_newlines(contract_text) +# contract_text = preprocess.clean_billed_charges(contract_text) +# text_dict = preprocess.split_text(contract_text) +# text_dict = table_funcs.align_and_format_tables(text_dict) +# text_dict = preprocess.highlight_rates(text_dict) # for page_num, page_text in text_dict.items(): # if page_num == '31': @@ -42,119 +42,9 @@ text_dict = preprocess.highlight_rates(text_dict) # print(bu_results) -bu_results = [ -{'SERVICE': 'Ancillary Provider clean claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '1.5%', -'FULL_METHODOLOGY': 'PCHP must pay Ancillary Provider interest on all claims that are not paid within 45 days at a rate of 1.5% per month (18% annual) for each month the claim remains unpaid.', -'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', -'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Monthly', -'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '18', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', -'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, -{'SERVICE': 'Ancillary Provider clean claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '1.5%', -'FULL_METHODOLOGY': 'PCHP must pay Physician interest on all claims that are not paid within 45 days at a rate of 1.5% per month (18% annual) for each month the claim remains unpaid.', -'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', -'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Annual', -'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '18', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', -'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, -{'SERVICE': 'claims', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '85%', -'FULL_METHODOLOGY': 'Notwithstanding the intent to audit the claim, PCHP must pay eighty-five percent (85%) of the claim at the rates set forth in this Agreement within such thirty (30) day period.', -'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'LESSER_OF_LANGUAGE_IND': 'N', 'GREATER_OF_LANGUAGE_IND': 'N', 'REIMBURSEMENT_METHODOLOGY': 'Contracted Rate', -'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'Y', -'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'Notwithstanding the intent to audit the claim, PCHP must pay eighty-five percent (85%) of the claim at the rates set forth in this Agreement within such thirty (30) day period.', -'RATE_ESCALATOR_IND': 'N', 'RATE_ESCALATOR_BASIS': 'N/A', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', -'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, -{'SERVICE': 'PCHP MEDICAID STAR HMO SERVICES', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '100%', -'FULL_METHODOLOGY': "One hundred percent (100%) of the prevailing yearly and current Medicaid fee schedule for the State of Texas or the Participating Ancillary Services Provider's usual and customary charge, whichever is less.", -'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicaid Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicaid', -'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'prevailing yearly and current', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', -'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', -'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, -{'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': "the Participating Ancillary Services Provider's usual and customary charge", 'REIMBURSEMENT_RATE': 'N/A', 'REIMBURSEMENT_FLAT_FEE': 'N/A', -'page_num': '31', 'SERVICE': 'PCHP MEDICAID STAR HMO SERVICES', 'REIMBURSEMENT_METHODOLOGY': "Provider's Charges", 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', -'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', 'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', -'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '100', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', -'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt'}, -{'SERVICE': 'PCHP CHIP HMO SERVICES', 'REIMBURSEMENT_FLAT_FEE': 'N/A', 'REIMBURSEMENT_RATE': '100%', -'FULL_METHODOLOGY': "One hundred percent (100%) of the prevailing yearly and current Medicaid fee schedule for the State of Texas or the Participating Ancillary Services Provider's usual and customary charge, whichever is less.", -'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'REIMBURSEMENT_METHODOLOGY': 'Medicaid Fee Schedule', 'REIMBURSEMENT_FEE_SCHEDULE': 'Medicaid', -'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'prevailing yearly and current', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', -'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': 'N/A', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', 'REIMBURSEMENT_DIAG_CODES': 'N/A', -'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', 'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A'}, -{'LESSER_OF_LANGUAGE_IND': 'Y', 'GREATER_OF_LANGUAGE_IND': 'N', 'FULL_METHODOLOGY': "the Participating Ancillary Services Provider's usual and customary charge", 'REIMBURSEMENT_RATE': 'N/A', -'REIMBURSEMENT_FLAT_FEE': 'N/A', 'page_num': '31', 'SERVICE': 'PCHP CHIP HMO SERVICES', 'REIMBURSEMENT_METHODOLOGY': "Provider's Charges", 'REIMBURSEMENT_FEE_SCHEDULE': 'N/A', -'REIMBURSEMENT_FEE_SCHEDULE_VERSION': 'N/A', 'REIMBURSEMENT_EXCEPTION_IND': 'N', 'REIMBURSEMENT_DESCRIBE_EXCEPTION': 'N/A', 'RATE_ESCALATOR_IND': 'Y', -'RATE_ESCALATOR_BASIS': 'Prevailing yearly and current Medicaid fee schedule for the State of Texas', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE': '100', 'REIMBURSEMENT_ADMITTYPE_CODES': 'N/A', -'REIMBURSEMENT_DIAG_CODES': 'N/A', 'REIMBURSEMENT_GROUPER_CODES': 'N/A', 'REIMBURSEMENT_GROUPER': 'N/A', 'REIMBURSEMENT_PLACEOFSERVICE_CODES': 'N/A', 'REIMBURSEMENT_PROC_CODES': 'N/A', -'REIMBURSEMENT_REVENUE_CODES': 'N/A', 'REIMBURSEMENT_STATUS_INDICATOR_CODES': 'N/A', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt'}] - - -td_results = [{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan Inc., OPTionah Care Care Options forkids Services', 'page_num': '1', -'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '2', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', 'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', -'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP PLAN', 'page_num': '3', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '4', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP PLAN', 'page_num': '5', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '6', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '7', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'N/A', 'page_num': '8', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '9', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP Plan, PCHP', 'page_num': '10', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '11', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan', 'page_num': '12', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '13', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'PCHP', 'page_num': '14', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '15', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '16', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '17', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP Plan', 'page_num': '18', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '19', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '20', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid,CHIP', 'CONTRACT_PROGRAM': 'STAR,CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '21', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid, CHIP', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '22', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '23', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid, Medicare', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '24', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'PCHP', 'page_num': '25', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '26', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '27', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'CHIP', 'CONTRACT_PROGRAM': 'CHIP', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan, Inc.', 'page_num': '28', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'Parkland Community Health Plan, Care opTions for kids', 'page_num': '29', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'MAY 01 2000', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'MAY 01 2000'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '30', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'Medicaid', 'CONTRACT_PROGRAM': 'STAR, CHIP', 'CONTRACT_NETWORK': 'HMO', 'PRODUCT': 'HEALTHfirst, KIDSfirst', 'page_num': '31', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}, -{'CONTRACT_LOB': 'N/A', 'CONTRACT_PROGRAM': 'N/A', 'CONTRACT_NETWORK': 'N/A', 'PRODUCT': 'N/A', 'page_num': '32', 'Filename': '2.2000_OptionalCare.INC_ServicesAgreement MU.txt', -'CONTRACT_MARKETPLACE_METAL_LEVEL': 'N/A', 'LOB_PRICING_TERMS_EFFECTIVE_DATE': 'N/A', 'LOB_PRICING_TERMS_TERMINATION_DATE': 'N/A'}] - - - - - +sample_results = pd.read_csv('output/ASMG MU/combined_results_post_processed.csv') + +df = postprocessingfuncs.clean_dates(sample_results) +print(df) From 7ac8f5a1baa699415ffa3723bb5135387a82f675 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 21 Jun 2024 15:04:33 -0700 Subject: [PATCH 64/78] Updated Lesser Of methodology --- src/postprocessingfuncs.py | 1 + src/prompt_funcs.py | 50 ++++++++++++++++++++++- src/prompts.py | 18 +++++---- src/test.py | 82 ++++++++++++++++++++++++++++---------- 4 files changed, 122 insertions(+), 29 deletions(-) diff --git a/src/postprocessingfuncs.py b/src/postprocessingfuncs.py index 344f4da..0f45fe5 100644 --- a/src/postprocessingfuncs.py +++ b/src/postprocessingfuncs.py @@ -452,3 +452,4 @@ def clean_dates(df): df.loc[idx, 'LOB_PRICING_TERMS_TERMINATION_DATE'] = 'N/A' return df + diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index db21445..674b185 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -1,5 +1,6 @@ import json +import difflib import config import prompts @@ -8,8 +9,44 @@ import dict_operations import utils import postprocessingfuncs +def get_least_similar(dict_list, original_dict, field): + print(f"Running similarity for {field}") + + min_similarity = float('inf') + least_similar_dict = None + for dictionary in dict_list: + # Extract the methodology text + field_text = dictionary[field] + print(f"Field text: {field_text}") + + # Compute similarity using difflib + similarity = difflib.SequenceMatcher(None, field_text, original_dict[field]).ratio() + print(f"Simlarity: {similarity}") + + # If the similarity is less than the current minimum, update the minimum and the corresponding dictionary + if similarity < min_similarity: + min_similarity = similarity + least_similar_dict = dictionary + + return least_similar_dict -def run_bottom_up_lesser(d, text_dict, tokens=8000): +def get_lesser_of_dict(dict_list, original_dict): + + unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list}) + unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list}) + + # If the rates are different, return the lesser of dict with a different rate than original + if len(unique_rates) > 1: + least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_RATE') + elif len(unique_fees) > 1: + least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_FLAT_FEE') + else: + least_similar_dict = get_least_similar(dict_list, original_dict, 'FULL_METHODOLOGY') + + return least_similar_dict + + +def run_bottom_up_lesser(d, text_dict, tokens=4000): """ Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of', using a language model to analyze and extract relevant data from the identified text segment. @@ -31,7 +68,16 @@ def run_bottom_up_lesser(d, text_dict, tokens=8000): prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - lesser_of_dict = dict_operations.secondary_string_to_dict(lesser_of_answer) + lesser_of_dict_list = dict_operations.primary_string_to_dict({d['page_num'] : lesser_of_answer}, d['Filename']) + print(lesser_of_dict_list) + # If lesser of is Y + if len(lesser_of_dict_list) > 1: + lesser_of_dict = get_lesser_of_dict(lesser_of_dict_list, d) + else: + d['LESSER_OF_LANGUAGE_IND'] = 'N' + d['GREATER_OF_LANGUAGE_IND'] = 'N' + return ({}, d) + lesser_of_dict['page_num'] = d['page_num'] if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': lesser_of_dict['SERVICE'] = d['SERVICE'] diff --git a/src/prompts.py b/src/prompts.py index bbd5729..33fabf9 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -78,25 +78,29 @@ def BOTTOM_UP_LESSER(d, page): ### PAGE START ### {page} ### PAGE END The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: -Service: {d['SERVICE']} -Original_Methodology: {d['FULL_METHODOLOGY']} +Original_Dictionary: +- Original_Service: {d['SERVICE']} +- Original_Rate: {d['REIMBURSEMENT_RATE']} +- Original_Flat_Fee: {d['REIMBURSEMENT_FLAT_FEE']} +- Original_Methodology: {d['FULL_METHODOLOGY']} Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above. Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply. -Read and analyze the page, then populate a JSON dictionary with the following fields (descriptions provided): +Read and analyze the page, then populate two JSON dictionaries with the following fields (descriptions provided): 'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. 'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. -'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence - only include the half of the lesser/greater of statement that is NOT Original_Methodology. For example, if the reimbursement is the lesser of X or Original_Methodology, return only X. +'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. 'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. Only include percentages. If not, write N/A. 'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. Only include dollar values. If not, write N/A. -For REIMBURSEMENT_RATE and REIMBURSEMENT_FLAT_FEE, do NOT include the Rate or Flat Fee from Original_Methodology. +If either LESSER_OF_LANGUAGE_IND or GREATER_OF_LANGUAGE_IND is 'Y', then both dictionaries must be populated. One of the dictionaries will have REIMBURSEMENT_RATE, REIMBURSEMENT_FLAT_FEE, and FULL_METHODOLOGY values equal to the Original term. +Fill the first dictionary with the values corresponding to the LEFT half of the Lesser/Greater Of Statement, and the second dictionary with the RIGHT half of the Lesser/Greater Of Statement. -Note that in some cases, the REIMBURSEMENT_RATE will not be explicitly written. It may say something like "Provider's billed charges", which is essentially equivalent to "100% of Provider's billed charges". In these cases, assign REIMBURSEMENT_RATE a value of 100%. +If both LESSER_OF_LANGUAGE_IND and GREATER_OF_LANGUAGE_IND are 'N', return a list with just the original dictionary. -Only return the dictionary, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +Only return the list of dictionaries, with no other commentary or explanation. Ensure you abide by proper JSON formatting. """ def BOTTOM_UP_EXCEPT_ESC(d, page): diff --git a/src/test.py b/src/test.py index 994e0d1..55d4330 100644 --- a/src/test.py +++ b/src/test.py @@ -17,34 +17,76 @@ import prompts import prompt_funcs import claude_funcs import merge_funcs +import dict_operations -# input_dict = utils.read_input() +input_dict = utils.read_input() -# (filename, contract_text) = list(input_dict.items())[0] +(filename, contract_text) = list(input_dict.items())[0] -# contract_text = preprocess.clean_newlines(contract_text) -# contract_text = preprocess.clean_billed_charges(contract_text) -# text_dict = preprocess.split_text(contract_text) -# text_dict = table_funcs.align_and_format_tables(text_dict) -# text_dict = preprocess.highlight_rates(text_dict) - -# for page_num, page_text in text_dict.items(): -# if page_num == '31': -# prompt = prompts.TOP_DOWN_PRIMARY(page_text) -# answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) -# answer_dict = json.loads(answer) - -# print(answer_dict) +contract_text = preprocess.clean_newlines(contract_text) +contract_text = preprocess.clean_billed_charges(contract_text) +text_dict = preprocess.split_text(contract_text) +text_dict = table_funcs.align_and_format_tables(text_dict) +text_dict = preprocess.highlight_rates(text_dict) -# bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries -# print(bu_results) +def BOTTOM_UP_LESSER(d, page): + return f""" +### PAGE START ### {page} ### PAGE END + +The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: +Original_Dictionary: +- Original_Service: {d['SERVICE']} +- Original_Rate: {d['REIMBURSEMENT_RATE']} +- Original_Flat_Fee: {d['REIMBURSEMENT_FLAT_FEE']} +- Original_Methodology: {d['FULL_METHODOLOGY']} + +Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above. + +Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply. + +Read and analyze the page, then populate two JSON dictionaries with the following fields (descriptions provided): +'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. +'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. +'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. +'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. Only include percentages. If not, write N/A. +'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. Only include dollar values. If not, write N/A. + +If either LESSER_OF_LANGUAGE_IND or GREATER_OF_LANGUAGE_IND is 'Y', then both dictionaries must be populated. One of the dictionaries will have REIMBURSEMENT_RATE, REIMBURSEMENT_FLAT_FEE, and FULL_METHODOLOGY values equal to the Original term. +Fill the first dictionary with the values corresponding to the LEFT half of the Lesser/Greater Of Statement, and the second dictionary with the RIGHT half of the Lesser/Greater Of Statement. + +If both LESSER_OF_LANGUAGE_IND and GREATER_OF_LANGUAGE_IND are 'N', return a list with just the original dictionary. + +Only return the list of dictionaries, with no other commentary or explanation. Ensure you abide by proper JSON formatting. +""" + + +d = {'SERVICE': 'Inpatient Services', + 'REIMBURSEMENT_FLAT_FEE': 'N/A', + 'REIMBURSEMENT_RATE': '105%', + 'FULL_METHODOLOGY': "The lesser of 105% of the West Virginia Medicaid DRG (based on Hospital's current DHHR payment rate) or 100% of Hospital's allowable billed charges.", + 'page_num': '26', + 'Filename': 'App Regional Healthcare_Hospital Agreement_20160414_Dually Executed WV MCD-C13439125AA.txt'} + +answer = prompt_funcs.run_bottom_up_lesser(d, text_dict) +print(answer) + + +# answer_strings = prompt_funcs.run_bottom_up_primary({'3' : text_dict['3']}, 8000) +# answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) +# answer_dicts_filtered = postprocessingfuncs.filter_service_column(answer_dicts) + +# print(answer_dicts_filtered) + +# for d in answer_dicts_filtered: +# print(f"Original dictionary: {d}") +# # Bottom Up Lesser +# if config.RUN_LESSER: +# lesser_object = prompt_funcs.run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens=4000) +# print(f"Lesser dictionary: {lesser_object}") -sample_results = pd.read_csv('output/ASMG MU/combined_results_post_processed.csv') -df = postprocessingfuncs.clean_dates(sample_results) -print(df) From 23ef950f36255b6b31d5b452ea07dbb781b3ad1b Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 24 Jun 2024 09:23:37 -0700 Subject: [PATCH 65/78] Postprocessing bug fixes - produced batch1 output on 6/24 --- src/config.py | 6 +++--- src/postprocess.py | 7 ++++--- src/preprocess.py | 4 +++- src/prompt_funcs.py | 10 +++++----- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/config.py b/src/config.py index 5ab60b8..fa341aa 100644 --- a/src/config.py +++ b/src/config.py @@ -54,9 +54,9 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXFKDEEC5H" -AWS_SECRET_ACCESS_KEY="T1pKiCx9I4sN2X31l9MY14JX7PN9o2nujjquhqER" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjELf//////////wEaCXVzLWVhc3QtMiJIMEYCIQDJEC9kUJpl9GatZyAGVs3gPioSsxInFuplep+B293IcQIhAIFh2IbYCFGL0JIFjSnJt3lRGGhci7kNZwz7qORA1iRCKoMDCGAQABoMNjYwMTMxMDY4NzgyIgzN96Wl+CMwk0MLRtsq4ALq4KyN8E6jUsJ5QR6THkpAR65ZN85WXZlWncCZlMatC8vquuHpQIuHgCHIuIM4W2KWmH12FlXepLcwcg0m2ZXm6bi74HXqk7RAK3afKlZRGjdIv+BUuu4Xm7kq1X9Eq8/5u8G80AZFdK33TzUBXoDu6UlawckSfB5zwy+hFKDwtRWGWgL/xr/bEUXoDcrx5XcklLAlxoyVNEwOQ/ZQEV6mOyGeB/VipZ/KlQc+sFXumtkPCydga/CtVs1/vcFzCGf5XOAoFfaSUrganjSSpUQutGkH0BlPLxmUyagNlbbEcBM410AviA4dGHujNnBKtYmNkrtj2xwMcx73PrKwxg53DyzIYCPB7jTLnkmea2WfozXpzcEP2D0k8JDlmxJNPlHCOwUPoqUppPQXQZviR34x/UpXtCbP65zGOd+za1gOcIAsQEwer9urYF8GiFFYWpQAT/Wed1Zv4QoY7rqDTL6PMN+n1rMGOqUBNlpi7WilJtJvxsuPrKz8gKwZEDIExcAcxWthTFRlsbavMTsxP8kInjcnBg2/r7ot0F6ddIp9iJXGDS9dROpopvSmBPnc2UnYbgpIqoebs+viawLUyXb/+b6KDmHhMQbCpeu4rczqMbBoUjG0plteugGEtYo5rNZHAvnJONBWOUxAxcAo1X88DyAWLpaHg44mnB2lSezviTnaOJvEBaqyrjF+Y9/J" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXDIZUYWWF" +AWS_SECRET_ACCESS_KEY="OFmTz8zdH+d63adfyKUXqefv9hH31K8YryJMWv8v" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEP///////////wEaCXVzLWVhc3QtMiJIMEYCIQDfd7FVLKfCelPBzBlulypHwvAT+Qe3/Z9PFy/5U+rdDgIhAJ9kXpN4c16rmv26mnKlSQ+mMNqKzz0g8gk+zvHUbsxKKowDCKj//////////wEQABoMNjYwMTMxMDY4NzgyIgyOxND9AUVES8laaowq4ALruIJOnTW8Wb1DODUG/ftp8Zr/J9tnqZX8gmTeFK6v1/h++iWTwvD2gdX/MIKDNBL7acFV7DqeffShDH8rdSR2KsGplsZUMkeu/nzt9TSvE69OvyTwIb/Jw45q3PCnnClytwZIuW2kcxawsDHUO/nEVr/e1Y6PyvHs4fhaC5+6BlIjVqpCK3bM4Pk3K+EgBbZkRXSqm3zEAwFYBbRnXONBHWSrpADGxhwJkqo8yUiyBoDysl8rDQTeATrwDawU9vmEmbRJH4XUHxdtOWixUhbzwyTnB1QW9lMdi6BL+GzSGj4XLZWzJj4EGW32g70yamnb6JyV9Ju7Uzsbrusz1Qxg9/apmGmP4fAigzV3r1chv/Vb8Lukm+ljB82+ZqSyiy5bukdAfDoObjdxe/BGLPGkLVN9bn3j29lJs8+tNjfmlTr4boXEj3s72KRwSyEpIw+Tr/MY9yP/mkf+vzti2l6rMKuO5rMGOqUBvP5K7JUpoUMIx2zhz+Legs1pDzcnhQ3Hc0wcd2QqS3crnENTGIFKNEJ/XIbDwTwcnIiiJYsPxKfedv8H1ZBFFNA7sNWggPtlPolKLyxZHfs3RZMsp+MOvR55fgyvC14REfALGRcgqNHsoYI61jrW715rSMI8di7Qv8MvSXICVNUI6+hGAPqs9tLeJBHnaEcclOqR5/YQ5rNq5gnQvD7117MSW5Sk" # File Paths LOCAL_PATH = 'data/test/' # Replace with local diff --git a/src/postprocess.py b/src/postprocess.py index 3d27cd2..e28b150 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -5,8 +5,6 @@ import postprocessingfuncs import config - - def postprocess_results(combined_df): # Define valid values dictionary @@ -17,7 +15,10 @@ def postprocess_results(combined_df): } # Sanitize the combined data - df = postprocessingfuncs.sanitize_combined(df) + try: + df = postprocessingfuncs.sanitize_combined(combined_df) + except Exception as e: + print(f"Postprocessing Error - santize_combined : {e}") # Clean specific columns for exact matches for field_list in [['CONTRACT_LOB', 'Corrected_LOB', config.VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', config.VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', config.VALID_NETWORKS]]: diff --git a/src/preprocess.py b/src/preprocess.py index 5298feb..d2bb5e1 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -121,7 +121,9 @@ def clean_billed_charges(contract_text): "Provider's Charges", "Allowable Charges", "Hospital's Charges", - "Billed Charges" + "Billed Charges", + "the rates set forth in this Exhibit", + "the rutes set forth in this Exhibit", ] max_substring_length = max([len(s) for s in substrings]) diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 674b185..3cba271 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -17,11 +17,11 @@ def get_least_similar(dict_list, original_dict, field): for dictionary in dict_list: # Extract the methodology text field_text = dictionary[field] - print(f"Field text: {field_text}") + # print(f"Field text: {field_text}") # Compute similarity using difflib - similarity = difflib.SequenceMatcher(None, field_text, original_dict[field]).ratio() - print(f"Simlarity: {similarity}") + similarity = difflib.SequenceMatcher(None, str(field_text), str(original_dict[field])).ratio() + # print(f"Simlarity: {similarity}") # If the similarity is less than the current minimum, update the minimum and the corresponding dictionary if similarity < min_similarity: @@ -31,7 +31,7 @@ def get_least_similar(dict_list, original_dict, field): return least_similar_dict def get_lesser_of_dict(dict_list, original_dict): - + unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list}) unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list}) @@ -69,7 +69,7 @@ def run_bottom_up_lesser(d, text_dict, tokens=4000): prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) lesser_of_dict_list = dict_operations.primary_string_to_dict({d['page_num'] : lesser_of_answer}, d['Filename']) - print(lesser_of_dict_list) + # print(lesser_of_dict_list) # If lesser of is Y if len(lesser_of_dict_list) > 1: lesser_of_dict = get_lesser_of_dict(lesser_of_dict_list, d) From 9d051af3a2e0d11f32df5f8ebb7f643e3e75d2f2 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 24 Jun 2024 21:08:03 -0700 Subject: [PATCH 66/78] Bug fixes --- src/file_processing.py | 6 ++++-- src/prompt_funcs.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/file_processing.py b/src/file_processing.py index b48ef8e..abf292f 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -13,6 +13,8 @@ import merge_funcs def process_file(file_object): """ + file_object = (filename , "file contents in one string"} + Processes a single file object containing text data from a contract. This function orchestrates the full processing workflow for a single document, including: @@ -70,10 +72,10 @@ def process_file(file_object): ################## RUN POSTPROCESSING ################## # combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated - combined_df = combined_df.apply(lambda x: x.map(postprocessingfuncs.sanitize_value)) + post_processed_combined_df = postprocess.postprocess_results(combined_df) - post_processed_combined_df.to_csv(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) + post_processed_combined_df.to_excel(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) print(f"Postprocessing Complete - {filename} ") print(f"Output Complete - {filename} ") diff --git a/src/prompt_funcs.py b/src/prompt_funcs.py index 3cba271..f06aeb5 100644 --- a/src/prompt_funcs.py +++ b/src/prompt_funcs.py @@ -10,7 +10,7 @@ import utils import postprocessingfuncs def get_least_similar(dict_list, original_dict, field): - print(f"Running similarity for {field}") + # print(f"Running similarity for {field}") min_similarity = float('inf') least_similar_dict = None @@ -31,7 +31,7 @@ def get_least_similar(dict_list, original_dict, field): return least_similar_dict def get_lesser_of_dict(dict_list, original_dict): - + unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list}) unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list}) From dc84a5ae24a90678a804a8fb1023fd8c5da066df Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 26 Jun 2024 12:05:21 -0700 Subject: [PATCH 67/78] Test commit - PROV_TYPE addition --- src/config.py | 20 ++++++++++---------- src/file_processing.py | 3 +-- src/prompts.py | 1 + 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/config.py b/src/config.py index fa341aa..63a678b 100644 --- a/src/config.py +++ b/src/config.py @@ -30,16 +30,16 @@ UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings -MAX_WORKERS = 3 +MAX_WORKERS = 5 # Prompt Debugging RUN_PRIMARY = True # Always True RUN_LOB = True -RUN_LESSER = True -RUN_METHODOLOGY = True -RUN_FS = True -RUN_EXCEPTION = True -RUN_CODES = True +RUN_LESSER = False +RUN_METHODOLOGY = False +RUN_FS = False +RUN_EXCEPTION = False +RUN_CODES = False # Postprocessing Settings FUZZY_MATCH_THRESHOLD = 0.8 @@ -54,12 +54,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXDIZUYWWF" -AWS_SECRET_ACCESS_KEY="OFmTz8zdH+d63adfyKUXqefv9hH31K8YryJMWv8v" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEP///////////wEaCXVzLWVhc3QtMiJIMEYCIQDfd7FVLKfCelPBzBlulypHwvAT+Qe3/Z9PFy/5U+rdDgIhAJ9kXpN4c16rmv26mnKlSQ+mMNqKzz0g8gk+zvHUbsxKKowDCKj//////////wEQABoMNjYwMTMxMDY4NzgyIgyOxND9AUVES8laaowq4ALruIJOnTW8Wb1DODUG/ftp8Zr/J9tnqZX8gmTeFK6v1/h++iWTwvD2gdX/MIKDNBL7acFV7DqeffShDH8rdSR2KsGplsZUMkeu/nzt9TSvE69OvyTwIb/Jw45q3PCnnClytwZIuW2kcxawsDHUO/nEVr/e1Y6PyvHs4fhaC5+6BlIjVqpCK3bM4Pk3K+EgBbZkRXSqm3zEAwFYBbRnXONBHWSrpADGxhwJkqo8yUiyBoDysl8rDQTeATrwDawU9vmEmbRJH4XUHxdtOWixUhbzwyTnB1QW9lMdi6BL+GzSGj4XLZWzJj4EGW32g70yamnb6JyV9Ju7Uzsbrusz1Qxg9/apmGmP4fAigzV3r1chv/Vb8Lukm+ljB82+ZqSyiy5bukdAfDoObjdxe/BGLPGkLVN9bn3j29lJs8+tNjfmlTr4boXEj3s72KRwSyEpIw+Tr/MY9yP/mkf+vzti2l6rMKuO5rMGOqUBvP5K7JUpoUMIx2zhz+Legs1pDzcnhQ3Hc0wcd2QqS3crnENTGIFKNEJ/XIbDwTwcnIiiJYsPxKfedv8H1ZBFFNA7sNWggPtlPolKLyxZHfs3RZMsp+MOvR55fgyvC14REfALGRcgqNHsoYI61jrW715rSMI8di7Qv8MvSXICVNUI6+hGAPqs9tLeJBHnaEcclOqR5/YQ5rNq5gnQvD7117MSW5Sk" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXOTBL4MTJ" +AWS_SECRET_ACCESS_KEY="hjtfO0nRThW+Yjx1ZosTP3ygIS8BsxugdFMpODzA" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDEaCXVzLWVhc3QtMiJHMEUCIEziuIpZOGtLwXWxemENiZ1kT6gOVOcuEmtlctZVIU+XAiEA2VaqgTrPEZDy0bwiURWs5mcr6A/wDzjo6SCtZUHjuCUqjAMI2v//////////ARAAGgw2NjAxMzEwNjg3ODIiDNRd1inJC+42e/lPmyrgAhhLfpQiMtAmKbEgupGdXCH3yjE3MCtuLg4aVP38uZAXXRt5YC5EDuUC1CQWpEwrQZhx68aQz+4Te+VluBgivY9jGAxomZiVPUDqpU7Q08EzbxJVCE7mjPWLbREk9wK+w8DsBc7VLub6qwDdzzsWtGYQTJP6j9cuOdl4Ddpt82mbCKce7GlsF1cRpumnh0wF+yo0LwVIzTY63HwIQbJIdDUGpGR5vC+3kn4tfT6XcHFAZx34EF6QmgWth/FBUCCyPbd24PaCCsmtSplnpWakUKnIOPv1Ie8RO71BO/z5SfoKrtXEPxy3MwqGM+1wjKqmWjsLfABqKyaUqv+RS8s6nPMY0lOUp2RrXr5WOkMfWjs+tts3YynPurp+D9PtE0ERaGI4bNRJyIUceNFphovQltprFH8EmnEDm4g+WO9xxiHNltdeLnqGl0BWDjGhF2IUzm7g9pigraE7yJlLR+/e0TUw8ZDxswY6pgH3XBkw9xYUk+9MKxwfs2Cm+fK2dwpjJ1VOuWzOXZF8fP1tuQ9UPF0NH97iuBz5pTNPYTc3lE5x7ycWP8xc1NuXDpyrTFboL6ISrKZPYJ5w+06+JsLtN8CNhEoa4pQCSBW1t1OcINEssRbtohM2tZnR0G0Bw1LaTIBmNLQDktqcWc1C3mKvrq1R8OxYmD3g9R66hovCfrBXdhsttucDAO6eMA5j7XPh" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/all_txt_updated_batch_3_c/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index abf292f..51634dd 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -73,9 +73,8 @@ def process_file(file_object): ################## RUN POSTPROCESSING ################## # combined_df = combined_df.applymap(postprocessingfuncs.sanitize_value) # Deprecated - post_processed_combined_df = postprocess.postprocess_results(combined_df) - post_processed_combined_df.to_excel(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) + post_processed_combined_df.to_csv(os.path.join(output_dir, config.PROCESSED_RESULTS_NAME), index=False) print(f"Postprocessing Complete - {filename} ") print(f"Output Complete - {filename} ") diff --git a/src/prompts.py b/src/prompts.py index 33fabf9..abae021 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -191,6 +191,7 @@ Here are the attributes to be included in each dictionary, and instructions on h 'CONTRACT_PROGRAM' : List any Programs mentioned on the page. Some examples of Programs are: CHIP, CHIP-P, CHIP-Perinate, STAR, STAR+PLUS. Do your best to infer the program from this page. 'CONTRACT_NETWORK' : List any Networks mentioned on the page. Networks are types of managed care organizations involved in the delivery of healthcare services. Some examples of Networks are: HMO, PPO, EMO, POS, FFS. Do your best to infer the network from this page. If it says anywhere that they are out of network then only return "OUT OF NETWORK" as your answer. 'PRODUCT' : List any Products listed on the page. These are typically the brand name of a health plan. You may also use this field as a catch-all for entities that you are unsure of. +'PROV_TYPE' : List any Provider Type listed on the page. This may be something like Inpatient, Outpatient, Ambulatory Surgery Center, DME, Home Health, Hospice, Skilled Nursing Facility, or similar. If any of the attributes are not found, return N/A. If there are multiple answers for a given attribute, return the answers in a comma-separated list. From 37ef006986cda77a7b05b126df237887857487bb Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 26 Jun 2024 18:42:07 -0700 Subject: [PATCH 68/78] Fixed output issues --- src/config.py | 22 +++++++++++----------- src/consolidate_output.py | 2 +- src/postprocess.py | 4 ++++ src/utils.py | 6 +++--- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/config.py b/src/config.py index 63a678b..cdadfca 100644 --- a/src/config.py +++ b/src/config.py @@ -23,23 +23,23 @@ READ_MODE = '_LOCAL_' # OR '_S3_' WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written OUTPUT_DIRECTORY = 'output' CONSOLIDATED_OUTPUT_DIRECTORY = 'output_consolidated' -OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.xlsx' +OUTPUT_CSV_PATH = f'consolidated_output_{TODAY}.csv' TD_RESULTS_NAME = 'td_results.csv' BU_RESULTS_NAME = 'bu_results.csv' UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings -MAX_WORKERS = 5 +MAX_WORKERS = 3 # Prompt Debugging RUN_PRIMARY = True # Always True RUN_LOB = True -RUN_LESSER = False -RUN_METHODOLOGY = False -RUN_FS = False -RUN_EXCEPTION = False -RUN_CODES = False +RUN_LESSER = True +RUN_METHODOLOGY = True +RUN_FS = True +RUN_EXCEPTION = True +RUN_CODES = True # Postprocessing Settings FUZZY_MATCH_THRESHOLD = 0.8 @@ -54,12 +54,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXOTBL4MTJ" -AWS_SECRET_ACCESS_KEY="hjtfO0nRThW+Yjx1ZosTP3ygIS8BsxugdFMpODzA" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDEaCXVzLWVhc3QtMiJHMEUCIEziuIpZOGtLwXWxemENiZ1kT6gOVOcuEmtlctZVIU+XAiEA2VaqgTrPEZDy0bwiURWs5mcr6A/wDzjo6SCtZUHjuCUqjAMI2v//////////ARAAGgw2NjAxMzEwNjg3ODIiDNRd1inJC+42e/lPmyrgAhhLfpQiMtAmKbEgupGdXCH3yjE3MCtuLg4aVP38uZAXXRt5YC5EDuUC1CQWpEwrQZhx68aQz+4Te+VluBgivY9jGAxomZiVPUDqpU7Q08EzbxJVCE7mjPWLbREk9wK+w8DsBc7VLub6qwDdzzsWtGYQTJP6j9cuOdl4Ddpt82mbCKce7GlsF1cRpumnh0wF+yo0LwVIzTY63HwIQbJIdDUGpGR5vC+3kn4tfT6XcHFAZx34EF6QmgWth/FBUCCyPbd24PaCCsmtSplnpWakUKnIOPv1Ie8RO71BO/z5SfoKrtXEPxy3MwqGM+1wjKqmWjsLfABqKyaUqv+RS8s6nPMY0lOUp2RrXr5WOkMfWjs+tts3YynPurp+D9PtE0ERaGI4bNRJyIUceNFphovQltprFH8EmnEDm4g+WO9xxiHNltdeLnqGl0BWDjGhF2IUzm7g9pigraE7yJlLR+/e0TUw8ZDxswY6pgH3XBkw9xYUk+9MKxwfs2Cm+fK2dwpjJ1VOuWzOXZF8fP1tuQ9UPF0NH97iuBz5pTNPYTc3lE5x7ycWP8xc1NuXDpyrTFboL6ISrKZPYJ5w+06+JsLtN8CNhEoa4pQCSBW1t1OcINEssRbtohM2tZnR0G0Bw1LaTIBmNLQDktqcWc1C3mKvrq1R8OxYmD3g9R66hovCfrBXdhsttucDAO6eMA5j7XPh" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXA4FIOPGS" +AWS_SECRET_ACCESS_KEY="sr07RX66lZFKkFM6C6gyOrfETnXX0z4kZOgwuB20" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJGMEQCIC1q2oui6BbOEUpov26xm4L85P+6STphw9V/eLDAQIGBAiBqZ6BQKomcXSApXh08SyJqCzAWjQxwXjmhODoCSnUwGyqMAwje//////////8BEAAaDDY2MDEzMTA2ODc4MiIMFy55wPIl3rmecfZuKuACTIz7Olv6aW2ooUVgxD74K0jAQUIVQmyAMPqMumJBaFkXg3MpB9nnedSplK/GLNVJZ4/cYTF+6DkdNLmnz9zi3tk4cexYggOKn/vIVuqJ9ha9fmCmP30Wrk3KTzILS3d+xNn2TMeQojj7PPsVjZv7VloQNzjqijN+ZmOg64/dnQmFLD8RFcjK6SbOk2aDa9rWB2qi2go2PN43quKAHR8rdPI3vCVh2mVkx/SlTI7M/mMfDeVIgjaY9kDxoA85YmmXmn1xV5kU3//K782Jhg6DydHE4Zpu06CrVQvp56Ks/+BP+R/Ygj8zGAODPyWRNzBnw0roVQ0LJ7O0b2O12/AZ3TuuLzhgygM3z3WW54bm++DVVz6m5A0oVVAjOkyty6rfFYpXL1jofkcf4Xk1Us2C7VLy9MMo1o5lJjPWnAFcjMekrQOYa4fGiIAmNuzsij1eLzzy39TW43elB02cgs55hDDfgvKzBjqnAeXxP0G7j+VW61r+N3xVyJEpp4Kof0jlEZKW7EFUWhQe0h9/6UdRo5j40vMuxpm+mhzZq11kbHoovi/8VPrkmGc/Es//Zdzyk54t+E86k7U9XBBXmm1HpY1kkXKtIBMgxG0DCHObnsK1rgv8367Tm3Htydxa35PYiPD7WdVGDS7nNWm6vlrvazLlLA+wQ6vRodVwLpc8j33eLxKWfzAPw5+Lgi/Ph38i" # File Paths -LOCAL_PATH = 'data/all_txt_updated_batch_3_c/' # Replace with local +LOCAL_PATH = 'data/test/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/consolidate_output.py b/src/consolidate_output.py index 3a7b3bf..e72493c 100644 --- a/src/consolidate_output.py +++ b/src/consolidate_output.py @@ -15,4 +15,4 @@ for file in os.listdir(config.OUTPUT_DIRECTORY): final_df = pd.concat(all_dfs, ignore_index=True) -final_df.to_excel(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) \ No newline at end of file +final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) \ No newline at end of file diff --git a/src/postprocess.py b/src/postprocess.py index e28b150..93c04ae 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -62,6 +62,10 @@ def postprocess_results(combined_df): except Exception as e: print(f"Postprocessing Error - clean_dates : {e}") + # Re order columns + column_order = [col for col in config.VALID_COLUMNS if col in df.columns] + [col for col in df.columns if col not in config.VALID_COLUMNS] + df = df[column_order] + return df diff --git a/src/utils.py b/src/utils.py index 43a16b0..d8b1e0f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -109,8 +109,8 @@ def format_td_check(td_dicts, dont_include_list): return final_str -def consolidate_csvs(output_dir, output_file): - os.makedirs(config.CONSOLIDATED_OUTPUT_DIRECTORY, exist_ok=True) +def consolidate_csvs(output_dir=config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file=config.OUTPUT_CSV_PATH): + os.makedirs(output_dir, exist_ok=True) df_list = [] # Walk through each folder in the output directory @@ -126,7 +126,7 @@ def consolidate_csvs(output_dir, output_file): pass concatenated_df = pd.concat(df_list, ignore_index=True) - concatenated_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, output_file), index=False) + concatenated_df.to_csv(os.path.join(output_dir, output_file), index=False) print(f"All CSV files have been consolidated into {output_file}") From d5e9b11f1b022e20f387b59345d88e5fda06c154 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 10 Jul 2024 08:17:05 -0700 Subject: [PATCH 69/78] Added fields, file restructuring --- src/{prompt_funcs.py => bottom_up_funcs.py} | 288 +++++------------- src/config.py | 16 +- src/file_processing.py | 8 +- src/main.py | 8 +- src/postprocess.py | 52 +--- ...essingfuncs.py => postprocessing_funcs.py} | 0 src/prompts.py | 5 +- src/test.py | 105 +------ src/top_down_funcs.py | 124 ++++++++ 9 files changed, 255 insertions(+), 351 deletions(-) rename src/{prompt_funcs.py => bottom_up_funcs.py} (61%) rename src/{postprocessingfuncs.py => postprocessing_funcs.py} (100%) create mode 100644 src/top_down_funcs.py diff --git a/src/prompt_funcs.py b/src/bottom_up_funcs.py similarity index 61% rename from src/prompt_funcs.py rename to src/bottom_up_funcs.py index f06aeb5..7bf0ff1 100644 --- a/src/prompt_funcs.py +++ b/src/bottom_up_funcs.py @@ -1,91 +1,39 @@ -import json +import dict_operations +import postprocessing_funcs +import prompts +import config +import claude_funcs + import difflib -import config -import prompts -import claude_funcs -import dict_operations -import utils -import postprocessingfuncs - -def get_least_similar(dict_list, original_dict, field): - # print(f"Running similarity for {field}") - - min_similarity = float('inf') - least_similar_dict = None - for dictionary in dict_list: - # Extract the methodology text - field_text = dictionary[field] - # print(f"Field text: {field_text}") - - # Compute similarity using difflib - similarity = difflib.SequenceMatcher(None, str(field_text), str(original_dict[field])).ratio() - # print(f"Simlarity: {similarity}") - - # If the similarity is less than the current minimum, update the minimum and the corresponding dictionary - if similarity < min_similarity: - min_similarity = similarity - least_similar_dict = dictionary - - return least_similar_dict - -def get_lesser_of_dict(dict_list, original_dict): - - unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list}) - unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list}) - - # If the rates are different, return the lesser of dict with a different rate than original - if len(unique_rates) > 1: - least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_RATE') - elif len(unique_fees) > 1: - least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_FLAT_FEE') - else: - least_similar_dict = get_least_similar(dict_list, original_dict, 'FULL_METHODOLOGY') - - return least_similar_dict - - -def run_bottom_up_lesser(d, text_dict, tokens=4000): +def run_bottom_up(filename, text_dict): """ - Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of', - using a language model to analyze and extract relevant data from the identified text segment. + Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information. - This function takes a dictionary representing initial results and a text dictionary, uses the page number from the initial - results to locate the relevant text, and generates a prompt for language model analysis. Based on the response, it - determines whether 'lesser of' or 'greater of' language is indicated, updates the results, and returns a new dictionary - for the updated data or retains the original if no such language is present. + This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further + refine and structure them into a dictionary form. Parameters: - d (dict): A dictionary containing initial processing results from previous analyses, including a 'page_num' key. + filename (str): The name of the file being processed, used to tag output data. text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed. - tokens (int): Token limit for language model invocations, default set to 8000. Returns: - tuple: A tuple containing two dictionaries, the first for any identified 'lesser of' or 'greater of' data and the second - containing the original or updated data depending on the presence of such language. + list of dict: A list of dictionaries with each dictionary containing refined and structured information + from both primary and secondary Bottom Up analyses, all tagged with the filename. """ - prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) - lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) - lesser_of_dict_list = dict_operations.primary_string_to_dict({d['page_num'] : lesser_of_answer}, d['Filename']) - # print(lesser_of_dict_list) - # If lesser of is Y - if len(lesser_of_dict_list) > 1: - lesser_of_dict = get_lesser_of_dict(lesser_of_dict_list, d) - else: - d['LESSER_OF_LANGUAGE_IND'] = 'N' - d['GREATER_OF_LANGUAGE_IND'] = 'N' - return ({}, d) + # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY + answer_strings = run_bottom_up_primary(text_dict, 8000) + answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) + answer_dicts_filtered = postprocessing_funcs.filter_service_column(answer_dicts) - lesser_of_dict['page_num'] = d['page_num'] - if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': - lesser_of_dict['SERVICE'] = d['SERVICE'] - return (lesser_of_dict, d) - else: - d['LESSER_OF_LANGUAGE_IND'] = 'N' - d['GREATER_OF_LANGUAGE_IND'] = 'N' - return ({}, d) + # Bottom Up Secondary + results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000) + + for d in results_dicts: + d['Filename'] = filename + return results_dicts # List of dictionaries def run_bottom_up_primary(text_dict, tokens): @@ -183,151 +131,83 @@ def run_bottom_up_secondary(answer_dicts, text_dict, tokens): return final_dicts -def run_bottom_up(filename, text_dict): - """ - Processes the text of a document using a two-tiered Bottom Up approach to extract key financial and operational information. - This function first runs BOTTOM_UP_PRIMARY and BOTTOM_UP_SECONDARY prompts, then processes these initial results to further - refine and structure them into a dictionary form. +def run_bottom_up_lesser(d, text_dict, tokens=4000): + """ + Processes specific clauses or conditions within the document text, such as 'lesser of' or 'greater of', + using a language model to analyze and extract relevant data from the identified text segment. + + This function takes a dictionary representing initial results and a text dictionary, uses the page number from the initial + results to locate the relevant text, and generates a prompt for language model analysis. Based on the response, it + determines whether 'lesser of' or 'greater of' language is indicated, updates the results, and returns a new dictionary + for the updated data or retains the original if no such language is present. Parameters: - filename (str): The name of the file being processed, used to tag output data. + d (dict): A dictionary containing initial processing results from previous analyses, including a 'page_num' key. text_dict (dict): A dictionary of text keyed by page numbers that contains the content to be analyzed. + tokens (int): Token limit for language model invocations, default set to 8000. Returns: - list of dict: A list of dictionaries with each dictionary containing refined and structured information - from both primary and secondary Bottom Up analyses, all tagged with the filename. + tuple: A tuple containing two dictionaries, the first for any identified 'lesser of' or 'greater of' data and the second + containing the original or updated data depending on the presence of such language. """ - # Bottom Up Primary - SERVICE, REIMBURSEMENT_FLAT_FEE, REIMBURSEMENT_RATE, METHODOLOGY - answer_strings = run_bottom_up_primary(text_dict, 8000) - answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) - answer_dicts_filtered = postprocessingfuncs.filter_service_column(answer_dicts) - - # Bottom Up Secondary - results_dicts = run_bottom_up_secondary(answer_dicts_filtered, text_dict, 8000) - - for d in results_dicts: - d['Filename'] = filename - return results_dicts # List of dictionaries - - -def run_top_down_metal_level(d, page): - """ - Identifies and extracts the metal level of a contract within a specific line of business (LOB) from the provided page text. - - This function determines whether the contract's LOB is associated with marketplace or commercial sectors by examining the - 'CONTRACT_LOB' key in the dictionary. If relevant, it constructs and sends a specific prompt to a language model to extract the - metal level (e.g., Bronze, Silver, Gold, Platinum). If the LOB is not relevant, it directly assigns 'N/A'. - - Parameters: - d (dict): A dictionary containing extracted information from primary Top Down analysis, specifically the 'CONTRACT_LOB' key. - page (str): The text content of the page that is being analyzed for metal level information. - - Returns: - str: The metal level of the contract as determined by the analysis, or 'N/A' if the contract LOB is not applicable. - """ - - if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper(): - prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + prompt = prompts.BOTTOM_UP_LESSER(d, text_dict[d['page_num']]) + lesser_of_answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) + lesser_of_dict_list = dict_operations.primary_string_to_dict({d['page_num'] : lesser_of_answer}, d['Filename']) + # print(lesser_of_dict_list) + # If lesser of is Y + if len(lesser_of_dict_list) > 1: + lesser_of_dict = get_lesser_of_dict(lesser_of_dict_list, d) else: - answer = 'N/A' - return answer + d['LESSER_OF_LANGUAGE_IND'] = 'N' + d['GREATER_OF_LANGUAGE_IND'] = 'N' + return ({}, d) + + lesser_of_dict['page_num'] = d['page_num'] + if lesser_of_dict['LESSER_OF_LANGUAGE_IND'] == 'Y' or lesser_of_dict['GREATER_OF_LANGUAGE_IND'] == 'Y': + lesser_of_dict['SERVICE'] = d['SERVICE'] + return (lesser_of_dict, d) + else: + d['LESSER_OF_LANGUAGE_IND'] = 'N' + d['GREATER_OF_LANGUAGE_IND'] = 'N' + return ({}, d) -def run_top_down_date(type_, d, page): - """ - Extracts specific date-related information from a page of text using a Top Down processing approach. - This function tailors the extraction to focus on either 'EFFECTIVE' or 'TERMINATION' dates by constructing - a prompt that directs a language model to search for and interpret date information relevant to the provided type. - It formats the input data to fit the processing needs, sends it along with the page text to the model, and captures - the model's response. - - Parameters: - type_ (str): Specifies the type of date to extract, 'EFFECTIVE' or 'TERMINATION'. - d (dict): A dictionary containing preliminary data extracted from the text, which may include metadata like filename or page number. - page (str): The text content of the page from which to extract the date. - - Returns: - str: The extracted date as a string, based on the model's interpretation of the input prompt and text context. - """ - formatted_d = utils.format_td_check([d], ['Filename', 'page_num']) - prompt = prompts.TOP_DOWN_DATE('EFFECTIVE', formatted_d, page) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - return answer - - -def top_down_secondary(td_results, text_dict): - """ - Conducts secondary processing on results obtained from the primary Top Down analysis of document text, enhancing detail and accuracy. - - This function iteratively enhances each dictionary result from the initial analysis by adding or refining information related to: - - Market metal levels by running a specific function to determine the contract marketplace metal level. - - Effective dates by analyzing the context around specific keywords in the text and extracting these dates. - - Termination dates, similarly extracted based on the presence and context of specific keywords. - - Parameters: - td_results (list of dict): Initial dictionaries from the primary Top Down processing containing basic extracted data. - text_dict (dict): Dictionary keyed by page numbers, providing the full text for corresponding analysis and extraction tasks. - - Returns: - list of dict: Updated list of dictionaries, each enhanced with additional detailed attributes like metal level and date specifics. - """ - updated_dicts = [] - for d in td_results: - # Run Metal Level - d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']]) +def get_least_similar(dict_list, original_dict, field): + # print(f"Running similarity for {field}") + + min_similarity = float('inf') + least_similar_dict = None + for dictionary in dict_list: + # Extract the methodology text + field_text = dictionary[field] + # print(f"Field text: {field_text}") - # Dates - d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) + # Compute similarity using difflib + similarity = difflib.SequenceMatcher(None, str(field_text), str(original_dict[field])).ratio() + # print(f"Simlarity: {similarity}") - # if auto_renewal == N: else: 'N/A' - d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) + # If the similarity is less than the current minimum, update the minimum and the corresponding dictionary + if similarity < min_similarity: + min_similarity = similarity + least_similar_dict = dictionary + + return least_similar_dict - updated_dicts.append(d) - return updated_dicts +def get_lesser_of_dict(dict_list, original_dict): + unique_rates = list({d['REIMBURSEMENT_RATE'] for d in dict_list}) + unique_fees = list({d['REIMBURSEMENT_FLAT_FEE'] for d in dict_list}) + # If the rates are different, return the lesser of dict with a different rate than original + if len(unique_rates) > 1: + least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_RATE') + elif len(unique_fees) > 1: + least_similar_dict = get_least_similar(dict_list, original_dict, 'REIMBURSEMENT_FLAT_FEE') + else: + least_similar_dict = get_least_similar(dict_list, original_dict, 'FULL_METHODOLOGY') -def run_top_down(filename, text_dict): - """ - Executes the Top Down processing strategy on a provided dictionary of text pages, extracting structured data based on specified prompts. - - The function first filters and processes each page of text that is correctly numbered, using a primary analysis prompt. The results - from this analysis are then parsed into dictionaries, tagged with their respective page numbers and filename, and collected. - After the initial processing, a secondary Top Down analysis is conducted to refine and possibly expand the extracted data. - - Parameters: - filename (str): The filename associated with the text, used to tag the output data. - text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages. - - Returns: - list of dict: A list of dictionaries where each dictionary contains structured results from both primary and secondary - Top Down analyses, associated with a specific page and the overall document. - """ - - all_results = [] - - # Primary - for page_num, page_text in text_dict.items(): - if not page_num.isdigit(): - continue - prompt = prompts.TOP_DOWN_PRIMARY(page_text) - answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) - answer_dict = json.loads(answer) - answer_dict.update({'page_num' : page_num, 'Filename' : filename}) - #answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to dictionaries - all_results.append(answer_dict) - - # td_primary = [i for d in all_results for i in d] # Consolidate to one list - # print(td_primary) - - # Secondaries - td_final = top_down_secondary(all_results, text_dict) - - return td_final # List of list of dictionaries - - + return least_similar_dict diff --git a/src/config.py b/src/config.py index cdadfca..d63ad69 100644 --- a/src/config.py +++ b/src/config.py @@ -46,20 +46,20 @@ FUZZY_MATCH_THRESHOLD = 0.8 VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'REIMBURSEMENT_RATE', 'FULL_METHODOLOGY', 'REIMBURSEMENT_METHODOLOGY', 'REIMBURSEMENT_FEE_SCHEDULE', 'REIMBURSEMENT_FEE_SCHEDULE_VERSION', 'LESSER_OF_LANGUAGE_IND', 'GREATER_OF_LANGUAGE_IND', -'CONTRACT_LOB', 'CONTRACT_NETWORK', 'PRODUCT', 'CONTRACT_PROGRAM', 'CONTRACT_MARKETPLACE_METAL_LEVEL', +'CONTRACT_LOB', 'CONTRACT_NETWORK', 'PRODUCT', 'CONTRACT_PROGRAM', 'CONTRACT_MARKETPLACE_METAL_LEVEL', 'PROV_TYPE', 'REIMBURSEMENT_PROC_CODES', 'REIMBURSEMENT_PROC_CODE_MODIFIERS', 'REIMBURSEMENT_REVENUE_CODES', 'REIMBURSEMENT_STATUS_INDICATOR_CODES', -'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', -'REIMBURSEMENT_ADMITTYPE_CODES', 'REIMBURSEMENT_EXCEPTION_IND', 'REIMBURSEMENT_DESCRIBE_EXCEPTION', -'RATE_ESCALATOR_IND', 'RATE_ESCALATOR_DESCRIBE', 'LOB_PRICING_TERMS_EFFECTIVE_DATE', 'LOB_PRICING_TERMS_TERMINATION_DATE', +'REIMBURSEMENT_DIAG_CODES', 'REIMBURSEMENT_GROUPER_CODES', 'REIMBURSEMENT_GROUPER', 'REIMBURSEMENT_PLACEOFSERVICE_CODES', 'REIMBURSEMENT_ADMITTYPE_CODES', +'REIMBURSEMENT_EXCEPTION_IND', 'REIMBURSEMENT_DESCRIBE_EXCEPTION', +'RATE_ESCALATOR_IND', 'RATE_ESCALATOR_BASIS', 'RATE_ESCALATOR_YEARLY_PERCENT_INCREASE', 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXA4FIOPGS" -AWS_SECRET_ACCESS_KEY="sr07RX66lZFKkFM6C6gyOrfETnXX0z4kZOgwuB20" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEDUaCXVzLWVhc3QtMiJGMEQCIC1q2oui6BbOEUpov26xm4L85P+6STphw9V/eLDAQIGBAiBqZ6BQKomcXSApXh08SyJqCzAWjQxwXjmhODoCSnUwGyqMAwje//////////8BEAAaDDY2MDEzMTA2ODc4MiIMFy55wPIl3rmecfZuKuACTIz7Olv6aW2ooUVgxD74K0jAQUIVQmyAMPqMumJBaFkXg3MpB9nnedSplK/GLNVJZ4/cYTF+6DkdNLmnz9zi3tk4cexYggOKn/vIVuqJ9ha9fmCmP30Wrk3KTzILS3d+xNn2TMeQojj7PPsVjZv7VloQNzjqijN+ZmOg64/dnQmFLD8RFcjK6SbOk2aDa9rWB2qi2go2PN43quKAHR8rdPI3vCVh2mVkx/SlTI7M/mMfDeVIgjaY9kDxoA85YmmXmn1xV5kU3//K782Jhg6DydHE4Zpu06CrVQvp56Ks/+BP+R/Ygj8zGAODPyWRNzBnw0roVQ0LJ7O0b2O12/AZ3TuuLzhgygM3z3WW54bm++DVVz6m5A0oVVAjOkyty6rfFYpXL1jofkcf4Xk1Us2C7VLy9MMo1o5lJjPWnAFcjMekrQOYa4fGiIAmNuzsij1eLzzy39TW43elB02cgs55hDDfgvKzBjqnAeXxP0G7j+VW61r+N3xVyJEpp4Kof0jlEZKW7EFUWhQe0h9/6UdRo5j40vMuxpm+mhzZq11kbHoovi/8VPrkmGc/Es//Zdzyk54t+E86k7U9XBBXmm1HpY1kkXKtIBMgxG0DCHObnsK1rgv8367Tm3Htydxa35PYiPD7WdVGDS7nNWm6vlrvazLlLA+wQ6vRodVwLpc8j33eLxKWfzAPw5+Lgi/Ph38i" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXHJKTYQ6L" +AWS_SECRET_ACCESS_KEY="H3wyMbmE7UbK/7GFBH0+t9p3pdEJsb5QIixO5BKx" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHQaCXVzLWVhc3QtMiJGMEQCIF9hoZcjho3AvCRXb8YyBYbogCNra8UI/gVKn+Batqz+AiB915kH5kk1WP2FWlZYBPSQ48Qi0pE+A7Cx5IZJGycscSqDAwg9EAAaDDY2MDEzMTA2ODc4MiIMqiCgRpeur6AXeuzTKuACnWFyPwZtV+xbLT94iBuMlSr/v/eecsdq0huqCayieInxYuo1KaLuLTlb2Uh6CweO9FDad58B4Axo/gtFIRZGqy1KArM3gKOEMVPnaMxWz3EHXLZVvCT31BzDNEJM2hC17AjC9cUnwn/+lx5xlL1UweOO5fphHWDBe0DWoF1CUl2ciXr2fxVLilLWjN91ZdJOV97Je2K+LyRpOi8S0076jPolBbENXgDfDzLoRbftDFJXn6aJXsFfOZJ1BRo3kbNBzAlPA+PGcyYqET2cFvA5RY+4yeKQ2jV5eFyeNIRcpmZaD+bW8AnKOe2OYAYg8ft+uSdQrM4pdmJpGT1jRfIfo5psOtvMbIcB4BvxOZFVOn+Zyni4KLFiAnss1ttMYR5skjfTfDzWGtVBl5KH9Crf6pDihWg35IbKHqVqk7HjVMorLAWbjOFJfaR/f8U6dL2ZNB/7ydUFQx0zdp396w4GszClg7i0BjqnAbJHP3p1uYlFN5/kTjzbA8vQHrRsbq6XurwojMfWvhS67zlipnRDqVFFBI7lEYu1au582p2yDZ7A10w1TAXbb/xaoOiU6qkAxk+w+LMlWB15Xmivuq4ahHsDKJ6d09hAEKT3wybzWlr4ps/cEipTiJtDAK9mfkj5Wa9J/Hwj+s223wY158r6yEvRIrDAiL0o5Gc+8+mvYTWD/UGC0qQyXp23mKaJcbJy" # File Paths -LOCAL_PATH = 'data/test/' # Replace with local +LOCAL_PATH = 'data/batch1/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/file_processing.py b/src/file_processing.py index 51634dd..42a6631 100644 --- a/src/file_processing.py +++ b/src/file_processing.py @@ -5,8 +5,8 @@ import pandas as pd import config import preprocess import table_funcs -import prompt_funcs -import postprocessingfuncs +import bottom_up_funcs +import top_down_funcs import postprocess import merge_funcs @@ -53,12 +53,12 @@ def process_file(file_object): text_dict = preprocess.highlight_rates(text_dict) ################## RUN TOP DOWN ################## - td_results = prompt_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page + td_results = top_down_funcs.run_top_down(filename, text_dict) # Returns list of dictionaries for each page pd.DataFrame(td_results).to_csv(os.path.join(output_dir, config.TD_RESULTS_NAME), index=False) print(f"Top Down Complete - {filename}") ################## RUN BOTTOM UP ################## - bu_results = prompt_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries + bu_results = bottom_up_funcs.run_bottom_up(filename, text_dict) # Returns list of dictionaries pd.DataFrame(bu_results).to_csv(os.path.join(output_dir, config.BU_RESULTS_NAME), index=False) print(f"Bottom Up Complete - {filename}") diff --git a/src/main.py b/src/main.py index 87c08b5..5357465 100644 --- a/src/main.py +++ b/src/main.py @@ -14,6 +14,7 @@ Functional Overview: # Imports import concurrent.futures import traceback +import os import utils import config @@ -31,6 +32,7 @@ def main(): def process_item(item): key, value = item + try: file_processing.process_file(item) #prompt_funcs.bottom_up(item) @@ -52,10 +54,8 @@ def main(): print(f"Error: {e}") # Write Consolidated Output - if config.WRITE_OUTPUT: - utils.consolidate_csvs(config.OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) - + # if config.WRITE_OUTPUT: + # utils.consolidate_csvs(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH) if __name__ == "__main__": main() - diff --git a/src/postprocess.py b/src/postprocess.py index 93c04ae..bde4f62 100644 --- a/src/postprocess.py +++ b/src/postprocess.py @@ -1,7 +1,7 @@ import pandas as pd import re -import postprocessingfuncs +import postprocessing_funcs import config @@ -16,81 +16,55 @@ def postprocess_results(combined_df): # Sanitize the combined data try: - df = postprocessingfuncs.sanitize_combined(combined_df) + df = postprocessing_funcs.sanitize_combined(combined_df) except Exception as e: print(f"Postprocessing Error - santize_combined : {e}") # Clean specific columns for exact matches for field_list in [['CONTRACT_LOB', 'Corrected_LOB', config.VALID_LOBS], ['CONTRACT_PROGRAM', 'Corrected_PROGRAM', config.VALID_PROGRAMS], ['CONTRACT_NETWORK', 'Corrected_NETWORK', config.VALID_NETWORKS]]: try: - postprocessingfuncs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) + postprocessing_funcs.clean_columns_combined(df, field_list[0], field_list[2], field_list[1]) except Exception as e: print(f"Postprocessing Error - clean_columns_combined : {e}") try: - postprocessingfuncs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) + postprocessing_funcs.clean_columns_combined_fuzzy(df, field_list[0], field_list[2], config.FUZZY_MATCH_THRESHOLD) except Exception as e: print(f"Postprocessing Error - clean_columns_combined_fuzzy : {e}") # Correct misplaced values across columns try: - df = postprocessingfuncs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) + df = postprocessing_funcs.correct_misplaced_values(df, ['CONTRACT_LOB', 'CONTRACT_PROGRAM', 'CONTRACT_NETWORK'], valid_values_dict) except Exception as e: print(f"Postprocessing Error - correct_misplaced_values : {e}") # Move percentages and large numbers to correct columns try: - df = postprocessingfuncs.move_percentage_to_rate(df) + df = postprocessing_funcs.move_percentage_to_rate(df) except Exception as e: print(f"Postprocessing Error - move_percentage_to_rate : {e}") # Replace N/As in "Not Covered" with 0% try: - df = postprocessingfuncs.set_rate_to_zero_if_not_covered(df) + df = postprocessing_funcs.set_rate_to_zero_if_not_covered(df) except Exception as e: print(f"Postprocessing Error - set_rate_to_zero_if_not_covered : {e}") # Adjust for adding or subtracting try: - df = postprocessingfuncs.adjust_reimbursement_rate(df) + df = postprocessing_funcs.adjust_reimbursement_rate(df) except Exception as e: print(f"Postprocessing Error - adjust_reimbursement_rate : {e}") + # -- Deprecated -- # Clean dates so that Termination and Effective aren't the same - try: - df = postprocessingfuncs.clean_dates(df) - except Exception as e: - print(f"Postprocessing Error - clean_dates : {e}") + # try: + # df = postprocessing_funcs.clean_dates(df) + # except Exception as e: + # print(f"Postprocessing Error - clean_dates : {e}") # Re order columns column_order = [col for col in config.VALID_COLUMNS if col in df.columns] + [col for col in df.columns if col not in config.VALID_COLUMNS] df = df[column_order] return df - - - - - - - - - - - - - - - - - - - - - - - # Order columns - cols = [col for col in config.VALID_COLUMNS if col in combined_df.columns] + [col for col in combined_df.columns if col not in config.VALID_COLUMNS] - #combined_df = combined_df.loc[:, cols] - combined_df = combined_df[cols] - return combined_df diff --git a/src/postprocessingfuncs.py b/src/postprocessing_funcs.py similarity index 100% rename from src/postprocessingfuncs.py rename to src/postprocessing_funcs.py diff --git a/src/prompts.py b/src/prompts.py index abae021..b09377e 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -66,6 +66,8 @@ Here are the attributes to be included in each dictionary, and instructions on h 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. +'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'. +'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A' Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. @@ -283,9 +285,6 @@ def get_carveout_list(): 'Treatment Room', 'Wound Care', 'Cardiac Computed Tomograpy & Angiography', 'Positron Emission Tomography Scan', 'PET Scan', 'Hematology'] - - - def prompt_contract_lob(page): return f"""### PAGE START ### {page} ### PAGE END diff --git a/src/test.py b/src/test.py index 55d4330..41e3efd 100644 --- a/src/test.py +++ b/src/test.py @@ -1,92 +1,19 @@ -import pandas as pd -import numpy as np -import re -import json -import csv -from io import StringIO -import config -import ast + + import os +import pandas as pd -import utils -import postprocess -import postprocessingfuncs -import preprocess -import table_funcs -import prompts -import prompt_funcs -import claude_funcs -import merge_funcs -import dict_operations - - -input_dict = utils.read_input() - -(filename, contract_text) = list(input_dict.items())[0] - -contract_text = preprocess.clean_newlines(contract_text) -contract_text = preprocess.clean_billed_charges(contract_text) -text_dict = preprocess.split_text(contract_text) -text_dict = table_funcs.align_and_format_tables(text_dict) -text_dict = preprocess.highlight_rates(text_dict) - - -def BOTTOM_UP_LESSER(d, page): - return f""" -### PAGE START ### {page} ### PAGE END - -The above text contains a number of reimbursement terms. For the rest of this request, focus only on the specific term listed below: -Original_Dictionary: -- Original_Service: {d['SERVICE']} -- Original_Rate: {d['REIMBURSEMENT_RATE']} -- Original_Flat_Fee: {d['REIMBURSEMENT_FLAT_FEE']} -- Original_Methodology: {d['FULL_METHODOLOGY']} - -Your job is to identify if there is any 'lesser of' language or 'greater of' language that pertains to the specific service and methodology above. - -Note that a 'lesser of'/'greater of' statement may not be directly written for a given Service, but that does not necessarily mean that it doesn't apply. - -Read and analyze the page, then populate two JSON dictionaries with the following fields (descriptions provided): -'LESSER_OF_LANGUAGE_IND' : Write 'Y' if there is 'lesser of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. -'GREATER_OF_LANGUAGE_IND' : Write 'Y' if there is 'greater of' language somewhere on the page that applies to the Service and Original_Methodology. Write 'N' if not. -'FULL_METHODOLOGY' : If either of the above are 'Y', what is the reimbursement the lesser or greater of? Do not write the full sentence. -'REIMBURSEMENT_RATE' : If the Lesser/Greater Of Methodology is a percent of something, write just the numeric percent value here. Only include percentages. If not, write N/A. -'REIMBURSEMENT_FLAT_FEE' : If the Lesser/Greater Of Methodology is a dollar value, write just the numeric value here. Only include dollar values. If not, write N/A. - -If either LESSER_OF_LANGUAGE_IND or GREATER_OF_LANGUAGE_IND is 'Y', then both dictionaries must be populated. One of the dictionaries will have REIMBURSEMENT_RATE, REIMBURSEMENT_FLAT_FEE, and FULL_METHODOLOGY values equal to the Original term. -Fill the first dictionary with the values corresponding to the LEFT half of the Lesser/Greater Of Statement, and the second dictionary with the RIGHT half of the Lesser/Greater Of Statement. - -If both LESSER_OF_LANGUAGE_IND and GREATER_OF_LANGUAGE_IND are 'N', return a list with just the original dictionary. - -Only return the list of dictionaries, with no other commentary or explanation. Ensure you abide by proper JSON formatting. -""" - - -d = {'SERVICE': 'Inpatient Services', - 'REIMBURSEMENT_FLAT_FEE': 'N/A', - 'REIMBURSEMENT_RATE': '105%', - 'FULL_METHODOLOGY': "The lesser of 105% of the West Virginia Medicaid DRG (based on Hospital's current DHHR payment rate) or 100% of Hospital's allowable billed charges.", - 'page_num': '26', - 'Filename': 'App Regional Healthcare_Hospital Agreement_20160414_Dually Executed WV MCD-C13439125AA.txt'} - -answer = prompt_funcs.run_bottom_up_lesser(d, text_dict) -print(answer) - - -# answer_strings = prompt_funcs.run_bottom_up_primary({'3' : text_dict['3']}, 8000) -# answer_dicts = dict_operations.primary_string_to_dict(answer_strings, filename) -# answer_dicts_filtered = postprocessingfuncs.filter_service_column(answer_dicts) - -# print(answer_dicts_filtered) - -# for d in answer_dicts_filtered: -# print(f"Original dictionary: {d}") -# # Bottom Up Lesser -# if config.RUN_LESSER: -# lesser_object = prompt_funcs.run_bottom_up_lesser(d.copy(), text_dict.copy(), tokens=4000) -# print(f"Lesser dictionary: {lesser_object}") - - - - +df_list = [] +fail_count = 0 +for folder in os.listdir('output/'): + if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): + try: + df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) + df_list.append(df) + except: + print(folder) + fail_count += 1 +final_df = pd.concat(df_list, ignore_index=True) +print(final_df.shape) +print(fail_count) \ No newline at end of file diff --git a/src/top_down_funcs.py b/src/top_down_funcs.py new file mode 100644 index 0000000..9a0a7cb --- /dev/null +++ b/src/top_down_funcs.py @@ -0,0 +1,124 @@ + +import prompts +import claude_funcs +import utils + +import json + +def run_top_down(filename, text_dict): + """ + Executes the Top Down processing strategy on a provided dictionary of text pages, extracting structured data based on specified prompts. + + The function first filters and processes each page of text that is correctly numbered, using a primary analysis prompt. The results + from this analysis are then parsed into dictionaries, tagged with their respective page numbers and filename, and collected. + After the initial processing, a secondary Top Down analysis is conducted to refine and possibly expand the extracted data. + + Parameters: + filename (str): The filename associated with the text, used to tag the output data. + text_dict (dict): A dictionary where keys are page numbers and values are the text on those pages. + + Returns: + list of dict: A list of dictionaries where each dictionary contains structured results from both primary and secondary + Top Down analyses, associated with a specific page and the overall document. + """ + + all_results = [] + + # Primary + for page_num, page_text in text_dict.items(): + if not page_num.isdigit(): + continue + prompt = prompts.TOP_DOWN_PRIMARY(page_text) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + answer_dict = json.loads(answer) + answer_dict.update({'page_num' : page_num, 'Filename' : filename}) + #answer_dicts = dict_operations.primary_string_to_dict({page_num : answer}, filename) # Convert to dictionaries + all_results.append(answer_dict) + + # td_primary = [i for d in all_results for i in d] # Consolidate to one list + # print(td_primary) + + # Secondaries + td_final = top_down_secondary(all_results, text_dict) + + return td_final # List of list of dictionaries + + +def top_down_secondary(td_results, text_dict): + """ + Conducts secondary processing on results obtained from the primary Top Down analysis of document text, enhancing detail and accuracy. + + This function iteratively enhances each dictionary result from the initial analysis by adding or refining information related to: + - Market metal levels by running a specific function to determine the contract marketplace metal level. + - Effective dates by analyzing the context around specific keywords in the text and extracting these dates. + - Termination dates, similarly extracted based on the presence and context of specific keywords. + + Parameters: + td_results (list of dict): Initial dictionaries from the primary Top Down processing containing basic extracted data. + text_dict (dict): Dictionary keyed by page numbers, providing the full text for corresponding analysis and extraction tasks. + + Returns: + list of dict: Updated list of dictionaries, each enhanced with additional detailed attributes like metal level and date specifics. + """ + updated_dicts = [] + for d in td_results: + # Run Metal Level + d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']]) + + # Dates + d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) + + # if auto_renewal == N: else: 'N/A' + d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) + + updated_dicts.append(d) + return updated_dicts + + + +def run_top_down_metal_level(d, page): + """ + Identifies and extracts the metal level of a contract within a specific line of business (LOB) from the provided page text. + + This function determines whether the contract's LOB is associated with marketplace or commercial sectors by examining the + 'CONTRACT_LOB' key in the dictionary. If relevant, it constructs and sends a specific prompt to a language model to extract the + metal level (e.g., Bronze, Silver, Gold, Platinum). If the LOB is not relevant, it directly assigns 'N/A'. + + Parameters: + d (dict): A dictionary containing extracted information from primary Top Down analysis, specifically the 'CONTRACT_LOB' key. + page (str): The text content of the page that is being analyzed for metal level information. + + Returns: + str: The metal level of the contract as determined by the analysis, or 'N/A' if the contract LOB is not applicable. + """ + + if 'MARKETPLACE' in str(d['CONTRACT_LOB']).upper() or 'COMMERCIAL' in str(d['CONTRACT_LOB']).upper(): + prompt = prompts.TOP_DOWN_METAL_LEVEL(d['CONTRACT_LOB'], page) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + else: + answer = 'N/A' + return answer + + + +def run_top_down_date(type_, d, page): + """ DEPRECATED + Extracts specific date-related information from a page of text using a Top Down processing approach. + + This function tailors the extraction to focus on either 'EFFECTIVE' or 'TERMINATION' dates by constructing + a prompt that directs a language model to search for and interpret date information relevant to the provided type. + It formats the input data to fit the processing needs, sends it along with the page text to the model, and captures + the model's response. + + Parameters: + type_ (str): Specifies the type of date to extract, 'EFFECTIVE' or 'TERMINATION'. + d (dict): A dictionary containing preliminary data extracted from the text, which may include metadata like filename or page number. + page (str): The text content of the page from which to extract the date. + + Returns: + str: The extracted date as a string, based on the model's interpretation of the input prompt and text context. + """ + formatted_d = utils.format_td_check([d], ['Filename', 'page_num']) + prompt = prompts.TOP_DOWN_DATE('EFFECTIVE', formatted_d, page) + answer = claude_funcs.invoke_claude_3(prompt, max_tokens=4000) + return answer From d4585b1b1c5e0ac4e69717f78f0cd479a6cda313 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 10 Jul 2024 13:11:55 -0700 Subject: [PATCH 70/78] Only run valid files --- src/bottom_up_funcs.py | 3 ++- src/config.py | 8 ++++---- src/main.py | 16 ++++++++-------- src/test.py | 3 ++- src/utils.py | 9 +++++++++ 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/bottom_up_funcs.py b/src/bottom_up_funcs.py index 7bf0ff1..a1253ae 100644 --- a/src/bottom_up_funcs.py +++ b/src/bottom_up_funcs.py @@ -4,6 +4,7 @@ import postprocessing_funcs import prompts import config import claude_funcs +import utils import difflib @@ -56,7 +57,7 @@ def run_bottom_up_primary(text_dict, tokens): #if '%' in chunk_dict[page_number] or '$' in chunk_dict[page_number]: #prompt = prompts.BOTTOM_UP_PRIMARY(chunk_dict[page_number], config.CLIENT_NAME) for page_number in text_dict.keys(): - if page_number.isdigit() and ('%' in text_dict[page_number] or '$' in text_dict[page_number] or 'percent' in text_dict[page_number].lower()): + if utils.contains_reimbursement(text_dict, page_number): # Run Primary prompt = prompts.BOTTOM_UP_PRIMARY(text_dict[page_number], config.CLIENT_NAME) answer = claude_funcs.invoke_claude_3(prompt, max_tokens=tokens) diff --git a/src/config.py b/src/config.py index d63ad69..13d2b7e 100644 --- a/src/config.py +++ b/src/config.py @@ -54,12 +54,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXHJKTYQ6L" -AWS_SECRET_ACCESS_KEY="H3wyMbmE7UbK/7GFBH0+t9p3pdEJsb5QIixO5BKx" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEHQaCXVzLWVhc3QtMiJGMEQCIF9hoZcjho3AvCRXb8YyBYbogCNra8UI/gVKn+Batqz+AiB915kH5kk1WP2FWlZYBPSQ48Qi0pE+A7Cx5IZJGycscSqDAwg9EAAaDDY2MDEzMTA2ODc4MiIMqiCgRpeur6AXeuzTKuACnWFyPwZtV+xbLT94iBuMlSr/v/eecsdq0huqCayieInxYuo1KaLuLTlb2Uh6CweO9FDad58B4Axo/gtFIRZGqy1KArM3gKOEMVPnaMxWz3EHXLZVvCT31BzDNEJM2hC17AjC9cUnwn/+lx5xlL1UweOO5fphHWDBe0DWoF1CUl2ciXr2fxVLilLWjN91ZdJOV97Je2K+LyRpOi8S0076jPolBbENXgDfDzLoRbftDFJXn6aJXsFfOZJ1BRo3kbNBzAlPA+PGcyYqET2cFvA5RY+4yeKQ2jV5eFyeNIRcpmZaD+bW8AnKOe2OYAYg8ft+uSdQrM4pdmJpGT1jRfIfo5psOtvMbIcB4BvxOZFVOn+Zyni4KLFiAnss1ttMYR5skjfTfDzWGtVBl5KH9Crf6pDihWg35IbKHqVqk7HjVMorLAWbjOFJfaR/f8U6dL2ZNB/7ydUFQx0zdp396w4GszClg7i0BjqnAbJHP3p1uYlFN5/kTjzbA8vQHrRsbq6XurwojMfWvhS67zlipnRDqVFFBI7lEYu1au582p2yDZ7A10w1TAXbb/xaoOiU6qkAxk+w+LMlWB15Xmivuq4ahHsDKJ6d09hAEKT3wybzWlr4ps/cEipTiJtDAK9mfkj5Wa9J/Hwj+s223wY158r6yEvRIrDAiL0o5Gc+8+mvYTWD/UGC0qQyXp23mKaJcbJy" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXCSJ3K64Q" +AWS_SECRET_ACCESS_KEY="Q4LV6HI9YeCdvQa4cbGabNbSrXlyqNsfZLi4+Fui" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEID//////////wEaCXVzLWVhc3QtMiJHMEUCIDDb4laRdzpdEkk8t0GXyzWl2oiZUYeYeAPep+hFI4O3AiEA7JYFyxxlXE/LV8n/vOYgMfJy8Ug4DoufwXmym6/6E+QqgwMISRAAGgw2NjAxMzEwNjg3ODIiDA2AdcQDY0DHevIjmCrgAg6mYz0uyTur4+1WSREcsWZO8/xAkUMa9WQ5j71Dw1k1PGS+o/KONNKzVa1UsT46xCTB05gXx1DsruKfeHnnZTgh0Bz13rl5lFw49jFRb7CdSSIL10q6MVW1URVuJ2CEEOjhRAw9s2EqVO1abki7HvcThEY5wOaQf2j/Qn3hdXGoLuShVHmdKJ3sf9gsY+4nzQB3JN1ZC17pJEmaMrFbpWjizaSaGprfw5G+FmrwfElvRD7kubS2P/NlInK7mD91HjyrKwiwgLaiCfB5UsfA7XPzMdjXH6gM6aWKzqGWDEQ1HiCFOwhZ4mv9kq+qOgtt1M/dJ8QdKlaohooCQoju1gSca7FVOf5NeIhNlZZLYoNx9RrPEZR8pvXrGtqs23AIJGw3hiU3jalb1JcpekOaxyOBBuMDtGeB9S+zfh3eb1Q4xHcCLVJ7Lcu5Fq2NlEynk7SPR2Oj28NorrQ6+92M5TAw1N26tAY6pgGcA4moH6SwW3HWcWO3tdCVLnc24TvW4U9cUlWjidO2UJNq/A1rgO22ccCrfVNXOzeGOxpTzBC1/nQlCnZfYxqFOWp8bFt/8r0yXbzyidLc4FGEJMRqfbudDPP8X4YTFxCL2gxlntD+HBEJDXoRgSdGYQiT5jqVP1FZXcwjJWIISzAJ6IHJbG+KGKT3qtZIVinnNp9EvrzF6UybGuRMzRBJf/f7+R7F" # File Paths -LOCAL_PATH = 'data/batch1/' # Replace with local +LOCAL_PATH = 'data/batch2/' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/main.py b/src/main.py index 5357465..fd9518c 100644 --- a/src/main.py +++ b/src/main.py @@ -32,14 +32,14 @@ def main(): def process_item(item): key, value = item - - try: - file_processing.process_file(item) - #prompt_funcs.bottom_up(item) - except Exception as e: - # Print the error message and traceback - print(f"Error processing item {key}: {e}") - traceback.print_exc() + if utils.contains_reimbursement(item[1],0): + try: + file_processing.process_file(item) + #prompt_funcs.bottom_up(item) + except Exception as e: + # Print the error message and traceback + print(f"Error processing item {key}: {e}") + traceback.print_exc() with concurrent.futures.ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor: # Process each item individually diff --git a/src/test.py b/src/test.py index 41e3efd..b447f95 100644 --- a/src/test.py +++ b/src/test.py @@ -16,4 +16,5 @@ for folder in os.listdir('output/'): final_df = pd.concat(df_list, ignore_index=True) print(final_df.shape) -print(fail_count) \ No newline at end of file + +final_df.to_csv("output_consolidated/CNC_TX_Batch1.csv") \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index d8b1e0f..67333e0 100644 --- a/src/utils.py +++ b/src/utils.py @@ -130,6 +130,15 @@ def consolidate_csvs(output_dir=config.CONSOLIDATED_OUTPUT_DIRECTORY, output_fil print(f"All CSV files have been consolidated into {output_file}") + +def contains_reimbursement(text, page): + if isinstance(text, dict): + return page.isdigit() and ('%' in text[page] or '$' in text[page] or 'percent' in text[page].lower()) + elif isinstance(text, str): + return ('%' in text or '$' in text or 'percent' in text.lower()) + else: + print("contains_reimbursement - Invalid data type") + From 0f6d1c07854aee59c9bdd091bffeeda568c79734 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 10 Jul 2024 13:20:11 -0700 Subject: [PATCH 71/78] Minor prompt change --- src/prompts.py | 2 +- src/test.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/prompts.py b/src/prompts.py index b09377e..5d62cd8 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -175,7 +175,7 @@ Read and analyze the page, then populate a JSON dictionary with the following co Return a value for each type of code - return N/A for any values not found. Multiple codes for each type may be found. In this case, return them all. They may also present as a range. If so, return the range. -Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them. +Note that some PROC_CODES may have 'Modifiers' listed as well. These are 2-character strings. If Modifiers are present, include them as part of this field. Ensure you ONLY return codes if they are associated directly with the Service listed. Codes referring to other services should not be included. diff --git a/src/test.py b/src/test.py index b447f95..67d0cad 100644 --- a/src/test.py +++ b/src/test.py @@ -15,6 +15,7 @@ for folder in os.listdir('output/'): fail_count += 1 final_df = pd.concat(df_list, ignore_index=True) +print(fail_count) print(final_df.shape) -final_df.to_csv("output_consolidated/CNC_TX_Batch1.csv") \ No newline at end of file +final_df.to_csv("output_consolidated/CNC_TX_Batch2.csv") \ No newline at end of file From 3d922c6ed7a6e2ca497a0d5042761e3c9b3740f7 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Wed, 10 Jul 2024 19:18:04 -0700 Subject: [PATCH 72/78] Added already processed filter --- src/config.py | 11 ++++++----- src/main.py | 5 +++-- src/test.py | 34 ++++++++++++++++++++-------------- src/utils.py | 8 +++++++- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/config.py b/src/config.py index 13d2b7e..f8472ba 100644 --- a/src/config.py +++ b/src/config.py @@ -18,6 +18,7 @@ VALID_NETWORKS = ['HMO', 'PPO', 'EPO', 'POS', 'FFS'] # Input Settings READ_MODE = '_LOCAL_' # OR '_S3_' +FILTER_ALREADY_PROCESSED = True # Output Settings WRITE_OUTPUT = True # True writes csvs, False prints result in console but no output written @@ -30,7 +31,7 @@ UNPROCESSED_RESULTS_NAME = 'combined_results_unprocessed.csv' PROCESSED_RESULTS_NAME = 'combined_results_post_processed.csv' # Multithread Settings -MAX_WORKERS = 3 +MAX_WORKERS = 10 # Prompt Debugging RUN_PRIMARY = True # Always True @@ -54,12 +55,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXCSJ3K64Q" -AWS_SECRET_ACCESS_KEY="Q4LV6HI9YeCdvQa4cbGabNbSrXlyqNsfZLi4+Fui" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEID//////////wEaCXVzLWVhc3QtMiJHMEUCIDDb4laRdzpdEkk8t0GXyzWl2oiZUYeYeAPep+hFI4O3AiEA7JYFyxxlXE/LV8n/vOYgMfJy8Ug4DoufwXmym6/6E+QqgwMISRAAGgw2NjAxMzEwNjg3ODIiDA2AdcQDY0DHevIjmCrgAg6mYz0uyTur4+1WSREcsWZO8/xAkUMa9WQ5j71Dw1k1PGS+o/KONNKzVa1UsT46xCTB05gXx1DsruKfeHnnZTgh0Bz13rl5lFw49jFRb7CdSSIL10q6MVW1URVuJ2CEEOjhRAw9s2EqVO1abki7HvcThEY5wOaQf2j/Qn3hdXGoLuShVHmdKJ3sf9gsY+4nzQB3JN1ZC17pJEmaMrFbpWjizaSaGprfw5G+FmrwfElvRD7kubS2P/NlInK7mD91HjyrKwiwgLaiCfB5UsfA7XPzMdjXH6gM6aWKzqGWDEQ1HiCFOwhZ4mv9kq+qOgtt1M/dJ8QdKlaohooCQoju1gSca7FVOf5NeIhNlZZLYoNx9RrPEZR8pvXrGtqs23AIJGw3hiU3jalb1JcpekOaxyOBBuMDtGeB9S+zfh3eb1Q4xHcCLVJ7Lcu5Fq2NlEynk7SPR2Oj28NorrQ6+92M5TAw1N26tAY6pgGcA4moH6SwW3HWcWO3tdCVLnc24TvW4U9cUlWjidO2UJNq/A1rgO22ccCrfVNXOzeGOxpTzBC1/nQlCnZfYxqFOWp8bFt/8r0yXbzyidLc4FGEJMRqfbudDPP8X4YTFxCL2gxlntD+HBEJDXoRgSdGYQiT5jqVP1FZXcwjJWIISzAJ6IHJbG+KGKT3qtZIVinnNp9EvrzF6UybGuRMzRBJf/f7+R7F" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXO54GJPVG" +AWS_SECRET_ACCESS_KEY="ob03unDK0Dzb+28dhD694VFxHAEHezIw40q9kuRq" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJGMEQCIEfMHwEkEKTZNMabXk7Fy+7ud/qKfcjm5fRdCK6tgj8CAiAmE8JGe+4kTXifj8C7/Ie8Dmgxxp6ULP0Q4xxXB08VnCqDAwhTEAAaDDY2MDEzMTA2ODc4MiIM/Si9n9b9iP4Mn1xMKuACuGlnRELGs+xOhxxtPTyUS5dUdBmzd8J3/m72GfzzeZhWAG4DkUBpjkyO3kYK+MwIx0W5y++kxADhvnBVSKrf/ZzB792X0ChNJQmjCWGFnvFsCLXXmSxTpqx0v85Scja5acwC/HxkVBvklOrDImDsIqJTcXbOwO5hCaohXKg5PckefMl0eDqO5oi52PvmGEFZeqmcuXtiz06XFFW5bIedpAH+zURnVeZFh+1OOWJiwObxY5wz89A6nj98O6kZ9ymXi/gZkfjxBf00dh7wBee8KoMJOZ+a5ZtktPpet7bW18Fj2eeF5tgGiC7K7o8eDadF5FWts7XoEGhjtiE9IhBY8sTXUSC3LjFeC32zUsT705qsF7qgCVTx7GiBeFrT7iK647dT39fZKcW+V7m40MstajoQmNvwTWlJ/uHgGtr4LEvd4I8OTsrxw9h4SgeZp3E59ubB2PBOJP1CjWXGlFU4ITCX7by0BjqnAdWd2x1obGnuIbb4O4f3W9qI0HnjYIWNgR6tpVpcb9IwJUvKL1Qy8LnxdKGq/VpWG06rKlMQ2yhea8thMxUVAAwR+AVoA/fcq58P6IKurwcvxemw24VfkBuRLDaIRHSH0KGrpZACHtpvF1IOvTSnKSkn1qL6E7MXd/+y+98njgEsccOE0YieYlEBE8uK/X9wynGLINg+i6DDnGiJaazpY4S5q3TSL+BT" # File Paths -LOCAL_PATH = 'data/batch2/' # Replace with local +LOCAL_PATH = 'data/centene_healthnet/hmrun2' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/main.py b/src/main.py index fd9518c..56f534f 100644 --- a/src/main.py +++ b/src/main.py @@ -27,8 +27,9 @@ def main(): else: # Read contract txt input_dict = utils.read_input() - # already_processed = [s.split('.txt_results')[0]+'.txt' for s in os.listdir('results/')] - # input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} + + if config.FILTER_ALREADY_PROCESSED: + input_dict = utils.filter_already_processed(input_dict) def process_item(item): key, value = item diff --git a/src/test.py b/src/test.py index 67d0cad..8d93a0b 100644 --- a/src/test.py +++ b/src/test.py @@ -1,21 +1,27 @@ +import utils +import config import os import pandas as pd -df_list = [] -fail_count = 0 -for folder in os.listdir('output/'): - if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): - try: - df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) - df_list.append(df) - except: - print(folder) - fail_count += 1 +# df_list = [] +# fail_count = 0 +# for folder in os.listdir('output/'): +# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): +# try: +# df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) +# df_list.append(df) +# except: +# print(folder) +# fail_count += 1 + +# final_df = pd.concat(df_list, ignore_index=True) +# print(fail_count) +# print(final_df.shape) + +# final_df.to_csv("output_consolidated/CNC_HNT_Batch1.csv") + + -final_df = pd.concat(df_list, ignore_index=True) -print(fail_count) -print(final_df.shape) -final_df.to_csv("output_consolidated/CNC_TX_Batch2.csv") \ No newline at end of file diff --git a/src/utils.py b/src/utils.py index 67333e0..83e2b2e 100644 --- a/src/utils.py +++ b/src/utils.py @@ -141,6 +141,12 @@ def contains_reimbursement(text, page): - +def filter_already_processed(input_dict): + already_processed = [] + for folder_name in os.listdir(config.OUTPUT_DIRECTORY): + if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, folder_name)): + already_processed.append(folder_name+'.txt') + input_dict = {key : input_dict[key] for key in input_dict.keys() if key not in already_processed} + return input_dict From e7872d6aea43892642282009394e1743f834ad41 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 11 Jul 2024 13:27:46 -0700 Subject: [PATCH 73/78] Config structure update --- src/config.py | 8 ++++---- src/test.py | 28 ++++++++++++++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/config.py b/src/config.py index f8472ba..3047208 100644 --- a/src/config.py +++ b/src/config.py @@ -55,12 +55,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXO54GJPVG" -AWS_SECRET_ACCESS_KEY="ob03unDK0Dzb+28dhD694VFxHAEHezIw40q9kuRq" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIr//////////wEaCXVzLWVhc3QtMiJGMEQCIEfMHwEkEKTZNMabXk7Fy+7ud/qKfcjm5fRdCK6tgj8CAiAmE8JGe+4kTXifj8C7/Ie8Dmgxxp6ULP0Q4xxXB08VnCqDAwhTEAAaDDY2MDEzMTA2ODc4MiIM/Si9n9b9iP4Mn1xMKuACuGlnRELGs+xOhxxtPTyUS5dUdBmzd8J3/m72GfzzeZhWAG4DkUBpjkyO3kYK+MwIx0W5y++kxADhvnBVSKrf/ZzB792X0ChNJQmjCWGFnvFsCLXXmSxTpqx0v85Scja5acwC/HxkVBvklOrDImDsIqJTcXbOwO5hCaohXKg5PckefMl0eDqO5oi52PvmGEFZeqmcuXtiz06XFFW5bIedpAH+zURnVeZFh+1OOWJiwObxY5wz89A6nj98O6kZ9ymXi/gZkfjxBf00dh7wBee8KoMJOZ+a5ZtktPpet7bW18Fj2eeF5tgGiC7K7o8eDadF5FWts7XoEGhjtiE9IhBY8sTXUSC3LjFeC32zUsT705qsF7qgCVTx7GiBeFrT7iK647dT39fZKcW+V7m40MstajoQmNvwTWlJ/uHgGtr4LEvd4I8OTsrxw9h4SgeZp3E59ubB2PBOJP1CjWXGlFU4ITCX7by0BjqnAdWd2x1obGnuIbb4O4f3W9qI0HnjYIWNgR6tpVpcb9IwJUvKL1Qy8LnxdKGq/VpWG06rKlMQ2yhea8thMxUVAAwR+AVoA/fcq58P6IKurwcvxemw24VfkBuRLDaIRHSH0KGrpZACHtpvF1IOvTSnKSkn1qL6E7MXd/+y+98njgEsccOE0YieYlEBE8uK/X9wynGLINg+i6DDnGiJaazpY4S5q3TSL+BT" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMHLAR2NR" +AWS_SECRET_ACCESS_KEY="bpFA10211VdheK6S1UWIO/G7jewQO0f9zRz8aKTw" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEJb//////////wEaCXVzLWVhc3QtMiJHMEUCIF/1N4l0BDqRwknJJszcfngRgmZc7meJcpL8gBDXRihEAiEApXrv5kEHbQkITAfzAnwmDpnba54QNfFwfpaHen1GwEwqgwMIYBAAGgw2NjAxMzEwNjg3ODIiDEI3SnlkIFmdZjBIryrgAvyIsUOmQeni7CZy4jE3Hx6YbLXtCqUVt8qdBFeGtnj9/ucIrtKNui1hGeLGdEhe6z5lGbW0LqgkRjBJ9ZyTyIeSOGT1UzgTXe8M2Sly6RyLsOkHuFd4BPkUoq5Hkw+vYTX2gv5JpunFJtt+su0FAZPuoL4c7FYGk1ZtgAxZdPUEkvBpnAbIDVP/kh3evvzPVw4eOTUtuEUSGvQh9q3xi/DvDeQ11q9RXJKePwO3YmBYKxuh+xaiq9fCxwo624Z6hyC0xd0wv58rEPTDfXDaznIkwsnO8j5HUfS4kD4KExviVhBqb8PZowhs3jveYkijOVuqUGmvh3GSt+8y/7bAY6SWPvuCeYlMXE1gtRFNiYqm7okMlFHG8FXAXJeZd/uJgxDQ7kBxoIu1hodzriQVAUYz8OnIE1Kj+GhgeDo9PM5m7dC3LKOdL5HaQo2f2vjJuFS3Sjlg6dw2n8SDXbd37Z8whte/tAY6pgHy7lsrRL1ESRF3qGmqKznS+3T8MUSfNwOVl64oEv84aU+hFt27EwdSExEfurZvVHvOjTjtuEnPEAj2MD+RbUM/0/Ycq94Tuq3GZ12dLaxfnpB9/xnDMk5+Y3hDV0B4796au8hRaQBwqWu5o588iwgr4Nz1DddTwAbnQRVnGDA8+8kz5qODg5F7tnAuuDrt7EDvT51k9DK3eHA8ulR6bK2gEdZ+RlDp" # File Paths -LOCAL_PATH = 'data/centene_healthnet/hmrun2' # Replace with local +LOCAL_PATH = 'data/centene_texas/batch4' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/test.py b/src/test.py index 8d93a0b..ec5a8b5 100644 --- a/src/test.py +++ b/src/test.py @@ -5,22 +5,22 @@ import config import os import pandas as pd -# df_list = [] -# fail_count = 0 -# for folder in os.listdir('output/'): -# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): -# try: -# df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) -# df_list.append(df) -# except: -# print(folder) -# fail_count += 1 +df_list = [] +fail_count = 0 +for folder in os.listdir('output/'): + if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): + try: + df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) + df_list.append(df) + except: + print(folder) + fail_count += 1 -# final_df = pd.concat(df_list, ignore_index=True) -# print(fail_count) -# print(final_df.shape) +final_df = pd.concat(df_list, ignore_index=True) +print(fail_count) +print(final_df.shape) -# final_df.to_csv("output_consolidated/CNC_HNT_Batch1.csv") +final_df.to_csv("output_consolidated/CNC_Texas_Batch_3-4.csv") From b362b875231eabe0753d72847046afdf9758b490 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 12 Jul 2024 11:48:03 -0700 Subject: [PATCH 74/78] Reimbursement Dates change --- src/consolidate_output.py | 21 ++++++++++++++++++++- src/prompts.py | 2 +- src/test.py | 17 ++--------------- src/top_down_funcs.py | 6 +++--- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/consolidate_output.py b/src/consolidate_output.py index e72493c..9617eb5 100644 --- a/src/consolidate_output.py +++ b/src/consolidate_output.py @@ -15,4 +15,23 @@ for file in os.listdir(config.OUTPUT_DIRECTORY): final_df = pd.concat(all_dfs, ignore_index=True) -final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) \ No newline at end of file +final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) + + + +# df_list = [] +# fail_count = 0 +# for folder in os.listdir('output/'): +# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): +# try: +# df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) +# df_list.append(df) +# except: +# print(folder) +# fail_count += 1 + +# final_df = pd.concat(df_list, ignore_index=True) +# print(fail_count) +# print(final_df.shape) + +# final_df.to_csv("output_consolidated/CNC_Texas_Batch_3-4.csv") \ No newline at end of file diff --git a/src/prompts.py b/src/prompts.py index 5d62cd8..92d138e 100644 --- a/src/prompts.py +++ b/src/prompts.py @@ -66,7 +66,7 @@ Here are the attributes to be included in each dictionary, and instructions on h 'REIMBURSEMENT_FLAT_FEE' : If the listed reimbursement is dollar value, return only that dollar value. There can only be one answer for each object. If more than one rate is found, create additional objects for them. 'REIMBURSEMENT_RATE' : If the listed reimbursement is a percent of something, return only that percent. There can only be one answer for each object. If more than one rate is found, create additional objects for them. Do NOT include multiple percentages in this value. 'FULL_METHODOLOGY' : If the listed reimbursement is a percent of something, what is it the percent of? If the listed reimbursement is a direct dollar value, how is that dollar value paid (per diem, per unit, etc)? Write the full sentence in text describing the payment methodology. -'DATE_RANGE' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'. +'REIMBURSEMENT_DATES' : If the listed reimbursement only applies to a specific date range, write that range here. Write only the exact text. Write only the range. If there is none, write 'N/A'. 'REIMBURSEMENT_TIN' : If the listed reimbursement only applies to a specific TIN, write only the TIN here. If not, write 'N/A' Note that only one of REIMBURSEMENT_FLAT_FEE or REIMBURSEMENT_RATE should be populated in each dictionary. Ensure all 4 keys are in the dictionary. diff --git a/src/test.py b/src/test.py index ec5a8b5..38b26ba 100644 --- a/src/test.py +++ b/src/test.py @@ -5,22 +5,9 @@ import config import os import pandas as pd -df_list = [] -fail_count = 0 -for folder in os.listdir('output/'): - if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): - try: - df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) - df_list.append(df) - except: - print(folder) - fail_count += 1 -final_df = pd.concat(df_list, ignore_index=True) -print(fail_count) -print(final_df.shape) - -final_df.to_csv("output_consolidated/CNC_Texas_Batch_3-4.csv") +df = pd.read_csv('output_consolidated/CNC_HNT_Batch_1-5.csv') +print(df.DATE_RANGE.unique()) diff --git a/src/top_down_funcs.py b/src/top_down_funcs.py index 9a0a7cb..e35c218 100644 --- a/src/top_down_funcs.py +++ b/src/top_down_funcs.py @@ -66,10 +66,10 @@ def top_down_secondary(td_results, text_dict): d['CONTRACT_MARKETPLACE_METAL_LEVEL'] = run_top_down_metal_level(d, text_dict[d['page_num']]) # Dates - d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) + # d['LOB_PRICING_TERMS_EFFECTIVE_DATE'] = run_top_down_date('EFFECTIVE' , d, text_dict[d['page_num']]) - # if auto_renewal == N: else: 'N/A' - d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) + # # if auto_renewal == N: else: 'N/A' + # d['LOB_PRICING_TERMS_TERMINATION_DATE'] = run_top_down_date('TERMINATION', d, text_dict[d['page_num']]) updated_dicts.append(d) return updated_dicts From 2573a16e9313ee1a3ce0019f7adb868226a994b7 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 15 Jul 2024 14:23:28 -0700 Subject: [PATCH 75/78] Testing --- src/test.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/test.py b/src/test.py index 38b26ba..7d0fd93 100644 --- a/src/test.py +++ b/src/test.py @@ -6,9 +6,17 @@ import os import pandas as pd -df = pd.read_csv('output_consolidated/CNC_HNT_Batch_1-5.csv') -print(df.DATE_RANGE.unique()) - - +# df = pd.read_csv('output_consolidated/CNC_HNT_Batch_1-5.csv') +# print(df.DATE_RANGE.unique()) + +directory = 'T:\\AArete Client Work\\Centene\\Restricted\\Texas\\Contract Documents for Doczy\\all_pdf' +pdf_files = [] +for root, dirs, files in os.walk(directory): + for file in files: + if file.lower().endswith('.pdf'): + pdf_files.append(file) + +pdf_df = pd.DataFrame(pdf_files) +pdf_df.to_csv('All_TX_Files.csv') From 9dda51480c7ad84e6fea19958c9a033794165a94 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 1 Aug 2024 09:25:43 -0500 Subject: [PATCH 76/78] Updated consolidation script --- src/config.py | 8 +++--- src/consolidate_output.py | 55 ++++++++++++++++++++------------------- src/test.py | 54 +++++++++++++++++++++++++++++--------- 3 files changed, 74 insertions(+), 43 deletions(-) diff --git a/src/config.py b/src/config.py index 3047208..0a1ca49 100644 --- a/src/config.py +++ b/src/config.py @@ -55,12 +55,12 @@ VALID_COLUMNS = ['Filename', 'page_num', 'SERVICE', 'REIMBURSEMENT_FLAT_FEE', 'R 'Corrected_LOB', 'Corrected_PROGRAM', 'Corrected_NETWORK'] # AWS Keys -AWS_ACCESS_KEY_ID="ASIAZTMXAXNXMHLAR2NR" -AWS_SECRET_ACCESS_KEY="bpFA10211VdheK6S1UWIO/G7jewQO0f9zRz8aKTw" -AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEJb//////////wEaCXVzLWVhc3QtMiJHMEUCIF/1N4l0BDqRwknJJszcfngRgmZc7meJcpL8gBDXRihEAiEApXrv5kEHbQkITAfzAnwmDpnba54QNfFwfpaHen1GwEwqgwMIYBAAGgw2NjAxMzEwNjg3ODIiDEI3SnlkIFmdZjBIryrgAvyIsUOmQeni7CZy4jE3Hx6YbLXtCqUVt8qdBFeGtnj9/ucIrtKNui1hGeLGdEhe6z5lGbW0LqgkRjBJ9ZyTyIeSOGT1UzgTXe8M2Sly6RyLsOkHuFd4BPkUoq5Hkw+vYTX2gv5JpunFJtt+su0FAZPuoL4c7FYGk1ZtgAxZdPUEkvBpnAbIDVP/kh3evvzPVw4eOTUtuEUSGvQh9q3xi/DvDeQ11q9RXJKePwO3YmBYKxuh+xaiq9fCxwo624Z6hyC0xd0wv58rEPTDfXDaznIkwsnO8j5HUfS4kD4KExviVhBqb8PZowhs3jveYkijOVuqUGmvh3GSt+8y/7bAY6SWPvuCeYlMXE1gtRFNiYqm7okMlFHG8FXAXJeZd/uJgxDQ7kBxoIu1hodzriQVAUYz8OnIE1Kj+GhgeDo9PM5m7dC3LKOdL5HaQo2f2vjJuFS3Sjlg6dw2n8SDXbd37Z8whte/tAY6pgHy7lsrRL1ESRF3qGmqKznS+3T8MUSfNwOVl64oEv84aU+hFt27EwdSExEfurZvVHvOjTjtuEnPEAj2MD+RbUM/0/Ycq94Tuq3GZ12dLaxfnpB9/xnDMk5+Y3hDV0B4796au8hRaQBwqWu5o588iwgr4Nz1DddTwAbnQRVnGDA8+8kz5qODg5F7tnAuuDrt7EDvT51k9DK3eHA8ulR6bK2gEdZ+RlDp" +AWS_ACCESS_KEY_ID="ASIAZTMXAXNXHQSWW2R5" +AWS_SECRET_ACCESS_KEY="W9zH4nDGaae8SuJqcPDXyTHFpQ4xV43eRjbDfPGo" +AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEIX//////////wEaCXVzLWVhc3QtMiJIMEYCIQCjQVKSzQSWuO8bDdeGjDUJNUc1NwOA795yLFiBGDSslwIhAOccVQAzlBZyOVtU5mZztHVRbqEbrFigeg2aPzlczG1QKoMDCG8QABoMNjYwMTMxMDY4NzgyIgyP2pSGtJAhII6aZIMq4AI3XnnJgTtwbj6u15RcBlTCYgyn+SqIASHvEaJzyWHP/b7Vf5ZNyWKBsccgIZ590De/biM4Gb7tLsnWYCkFuyj3XswHKgyInBDqDg6b4olgyMGoDxrDmNUI28tQxB5XpwKxsas/azzwNPAgRAyXWMK4b1DcslPGpTRBBi+eBF+hTLiGk1jQnip0qR+GuJnCwXe66gcIGmgscjVxGSg2Q5n0O7VD2UOrr5T7BhvnGLiNtlrXqd3f+BFVk6/SzmuQJgsK33rtsmQUI7R5LXlyZEfH3cEkCXrdch6bFO4Wu53oBmiDtNEkle5n95TXCnXC1EchWzuDA6iP061kLRF3jdihMls8St/RoNACCR41YtOuU5oH1n5qXGvH1HQzOvBuSQBCKIqVGgB7hU63mbJlQCdX+0qd0jgteXFHMyjrwr604zPCxdcfbyehAwpxJR8xLVMimgxJqvyzvAoUjzW1sZPPMKG4rLUGOqUBOobUcGB1mI6GHaV6p6+t+qU/BG9FCt9BZbC/IJsHQkWWemCWpvV6lGVM5JPBmmgGQc3LiwH8g5JSHnoUDjTWlLvlHjXZ2ODLJP+aDFcuS4b1WZmXedL6RTsMvg8qhiELEB7rR+6jEmBmXvh56c+R4mkCG5AiWRBZ/ah910JnvKy5wQTOGJGzNxflyiJmwoO2JOZAKBLT0jg78iG1T3DMJ2qc+Qx7" # File Paths -LOCAL_PATH = 'data/centene_texas/batch4' # Replace with local +LOCAL_PATH = 'data/texas_childrens' # Replace with local # S3 Settings S3_CLIENT = boto3.client('s3', diff --git a/src/consolidate_output.py b/src/consolidate_output.py index 9617eb5..a7c6f76 100644 --- a/src/consolidate_output.py +++ b/src/consolidate_output.py @@ -4,34 +4,35 @@ import pandas as pd import config -all_dfs = [] -for file in os.listdir(config.OUTPUT_DIRECTORY): - if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, file)): - try: - df = pd.read_csv(f'{config.OUTPUT_DIRECTORY}/{file}/{config.PROCESSED_RESULTS_NAME}') - all_dfs.append(df) - except: - continue - -final_df = pd.concat(all_dfs, ignore_index=True) - -final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) - - - -# df_list = [] -# fail_count = 0 -# for folder in os.listdir('output/'): -# if 'combined_results_post_processed.csv' in os.listdir(os.path.join('output', folder)): +# all_dfs = [] +# for file in os.listdir(config.OUTPUT_DIRECTORY): +# if config.PROCESSED_RESULTS_NAME in os.listdir(os.path.join(config.OUTPUT_DIRECTORY, file)): # try: -# df = pd.read_csv(os.path.join('output', folder, 'combined_results_post_processed.csv')) -# df_list.append(df) +# df = pd.read_csv(f'{config.OUTPUT_DIRECTORY}/{file}/{config.PROCESSED_RESULTS_NAME}') +# all_dfs.append(df) # except: -# print(folder) -# fail_count += 1 - -# final_df = pd.concat(df_list, ignore_index=True) -# print(fail_count) +# continue + +# final_df = pd.concat(all_dfs, ignore_index=True) # print(final_df.shape) +# print(len(final_df.Filename.unique())) + +# final_df.to_csv(os.path.join(config.CONSOLIDATED_OUTPUT_DIRECTORY, config.OUTPUT_CSV_PATH)) + + +ac_path = 'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\TX_Childrens_1-1.csv' +b_path = 'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\TX_Childrens_1-N.csv' +output_path = f'T:\\AArete Client Work\\Texas Children’s Health Plan\\Restricted\\Doczy_Output\\{config.TODAY}_TX_Childrens_460_Doczy_AI_Output.csv' + +# Read the files into pandas DataFrames +ac = pd.read_csv(ac_path) +b = pd.read_csv(b_path) + +print(b.shape) + +# Merge the DataFrames on the 'Filename' column +merged_df = pd.merge(ac, b, on='Filename', how='right') +merged_df.to_csv(output_path) + +print(merged_df.shape) -# final_df.to_csv("output_consolidated/CNC_Texas_Batch_3-4.csv") \ No newline at end of file diff --git a/src/test.py b/src/test.py index 7d0fd93..e92ec15 100644 --- a/src/test.py +++ b/src/test.py @@ -4,19 +4,49 @@ import config import os import pandas as pd +import boto3 -# df = pd.read_csv('output_consolidated/CNC_HNT_Batch_1-5.csv') -# print(df.DATE_RANGE.unique()) - -directory = 'T:\\AArete Client Work\\Centene\\Restricted\\Texas\\Contract Documents for Doczy\\all_pdf' -pdf_files = [] -for root, dirs, files in os.walk(directory): - for file in files: - if file.lower().endswith('.pdf'): - pdf_files.append(file) - -pdf_df = pd.DataFrame(pdf_files) -pdf_df.to_csv('All_TX_Files.csv') + + +s3 = boto3.Session(aws_access_key_id='ASIA6GBMBVWOJOOHVXUE', +aws_secret_access_key='YsXnoYhY599uOifnQ+hZnqQJpWTyjhZ8XppJr43V', +aws_session_token='IQoJb3JpZ2luX2VjEHcaCXVzLWVhc3QtMiJHMEUCIBdFMYRAL20l87z9xZJzlvOW8H7+jOIoI2f6zOkNtMdQAiEAomTpoulfRpLcPhzbNFQVR5atGD8xbM49/gKX0rNDZYgqgQMIYRAAGgw5NzUwNDk5NjA4NjAiDGoxXZfA4RDhWkpflCreAmgKA3i7fZYV6z/e5WejrcnSfffciNhUKpen98pi1gJT6xgB9Cy4k19DAkF8ab8dQcyTO80K20hd2QjzwaTXDDiIpXYJq1TRHiJVN+g0QKOx10l4KOo7g8hGPzD/QPmA80aTxC96PqRklnJkUGNKe9zfZ7nCujts/i/rs0oAeuRCCnEKsh3lOtj4YEhoQGsNlKUsx4Pfh2cTn/JTZ4hma0zO8HnVfPh2f4i4hpa8Ula/0arXZJrkJyHdbQV85w+lmjypILQwAy6kQUd57lURDXDF1uPtziKZ2WqkozpEdeblmUUAT24rXKBhv3m86oqN59pb1IiFoDE7IOmJgRqNbh58OhOwifijMRWYouvSFM8pcBDTD725UmTzNu5GKgGJE00Q4PyBu5K2YXX1rHFmOhr29wq11mPMBPVRTF/zMLjG8ZloHYQWtt8VZwso8WE7ezV7RyScz0SBwBvhOX0mMPSwqbUGOqYBhMdcVuF9yw0Norg5U7G8SDl2WmiYCh/Anfea87h/1KzPs6ZNphtuSLcaH9+C7hVx5DVAJUW7gI0xx5jhPgqHbcptDPkNWCL129URUMPFOmuRiPxyTl0Xl5jSeh0Mj+RJz81OCWaEQyTQYOTtNUir77f1KAm8y+ClFrkYFrk6uz6HyiYknmcxAokjdNMIOS/83O7GTUv1Vo6/fIXo2MKX7gxhuNUoNw==' +).client('s3') + +def get_s3_files(bucket_name, folder): + paginator = s3.get_paginator('list_objects_v2') + result = paginator.paginate(Bucket=bucket_name, Prefix=folder) + files = set() + + for page in result: + if 'Contents' in page: + for obj in page['Contents']: + file_name = obj['Key'] + if file_name: # Ensure it's not a folder + files.add(file_name) + + return files + +bucket_name = 'texas-children-files' +folder = '510_text_files/' +files_in_folder = get_s3_files(bucket_name, folder) + +source_bucket='texas-children-files' +for source_key in files_in_folder: + filename = source_key.split('510_text_files/')[1] + print(filename) + + if filename not in os.listdir('data/texas_childrens'): + try: + response = s3.get_object(Bucket=source_bucket, Key=source_key)['Body'] + file_content = response.read().decode('utf-8') + + with open(f'data/texas_childrens/{filename}', 'w') as file: + file.write(file_content) + except: + pass + + From 5df14c56e3f5083b2bd297b237b95480f9bab04c Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Fri, 27 Sep 2024 13:32:27 +0100 Subject: [PATCH 77/78] carvouts --- {src => carveouts/src}/bottom_up_funcs.py | 0 {src => carveouts/src}/carveouts.py | 0 {src => carveouts/src}/claude_funcs.py | 0 {src => carveouts/src}/config.py | 0 {src => carveouts/src}/consolidate_output.py | 0 {src => carveouts/src}/dict_operations.py | 0 {src => carveouts/src}/file_processing.py | 0 {src => carveouts/src}/integration_testing.ipynb | 0 {src => carveouts/src}/main.py | 0 {src => carveouts/src}/merge_funcs.py | 0 {src => carveouts/src}/postprocess.py | 0 {src => carveouts/src}/postprocessing_funcs.py | 0 {src => carveouts/src}/preprocess.py | 0 {src => carveouts/src}/prompts.py | 0 {src => carveouts/src}/table_funcs.py | 0 {src => carveouts/src}/test.py | 0 {src => carveouts/src}/textract_template.py | 0 {src => carveouts/src}/top_down_funcs.py | 0 {src => carveouts/src}/utils.py | 0 19 files changed, 0 insertions(+), 0 deletions(-) rename {src => carveouts/src}/bottom_up_funcs.py (100%) rename {src => carveouts/src}/carveouts.py (100%) rename {src => carveouts/src}/claude_funcs.py (100%) rename {src => carveouts/src}/config.py (100%) rename {src => carveouts/src}/consolidate_output.py (100%) rename {src => carveouts/src}/dict_operations.py (100%) rename {src => carveouts/src}/file_processing.py (100%) rename {src => carveouts/src}/integration_testing.ipynb (100%) rename {src => carveouts/src}/main.py (100%) rename {src => carveouts/src}/merge_funcs.py (100%) rename {src => carveouts/src}/postprocess.py (100%) rename {src => carveouts/src}/postprocessing_funcs.py (100%) rename {src => carveouts/src}/preprocess.py (100%) rename {src => carveouts/src}/prompts.py (100%) rename {src => carveouts/src}/table_funcs.py (100%) rename {src => carveouts/src}/test.py (100%) rename {src => carveouts/src}/textract_template.py (100%) rename {src => carveouts/src}/top_down_funcs.py (100%) rename {src => carveouts/src}/utils.py (100%) diff --git a/src/bottom_up_funcs.py b/carveouts/src/bottom_up_funcs.py similarity index 100% rename from src/bottom_up_funcs.py rename to carveouts/src/bottom_up_funcs.py diff --git a/src/carveouts.py b/carveouts/src/carveouts.py similarity index 100% rename from src/carveouts.py rename to carveouts/src/carveouts.py diff --git a/src/claude_funcs.py b/carveouts/src/claude_funcs.py similarity index 100% rename from src/claude_funcs.py rename to carveouts/src/claude_funcs.py diff --git a/src/config.py b/carveouts/src/config.py similarity index 100% rename from src/config.py rename to carveouts/src/config.py diff --git a/src/consolidate_output.py b/carveouts/src/consolidate_output.py similarity index 100% rename from src/consolidate_output.py rename to carveouts/src/consolidate_output.py diff --git a/src/dict_operations.py b/carveouts/src/dict_operations.py similarity index 100% rename from src/dict_operations.py rename to carveouts/src/dict_operations.py diff --git a/src/file_processing.py b/carveouts/src/file_processing.py similarity index 100% rename from src/file_processing.py rename to carveouts/src/file_processing.py diff --git a/src/integration_testing.ipynb b/carveouts/src/integration_testing.ipynb similarity index 100% rename from src/integration_testing.ipynb rename to carveouts/src/integration_testing.ipynb diff --git a/src/main.py b/carveouts/src/main.py similarity index 100% rename from src/main.py rename to carveouts/src/main.py diff --git a/src/merge_funcs.py b/carveouts/src/merge_funcs.py similarity index 100% rename from src/merge_funcs.py rename to carveouts/src/merge_funcs.py diff --git a/src/postprocess.py b/carveouts/src/postprocess.py similarity index 100% rename from src/postprocess.py rename to carveouts/src/postprocess.py diff --git a/src/postprocessing_funcs.py b/carveouts/src/postprocessing_funcs.py similarity index 100% rename from src/postprocessing_funcs.py rename to carveouts/src/postprocessing_funcs.py diff --git a/src/preprocess.py b/carveouts/src/preprocess.py similarity index 100% rename from src/preprocess.py rename to carveouts/src/preprocess.py diff --git a/src/prompts.py b/carveouts/src/prompts.py similarity index 100% rename from src/prompts.py rename to carveouts/src/prompts.py diff --git a/src/table_funcs.py b/carveouts/src/table_funcs.py similarity index 100% rename from src/table_funcs.py rename to carveouts/src/table_funcs.py diff --git a/src/test.py b/carveouts/src/test.py similarity index 100% rename from src/test.py rename to carveouts/src/test.py diff --git a/src/textract_template.py b/carveouts/src/textract_template.py similarity index 100% rename from src/textract_template.py rename to carveouts/src/textract_template.py diff --git a/src/top_down_funcs.py b/carveouts/src/top_down_funcs.py similarity index 100% rename from src/top_down_funcs.py rename to carveouts/src/top_down_funcs.py diff --git a/src/utils.py b/carveouts/src/utils.py similarity index 100% rename from src/utils.py rename to carveouts/src/utils.py From 6903fe15789ffa618a353a5011e701a4cb6ede87 Mon Sep 17 00:00:00 2001 From: Michael McGuinness Date: Fri, 27 Sep 2024 13:33:52 +0100 Subject: [PATCH 78/78] dotgitignore --- carveouts/.gitignore | 71 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 carveouts/.gitignore diff --git a/carveouts/.gitignore b/carveouts/.gitignore new file mode 100644 index 0000000..d2bbe27 --- /dev/null +++ b/carveouts/.gitignore @@ -0,0 +1,71 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore +# Node artifact files +node_modules/ +dist/ +# Compiled Java class files +*.class +# Compiled Python bytecode +*.py[cod] +# Log files +*.log +# Package files +*.jar +# Maven +target/ +dist/ +# JetBrains IDE +.idea/ +# Unit test reports +TEST*.xml +# Generated by MacOS +.DS_Store +# Generated by Windows +Thumbs.db +# Applications +*.app +*.exe +*.war +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv +# Terraform files +.terraform/ +terraform.tfstate.backup +terraform.tfstate +.terraform.lock.hcl +build/ +textract-pipeline/terraform/builds/ +# Data Files +streamlit/history.csv +streamlit/RESULTS +streamlit/DB/ +streamlit/RAW_DOCUMENTS/ +streamlit/SOURCE_DOCUMENTS/ +streamlit/contract_field_values.csv +streamlit/contract_fields.csv +streamlit/sample.csv +streamlit/temp1.csv +streamlit/temp2.csv +streamlit/results.csv +# env +streamlit/venv +*.tfplan +*.pem +*.pem +textfiles/ +texts/ +*.csv +*.json +myenv/ +*.env +*.txt +subset/ +output/ +docs/ \ No newline at end of file