doczyai-pipelines/fieldExtraction/scripts/cnc_hotfixes/cnc_hotfix.py

import itertools
import re
import warnings

import numpy as np
import pandas as pd

import claude_funcs
import config
import postprocessing_funcs
import string_funcs
from one_to_n_funcs import get_short_methodology
from prompts import BOTTOM_UP_METHODOLOGY_BREAKOUT
from valid import STATE_MAP

warnings.simplefilter(action='ignore', category=FutureWarning)


import claude_funcs
import scripts.cnc_hotfixes.cnc_hotfix_effective_date_utils as cnc_hotfix_effective_date_utils
import config
import keywords
import postprocessing_funcs
import prompts
import smart_chunking_funcs
import string_funcs
import valid
from hotfix_helper_funcs import (check_field_for_matches,
                                 clean_output_postprocess, irs_hotfix,
                                 npi_hotfix, npi_post_process, state_check)


####### Exhibit #######
def clean_exhibit(abc, text_dict):
    abc["Attachment/Exhibit_fixed"], abc["Attachment/Exhibit_page"] = "", ""

    def adhoc_exhibit_check(page):
        prompt = prompts.TOP_DOWN_EXHIBIT_CHECK(page[0:100])
        # Note: Claude 2 support has been removed - using Claude 3 Haiku instead
        answer = claude_funcs.invoke_claude(
            prompt, config.MODEL_ID_CLAUDE3_HAIKU, "", max_tokens=10
        )
        if "Y" in answer:
            return True
        else:
            return False

    def adhoc_exhibit_prompt(page):
        prompt = f"""
        ### PAGE_START ### {page} ### PAGE_END ###
        List any exhibit, attachment, or amendment names found on the page. Write the full name of the exhibit, including the exhibit number or letter, as well as any other subtitles describing the contents of the exhibit. Do NOT write the page number. 
        If no Exhibit, Attachment, or Amendment is found, write 'N/A'. Enclose only your final answer in |pipes|.
        """
        answer = claude_funcs.invoke_claude(
            prompt, config.MODEL_ID_CLAUDE35_SONNET, "", max_tokens=1000
        )
        return re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=answer)[-1]
    
    answer_dict = {}
    for index, row in abc.iterrows():
        if pd.notna(row['Page_Num']):
            page_num = str(int(row['Page_Num']))
            original_page_num = page_num

            # If we've already seen this page, get the answer previously found
            if original_page_num in answer_dict.keys():
                abc.loc[index, "Attachment/Exhibit_fixed"] = answer_dict[original_page_num]
                abc.loc[index, "Attachment/Exhibit_page"] = original_page_num
                continue

            exhibit_val = str(abc.loc[index, "Attachment/Exhibit"]).strip()
            normalized_exhibit_val = exhibit_val.replace(" ", "").replace("\n", "").upper()
        
            # If the exhibit is already on that page, then it's correct
            if normalized_exhibit_val in text_dict[page_num].replace(" ", "").replace("\n", "").upper():
                abc.loc[index, "Attachment/Exhibit_fixed"] = exhibit_val
                abc.loc[index, "Attachment/Exhibit_page"] = page_num
                answer_dict[original_page_num] = exhibit_val
            else:
                contains_exhibit = False
                while not contains_exhibit:
                    # Run TD Exhibit Check
                    if page_num in text_dict.keys():
                        contains_exhibit = adhoc_exhibit_check(text_dict[page_num])
                        if contains_exhibit:
                            page_exhibit = adhoc_exhibit_prompt(text_dict[page_num])
                            if 'N/A' not in page_exhibit:
                                abc.loc[index, "Attachment/Exhibit_fixed"] = page_exhibit
                                abc.loc[index, "Attachment/Exhibit_page"] = page_num
                                answer_dict[original_page_num] = page_exhibit
                            else:
                                contains_exhibit = False
                                page_num = str(int(page_num) - 1)
                        else:
                            page_num = str(int(page_num) - 1)
                    else:
                        contains_exhibit=True # Set to True to exit loop
    return abc


####### Default Term/Rate #######
def concatenate_lists(row):
    total = []
    for col in ['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']:
        value = row[col]
        if isinstance(value, list):
            total.extend([str(item) for item in value])
        else:
            total.append(str(value))
    return total

def check_default_term(df: pd.DataFrame):

    df['Contains Only'] = df['Default Term'].str.lower().str.contains('only')
    if df['Contains Only'].any():
        df_only_grouped = df[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type', 'Default Term']]
        df_transformed = df_only_grouped.groupby(['Attachment/Exhibit', 'Default Term'], as_index=False).transform(lambda x: list(set(x.dropna().to_list())))
        df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']] = df_transformed

        
        df_only_grouped['All Checks'] = df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']].apply(concatenate_lists, axis=1)
        df_only_grouped['All Checks'] = df_only_grouped['All Checks'].apply(lambda x: [y.replace('OP','Outpatient').replace('IP', 'Inpatient') for y in x])
        df_only_grouped = df_only_grouped[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type','All Checks']]
        
        df = df.merge(df_only_grouped, on = ['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type'], how = 'left')

        df['Default Term'].fillna('', inplace=True)
        df['All Checks'] = df['All Checks'].apply(lambda d: d if isinstance(d, list) else [])

        df['Default Match?'] = df.apply(lambda row: any([x.lower() in row['Default Term'].lower() for x in row['All Checks']]), axis = 1)

        df['Default Rate Corrected Step 1'] = df['Default Rate']
        df.loc[(df['Contains Only']==True) & (df['Default Match?']==False), 'Default Rate Corrected Step 1'] = ''

    
        return df

    else: 
        df['All Checks'] = ''
        df['Default Match?'] = True
        df['Default Rate Corrected Step 1'] = df['Default Rate']
        return df

def check_default_rate(df: pd.DataFrame):

    mapping = {
        'medicare' : 'MCR',
        'medicaid' : 'MCD',
        'billed charge' : 'BC',
        'allowable charge' : 'AC',
        'allowed charge' : 'AC',
        'average wholesale price' : 'AWP'
    }

    def replace_substring(string):
        for k, v in mapping.items():
            string = re.sub(r'\b' + k + r'\b', v, string, flags=re.IGNORECASE)
        return string

    def safe_extract(pattern, text):
        match = re.search(pattern, text)
        return match.group(0) if match else ''

    df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
    df_relevant = df.loc[(df['Default Rate Corrected Step 1'].isna()==False) & (df['Default Rate Corrected Step 1'] != '') & ((df['Default Term']=='') | (df['Default Term']!=df['Default Term'])), :].copy()

    if not df_relevant.empty:
        pct_pattern = r'\d+\% of '
        reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price)'
        df_relevant.loc[:, 'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Corrected Step 1'].apply(lambda x: 
                                                                                         safe_extract(pct_pattern, x.lower()) + safe_extract(reimb_pattern, x.lower())
                                                                                        )
        df_relevant.loc[:,'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Shortened'].apply(replace_substring)
        df_grouped = df_relevant.groupby(['Contract Name', 'Attachment/Exhibit', 'Default Rate Shortened'], as_index = False)[r'If rate is % of Payor or MCR [STANDARD]'].agg(lambda x: list(set(x.dropna().to_list()))).reset_index()
        df_grouped = df_grouped.rename(columns = {r'If rate is % of Payor or MCR [STANDARD]' : 'Rate Standard List'})
        try:
            df_grouped['Remove Default Rate?'] = df_grouped.apply(lambda row: any([x.lower() in row['Default Rate Shortened'].lower() for x in row['Rate Standard List']]), axis=1)
            df = df.merge(df_grouped, how = 'left', on = ['Contract Name', 'Attachment/Exhibit'])
            df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
            df.loc[df['Remove Default Rate?'] == True, 'Default Rate Corrected Step 2'] = ''

        except ValueError:
            pass

        #df.drop(columns = ['Default Rate Shortened'], inplace=True)
    return df

def populate_default_term(df, text_dict):

    file_df = df.loc[(df['Default Rate Corrected Step 2'].isna()==False) & (df['Default Rate Corrected Step 2'] != '') & ((df['Default Term'].isna()==True) | (df['Default Term'] == '')), :]
    
    file_df = file_df.dropna(subset=['Page_Num', 'Attachment/Exhibit'])

    if not file_df.empty:
        exhibits = file_df['Attachment/Exhibit'].unique()
        for exhibit in exhibits:
            start_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Page_Num'].min()
            
            end_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Pages'].min() + 1

            extracted_text = ''

            for page in range(int(round(start_page)), int(round(end_page))):
                str_page = f'{page:.0f}'
                if str_page in text_dict.keys():
                    exhibit_text = text_dict[str_page]

                    def_rate = file_df.loc[file_df['Page_Num'] == start_page, 'Default Rate Corrected Step 2']

                    if isinstance(def_rate, pd.Series):
                        def_rate = def_rate.min()
                    elif not isinstance(def_rate, str):
                        def_rate = str(def_rate)
                    
                    sentences = exhibit_text.split('.')

                    pct_pattern = r'\d+\%'
                
                    reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price|fee schedule)'
                                    
                    if (def_rate == def_rate) and (def_rate != '') and (re.search(pct_pattern, def_rate)) and (re.search(reimb_pattern, def_rate.lower())):
                        def_pct = re.search(pct_pattern, def_rate).group(0)
                        def_reimb = re.search(reimb_pattern, def_rate.lower()).group(0)
                        for sentence in sentences:
                            sentence = sentence.replace('\n', ' ')                        
                            if (def_pct in sentence) and (def_reimb in sentence.lower()):
                                extracted_text = sentence
                                break
                            
                        if extracted_text != '':
                            break

            file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Default Term Corrected Step 3'] = extracted_text.strip()                    
        file_df = file_df[['Attachment/Exhibit', 'Default Term Corrected Step 3']]
        file_df.drop_duplicates(inplace=True)
        df = df.merge(file_df, how='left', on=['Attachment/Exhibit'])
    else:
        file_df['Default Term Corrected Step 3'] = file_df['Default Term']
        file_df = file_df[['Page_Num', 'Attachment/Exhibit', 'Default Term Corrected Step 3']]
        df = df.merge(file_df, how = 'left', on = ['Page_Num', 'Attachment/Exhibit'])

    df['Default Term Corrected Step 3'] = df.apply(lambda x: x['Default Term'] if (x['Default Term Corrected Step 3'] == '') or (x['Default Term Corrected Step 3'] != x['Default Term Corrected Step 3']) else x['Default Term Corrected Step 3'], axis = 1)

    for col in ['Contains Only','All Checks','Default Match?','index', 'Default Rate Corrected Step 1','Default Rate Shortened','Rate Standard List','Remove Default Rate?']:
        if col in df.columns:
            df.drop(col, axis=1, inplace=True)
        else:
            pass

    df.rename(columns={'Default Rate Corrected Step 2' : 'Default Rate Corrected', 'Default Term Corrected Step 3' : 'Default Term Corrected'}, inplace=True)

    return df


#### HEALTH PLAN STATE ####
def clean_healthplan_state(
    df: pd.DataFrame,
    contract_name: str,
    text_dict: dict[str, str],
) -> pd.DataFrame:

    if not isinstance(df, pd.DataFrame):
        # raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
        return df

    if contract_name is None:
        # raise ValueError(
        #     f"Expected 'contract_name' parameter, got {contract_name}"
        # )
        return df

    if "Health Plan State" not in df.columns:
        # raise KeyError(
        #     f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}"
        # )
        return df

    health_plan_state = df['Health Plan State'].unique().tolist()[0]
    
    answer = state_check(health_plan_state, text_dict)
    
    df["Health Plan State_corrected"] = answer
    
    return df

#### LINE OF BUSINESS ####
def clean_lob(df, text_dict):

    df['Line of Business'] = df['Line of Business'].str.upper()
    incorrect_map = {"MMP PLAN": "", "COMPLETE": "", "ALL OTHER HEALTH PLANS": "", "NAN" : ""}
    df['Line of Business'] = df['Line of Business'].replace(incorrect_map)
    df['LOB_Correct'] = df['Line of Business']
    
    match_pattern = r'|'.join(valid.VALID_LOBS)

    for index, row in df.iterrows():
        if 'Attachment/Exhibit_fixed' in df.columns:
            exhibit_col = 'Attachment/Exhibit_fixed'
        else:
            exhibit_col = 'Attachment/Exhibit'
        
        exhibit_fixed = str(row[exhibit_col]).upper()

        # Issue 0: Incorrect - check exhibit fixed (if the LOB is not in the Exhibit, but another LOB is in the exhibit)
        if (str(row['Line of Business']).upper() not in exhibit_fixed) and any([lob.upper() in exhibit_fixed for lob in valid.VALID_LOBS]):
            df.at[index, 'LOB_Correct'] = ', '.join(set([m for m in valid.VALID_LOBS if m in exhibit_fixed]))
            continue

        # Issue 1: Missing
        if string_funcs.is_empty(row['Line of Business']) and pd.notna(row['Page_Num']):  # Check if 'Line of Business' is missing

            # Prepare to search text
            page_text = text_dict.get(str(int(row['Page_Num'])), "")
            next_page_text = text_dict.get(str(int(row['Page_Num']) + 1), "")  # Assuming page_num is a numerical index
            second_page_text = text_dict.get(str(int(row['Page_Num']) + 2), "")

            # First check in or near exhibit
            exhibit_index = page_text.find(str(row[exhibit_col]).strip())
            if exhibit_index != -1 and pd.notna(row[exhibit_col]):
                search_area = page_text[exhibit_index:exhibit_index + 200 + len(row[exhibit_col])]
                matches = re.findall(match_pattern, search_area, re.IGNORECASE)
                if matches:
                    df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                    continue

            # Then check on entire page
            matches = re.findall(match_pattern, page_text, re.IGNORECASE)
            if matches:
                df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                continue

            # Then check on next page
            matches = re.findall(match_pattern, next_page_text, re.IGNORECASE)
            if matches:
                df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                continue
            
            df.at[index, 'LOB_Correct'] = 'NO MATCH FOUND - REVIEW'

        # Issue 2: When Payer..., Medicare and/not Medicaid
        elif pd.notna(row["Service Type"]):
            service = str(row['Service Type']).lower()
            if 'where' in service and ('payer' in service or 'payor' in service):
                # Check Exhibit first
                matches = re.findall(match_pattern, str(row[exhibit_col]), re.IGNORECASE)
                if matches:
                    df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
            else:
                df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
        
        # Non-Issue
        else:
            df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()

    return df


#### Rate Standard ####
def reimb_methodology_fix(rm: str):

    d = {'FULL_METHODOLOGY' : rm}
    prompt = BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
    answer = claude_funcs.invoke_claude(
        prompt, config.MODEL_ID_CLAUDE35_SONNET, '', max_tokens=512
    )
    answer_dict = string_funcs.secondary_string_to_dict(answer, '')
    d.update(answer_dict)

    # Ensure all bottom up methodology keys are present
    for key in [
        "LESSER",
        "RATE_STANDARD",
        "RATE_SHORT",
        "FLAT_FEE_STANDARD",
        "LESSER_RATE",
        "NOT_TO_EXCEED",
    ]:
        if key not in d.keys():
            d[key] = ""

    # SHORT_METHODOLOGY
    if "%" in d["RATE_STANDARD"]:
        d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"])
    elif "$" in d["FLAT_FEE_STANDARD"]:
        d["SHORT_METHODOLOGY"] = "Flat Fee"
    else:
        d["SHORT_METHODOLOGY"] = "N/A"

    return d

def clean_rate_standard(df: pd.DataFrame) -> pd.DataFrame:

    df_relevant = df.loc[(df['Reimb. Methodology'].isna() == False) & (df[r'If rate is % of Payor or MCR [STANDARD]'].isna() == True) & (df['FLAT FEE'].isna() == True), :]

    if not df_relevant.empty:
        rm_list = df_relevant['Reimb. Methodology'].unique()

        dict_list = []

        for rm in rm_list:

            d = reimb_methodology_fix(rm)

            dict_list.append(d)

        results_df = pd.DataFrame.from_records(dict_list)

        results_df = results_df.add_suffix('_fixed')

        df = pd.merge(df, results_df, how = 'left', right_on = 'FULL_METHODOLOGY_fixed', left_on = 'Reimb. Methodology')

    return df


#### LESSER ####
def clean_lesser(df, text_dict):
    df['Lesser_Flag'] = ""
    # For Exhibit
    all_dfs = []
    for page_num, page_df in df.groupby('Page_Num', dropna=False):
        if pd.notna(page_num) and str(int(page_num)) in text_dict.keys():
            contract_page = str(int(page_num))
            try:
                contract_text = '\n'.join([text_dict[str(int(contract_page)+i)] for i in range(2)])
            except:
                contract_text = text_dict[str(int(contract_page))]

            output_lesser_count = (page_df['Lesser of Logic language, included (Y/N)'] == 'Y').sum()
            row_count = page_df.shape[0]
            page_lesser_count = len(re.findall(r"lesser|lessor", contract_text, re.IGNORECASE))
            # Likely Incorrect
            if output_lesser_count == 1 and row_count > 1 and page_lesser_count == 1:
                page_df.loc[:, 'Lesser_Flag'] = 'Likely Incorrect'
            # Likely Correct
            elif page_lesser_count == output_lesser_count == row_count:
                page_df.loc[:, 'Lesser_Flag'] = 'Likely Correct'
            # Other
            else:
                page_df.loc[:, 'Lesser_Flag'] = "No Flag"
        else:
            page_df.loc[:, 'Lesser_Flag'] = "No Flag"
        
        all_dfs.append(page_df)
    
    try:
        final_df = pd.concat(all_dfs, ignore_index=True)
    except:
        final_df = df.copy()

    # Second Column
    final_df['Service-Methodology'] = final_df['Service Type'].astype(str) + ' - ' + final_df['Reimb. Methodology'].astype(str)
    return final_df


#### PROVIDER TYPE LEVEL 2 ####
def clean_provider_type_2(df):
    valid_lists = {
        'PROF': [
            "Physician", "Primary Care Provider", "Dual Capacity Physician",
            "Mid-Level Practicioner", "Specialist", "OB/GYN", "Behavioral Health",
            "Therapist", "Surgery", "Independent RHC", "Ambulance Services",
            "Home Health", "SNF"
        ],
        'ANC': [
            "DME", "Durable Medical Equipment", "Radiology", "Lab", "Home Health",
            "Hospice", "Dialysis", "PT/OT/ST", "Urgent Care", "Home Infusion",
            "Ambulatory Surgery Center", "ASC", "Ambulance Services", "Home Health"
        ],
        'FAC': [
            "Skilled Nursing Facility", "Ambulatory Surgery Center", "ASC",
            "Hospital", "Clinic", "Rural Health Clinic", "FQHC", "Long Term Care",
            "Community Mental Health Center"
        ]
    }
    result_df = df.copy()
    result_df['PROV_TYPE_LEVEL_2_Corrected'] = result_df['Provider Type - Level 2']
    unique_exhibits = df['Attachment/Exhibit'].unique()
    for exhibit in unique_exhibits:
        if pd.isna(exhibit):
            continue
        match, _ = check_field_for_matches(exhibit, valid_lists)
        if match:
            exhibit_mask = (result_df['Attachment/Exhibit'] == exhibit) & (result_df['PROV_TYPE_LEVEL_2_Corrected'].isna())
            if exhibit_mask.any():
                result_df.loc[exhibit_mask, 'PROV_TYPE_LEVEL_2_Corrected'] = match
    blank_mask = result_df['PROV_TYPE_LEVEL_2_Corrected'].isna()
    for idx, row in result_df[blank_mask].iterrows():
        for field in df.columns:
            if field != 'PROV_TYPE_LEVEL_2_Corrected':
                match, _ = check_field_for_matches(row[field], valid_lists)
                if match:
                    result_df.at[idx, 'PROV_TYPE_LEVEL_2_Corrected'] = match
                    break
    return result_df
 
#### IRS ####
def clean_irs(abc_df, filename, text_dict ):
    """Gets the answers for fixed IRS, TIN Others and merges them with clean output
    Args:
        abc_df(df): Clean input which is grouped on 'Contract Name'
        filename: Name of the contract
        text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
    Returns:
        merged_df(df): Left joined [abc_df, irs_hotfix_df] on 'Contract Name' with additional '_corrected' columns 
    """
    
    # gets the irs hotfixes in
    irs_answers = irs_hotfix(filename, text_dict)
    irs_df = pd.DataFrame([irs_answers])
    
    # adds corrected suffix for the hotfixes
    irs_df.columns = [col + '_corrected' for col in irs_df.columns]

    # join on 'Contract Name'
    irs_df["Contract Name"] = str(filename)
    df_merged = pd.merge(abc_df, irs_df, on='Contract Name', indicator=True, how='left')
    df_merged = clean_output_postprocess(df_merged)

    # overwrite the corrected columns with clean output if it already has an answer
    df_merged['PROV_GROUP_TIN_corrected'] = df_merged['dummy'].fillna(df_merged['PROV_GROUP_TIN_corrected'])

    if 'dummy_other' in df_merged.columns:
        df_merged['PROV_TIN_OTHER_corrected'] = df_merged['dummy_other'].fillna(df_merged['PROV_TIN_OTHER_corrected'])

    if 'dummy_other_signatory' in df_merged.columns:
        df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'] = df_merged['dummy_other_signatory'].fillna(df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'])

    df_merged = df_merged.rename(columns={'PROV_GROUP_TIN_corrected': 'IRS_corrected'})
    
    df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]
    return df_merged


##### CONTRACT EFFECTIVE DATE #####
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False):
    # perform smart chunking    
    page_list,d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
                                                                        include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"],
                                                                        exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"],
                                                                        )
    if len(page_list) > 0:
        context = "\n".join(text_dict[page] for page in page_list)
        prompt = prompts.get_effective_date_prompt(context)    # get effective date - this is for both meridian and non-meridian
        
        try:
            claude_answer_raw = claude_funcs.invoke_claude(
                        prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
            claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]

        except Exception as e:
            raise
        
        # meridian special case
        if is_meridian:
            if claude_answer_extracted == "N/A":
                # perform smart chunking with signature date pages included
                
                included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
                
                included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
                
                excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
                
                page_list, d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
                                                                                include_keywords = included_keywords,
                                                                                exclude_keywords = excluded_keywords
                                                                                )

                context = "\n".join(text_dict[page] for page in page_list)
                prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
            
                try:
                    claude_answer_raw = claude_funcs.invoke_claude(
                                prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
                    claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
                except Exception as e:
                    raise
        
        return claude_answer_extracted, d, page_list, claude_answer_raw
    
    else:
        return "N/A", d, page_list, ""

def clean_contract_effective_date(
    df: pd.DataFrame,
    file_name: str,
    text_dict: dict[str, str]
) -> pd.DataFrame:
    """Apply `contract_effective_date_fix` to a grouped input dataframe, `df`.  A new column
        is added.

    Args:
        df (pd.DataFrame): Grouped dataframe for one contract.  Can have multiple rows because
            of B-fields.  Contract effective date is an AC-field, so one per document.
        file_name (str): Filename of the contract
        text_dict (dict[str, str]): String-keyed dictionary valued by contract text

    Raises:
        TypeError: If a non-dataframe is passed in as `df`
        KeyError: If 'Contract Effective Date' is not found in the input `df`

    Returns:
        pd.DataFrame: A grouped dataframe with an extra column, 
            `Contract Effective Date_corrected`
    """
    
    if not isinstance(df, pd.DataFrame):
        raise TypeError(f"Expected a dataframe, got {type(df).__name__}")

    if "Contract Effective Date" not in df.columns:
        raise KeyError(
            f"Column 'Contract Effective Date' not found in DataFrame. Available columns are: {list(df.columns)}"
        )
    
    contract_effective_date = df['Contract Effective Date'].dropna().unique().tolist()[0] if not df['Contract Effective Date'].dropna().empty else None # previous run's contract effective date (if any)
    
    payer_name = df["PAYER NAME"].dropna().unique().tolist()
    
    if len(payer_name) == 0:
        payer_name = ""
    else:
        payer_name = payer_name[0]
  
    is_meridian = any(re.search(r'\b' + keyword + r'\b', payer_name, re.IGNORECASE) for keyword in ['meridian'])
    
    if string_funcs.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing
        try:
            answer,d,page_list,claude_answer_raw = contract_effective_date_fix(file_name,text_dict,is_meridian)
        except Exception as e:
            # print(f"Error processing {file_name}, got error: {e}")
            answer,d,page_list,claude_answer_raw = "N/A",{},[],""
        try:
            corrected_date = postprocessing_funcs.convert_to_us_date_format(str(answer))  # apply date formatting fix
        except postprocessing_funcs.InvalidDateException as e:
            corrected_date = "N/A"
    else:
        corrected_date = contract_effective_date # don't correct format when we already have an answer
        d,page_list,claude_answer_raw = {},[],""
    

            
    df["Contract Effective Date_corrected"] = corrected_date
    df['Log Information'] = str(d)
    df['Pages selected'] = str(page_list)
    df["LLM Justification"] = claude_answer_raw
    
    return df


#### NPI ####
def clean_npi(abc_df, filename, text_dict, top_sheet_dict ):
    npi_answers = npi_hotfix(text_dict=text_dict)
    if npi_answers["NPI"] == "N/A" and npi_answers["NPI_other"] == "N/A":
            npi_answers = npi_hotfix(top_sheet_dict)
    
    npi_df = pd.DataFrame([npi_answers])
    npi_df.columns = [col + '_corrected' for col in npi_df.columns]
    npi_df["Contract Name"] = str(filename)
    
    df_merged = pd.merge(abc_df, npi_df, on='Contract Name', how='left')
    df_merged = npi_post_process(df_merged)

    df_merged['NPI_corrected'] = df_merged['dummy'].fillna(df_merged['NPI_corrected'])
    if 'dummy_other' in df_merged.columns:
        df_merged['NPI_other_corrected'] = df_merged['dummy_other'].fillna(df_merged['NPI_other_corrected'])
    df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy', na=False)]

    return df_merged


#### TERM GROUP ####
def clean_term_group(df: pd.DataFrame, 
                     contract_name: str, 
                     text_dict: dict[str, str], 
                     ac_chunks: dict[str, str]) -> pd.DataFrame:
    """this function takes previously generated file as dataframe and one contract name as input. If value for term is null
    it runs the term group prompts and sets term clause_corrected, auto-renewal_corrected and termination date_columns
    """
    df["Term Clause_corrected"] = df['Term Clause']
    df["Contract Auto-Renewal Indicator_corrected"] = df['Contract Auto-Renewal Indicator']
    df["Termination Date_corrected"] = df['Termination Date']

    # identify contracts where term is blank
    term_df = df[(df['Term Clause'].isna())|(df['Term Clause'].str.len() < 10)]
    term_df['Filename'] = term_df['Contract Name']
    contract_list = term_df['Filename'].to_list()
    contract_list = list(set(contract_list))

    # get term group fields and corresponding prompts
    fields = keywords.GROUPED_KEYWORD_MAPPINGS['term_group']["fields"]
    fields.sort()
    questions = {}
    for field in fields:
        questions[field] = prompts.AC_DICT[field]

    # process the contract
    if contract_name in contract_list:
        ac_answers_dict = {}
        context = ac_chunks['term_group']
        context = context[0 : min(100000, len(context)) - 1]
        prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
        field_group_answer_raw = claude_funcs.invoke_claude(
            prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192
        )
        field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
        field_group_answer_dict = string_funcs.json_parsing_search(field_group_answer_raw, fields)

        for field in fields:
            if field in field_group_answer_dict:
                field_answer = field_group_answer_dict[field]
                ac_answers_dict[field] = field_answer
            else:
                ac_answers_dict[field] = field_answer

        ac_df = pd.DataFrame([ac_answers_dict])
        ac_df = postprocessing_funcs.clean_term_clause(ac_df)
        ac_df = postprocessing_funcs.clean_auto_renewal_ind(ac_df)

        if 'TERM_CLAUSE' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Term Clause_corrected'] = ac_df['TERM_CLAUSE'].iloc[0]
        if 'CONTRACT_AUTO_RENEWAL_IND' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Contract Auto-Renewal Indicator_corrected'] = ac_df['CONTRACT_AUTO_RENEWAL_IND'].iloc[0]
        if 'CONTRACT_TERMINATION_DT' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Termination Date_corrected'] = ac_df['CONTRACT_TERMINATION_DT'].iloc[0]

    return df

#### Agreement Name ####
# ensure that prompt for CONTRACT_TITLE is updated and df contains field Agreement Name (Contract Title)_corrected
def clean_agreement_name(df: pd.DataFrame,
                     contract_name: str,
                     text_dict: dict[str, str]) -> pd.DataFrame:
    """this function first runs agreement name prompt on first 5 pages.
    If agreement name is not found, it runs on the context selected based on keywords.
    """

    df["Agreement_Name (Contract Title)_corrected"] = df['Agreement_Name (Contract Title)']
    agreement_df = df[(df['Agreement_Name (Contract Title)'].isna())|(df['Agreement_Name (Contract Title)'].astype(str).str.len() < 5)]
    agreement_df['Filename'] = agreement_df['Contract Name']
    contract_list = agreement_df['Filename'].to_list()
    contract_list = list(set(contract_list))
    
    # process the contract
    if contract_name in contract_list:
        contract_title_chunk_dict =  dict(itertools.islice(text_dict.items(), 5))
        question = prompts.AC_DICT["CONTRACT_TITLE"]

        context = "\n".join(
                [contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()]
            )
        
        prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
        prompt_answer_raw = claude_funcs.invoke_claude(
                    prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
        
        if prompt_answer_raw == "N/A":
            agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"]
            page_list = smart_chunking_funcs.chunk_hierarchical(
                text_dict, agreement_keywords, False
            )
            context = "\n".join(
                    [text_dict[str(page_num)] for page_num in page_list]
                )
            prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
            prompt_answer_raw = claude_funcs.invoke_claude(
                        prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
            
        df.loc[df['Contract Name'] == contract_name, 'Agreement_Name (Contract Title)_corrected'] = prompt_answer_raw

    return df


HOTFIX_ORDER = [
    "Contract Name",
"Agreement_Name (Contract Title)",
"PAYER NAME",
"Health Plan State",
"Affiliate (Y/N)",
"Credentialing Application Indicator",
"Term Clause",
"Contract Auto-Renewal Indicator",
"Termination Date",
"Termination Upon Notice - Days",
"Termination With Cause - Days",
"Non-Renewal Language",
"Non-Renewal - Days",
"Amend Contract Upon Notice Flag (Y/N)",
"Timeframe to Object - Days",
"Assignments Clause (Y/N)",
"Contract Effective Date",
"IRS #",
"IRS Name",
"NPI (10-digits)",
"NPI Name",
"PROV_GROUP_TIN_SIGNATORY",
"PROV_TIN_OTHER",
"PROV_NPI_OTHER",
"Notice to Provider Name",
"Notice to Provider Address",
"Sequestration Language",
"Sequestration Reductions (Y/N)",
"Parent Agreement Code",
"Pages",
"Page_Num",
"Attachment/Exhibit",
"Line of Business",
"Provider Type",
"Provider Type - Level 2",
"IP/OP",
"Service Type",
"Plan Type",
"Lesser of Logic language, included (Y/N)",
"Lesser of Rate",
"Reimb. Methodology",
"Reimb. Methodology_Short",
"If rate is % of Payor or MCR [STANDARD]",
'If rate is % of Payor or MCR [STANDARD]_Short',
"FLAT FEE",
"Default Term",
"Default Rate",
'Inclusion of Essential RBRVS "Fee Source" Language (Y/N)',
'CDM Neutralization Language, included (Y/N)',
"CONTRACT_CHARGEMASTER_PROTECTION_LANGUAGE",
"IP - DSH/IME/UC, included (Y/N)",
"IP - Stoploss Catastrophic Threshold",
"Exclusions",
"Not to Exceed",
"Escalator or COLA (Y/N)",
"Escalator I, Eff. Date",
"Delegated Function Indicator",
"Delegated Terms",
"ECM",
"National Agreement Indicator",
"Cost Settlement (Y/N)",
"Cost Settlement (Language)",
"Late Paid Claims (Y/N)",
"Late Paid Claims (Language)",
"Deemer Amendment",
"Regulatory Requirements",
"Recovery Rights",
"Arbitration and Disputes",
"Exclusivity Requirement (Y/N)",
"Exclusivity Requirement (Language)",
"Payor",
"Participation in Products",
"Clean Claim",
"Independent Review (Y/N)",
"Independent Review (Language)",
"Indemnification",
"Access to Medical Records",
"Member Confinement Days Language (Y/N)",
"Member Confinement Days Language (Language)",
"Network Access Fees (Y/N)",
"Network Access Fees (Language)",
"Payment in Advance of Claims Submission Language (Y/N)",
"Payment in Advance of Claims Submission Language",
"Eligibility Verification",
"Preauthorization",
"Policies and Procedures",
"Insurance Requirement",
"Carve-Out Vendors",
"Conflicts Between Certain Documents (Y/N)",
"Conflicts Between Certain Documents (Language)",
"Relationship of Parties (Y/N)",
"Relationship of Parties (Language)",
"Nonstandard Appeals Process (Y/N)",
"Nonstandard Appeals Process (Language)",
"Product Removal",
"Disparagement Prohibition (Y/N)",
"Disparagement Prohibition (Language)",
"Claims Editing Language (Y/N)",
"Claims Editing Language (Language)",
"Guarantee of Provider Yield (Y/N)",
"Guarantee of Provider Yield (Language)",
"HCBS Services",
"Add On Reimbursement (Y/N)",
"Add On Reimbursement (Language)",
"PMPM",
"Single Code Multiple Rates (Y/N)",
"Single Code Multiple Rates (Language)",
"Invoice Pricing (Y/N)",
"Invoice Pricing (Language)",
"Medical Necessity Language (Y/N)",
"Medical Necessity Language (Language)",
"Template",
"Provider-Based Billing Exclusion (Y/N)",
"Provider-Based Billing Exclusion (Language)",
"Agreement Name (Contract Title)_corrected",
"Health Plan State_corrected",
"Term Clause_corrected",
"Contract Auto-Renewal Indicator_corrected",
"Termination Date_corrected",
"Contract Effective Date_corrected",
"IRS_corrected",
"PROV_TIN_OTHER_corrected",
"PROV_TIN_GROUP_SIGNATORY_corrected",
"NPI_corrected",
"NPI_other_corrected",
"LOB_Correct",
"PROV_TYPE_LEVEL_2_Corrected",
"Lesser_Flag",
"Service-Methodology",
"Default Term Corrected",
"Default Rate Corrected",
"FULL_METHODOLOGY_fixed",
"LESSER_fixed",
"RATE_STANDARD_fixed",
"RATE_SHORT_fixed",
"FLAT_FEE_STANDARD_fixed",
"LESSER_RATE_fixed",
"NOT_TO_EXCEED_fixed",
"SHORT_METHODOLOGY_fixed",
"Attachment/Exhibit_fixed",
"Attachment/Exhibit_page",
"_merge",
"key1_fixed",
"key2_fixed",
"key3_fixed"]