doczyai-pipelines/fieldExtraction/src/cnc_hotfix.py

import utils
import pandas as pd
import itertools
import numpy as np
import re
import pandas as pd
from valid import STATE_MAP
from prompts import BOTTOM_UP_METHODOLOGY_BREAKOUT
from bottom_up_funcs import get_short_methodology
import claude_funcs
from utils import is_empty
import dict_operations
import config
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)


import ac_smart_chunking
import claude_funcs
import cnc_hotfix_effective_date_utils
import config
import utils
import valid
import table_funcs
from hotfix_helper_funcs import irs_hotfix, clean_output_postprocess, state_check, check_field_for_matches, npi_hotfix, npi_post_process
import keywords
import prompts
import ac_funcs
import postprocessing_funcs

####### Default Term/Rate #######
def concatenate_lists(row):
    total = []
    for col in ['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']:
        value = row[col]
        if isinstance(value, list):
            total.extend([str(item) for item in value])
        else:
            total.append(str(value))
    return total

def check_default_term(df: pd.DataFrame):

    df['Contains Only'] = df['Default Term'].str.lower().str.contains('only')
    if df['Contains Only'].any():
        df_only_grouped = df[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type', 'Default Term']]
        df_transformed = df_only_grouped.groupby(['Attachment/Exhibit', 'Default Term'], as_index=False).transform(lambda x: list(set(x.dropna().to_list())))
        df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']] = df_transformed

        
        df_only_grouped['All Checks'] = df_only_grouped[['Line of Business List', 'Provider Type List', 'Provider Type - Level 2 List', 'IP/OP List', 'Service Type List', 'Plan Type List']].apply(concatenate_lists, axis=1)
        df_only_grouped['All Checks'] = df_only_grouped['All Checks'].apply(lambda x: [y.replace('OP','Outpatient').replace('IP', 'Inpatient') for y in x])
        df_only_grouped = df_only_grouped[['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type','All Checks']]
        
        df = df.merge(df_only_grouped, on = ['Attachment/Exhibit', 'Line of Business', 'Provider Type', 'Provider Type - Level 2', 'IP/OP', 'Service Type', 'Plan Type'], how = 'left')

        df['Default Term'].fillna('', inplace=True)
        df['All Checks'] = df['All Checks'].apply(lambda d: d if isinstance(d, list) else [])

        df['Default Match?'] = df.apply(lambda row: any([x.lower() in row['Default Term'].lower() for x in row['All Checks']]), axis = 1)

        df['Default Rate Corrected Step 1'] = df['Default Rate']
        df.loc[(df['Contains Only']==True) & (df['Default Match?']==False), 'Default Rate Corrected Step 1'] = ''

    
        return df

    else: 
        df['All Checks'] = ''
        df['Default Match?'] = True
        df['Default Rate Corrected Step 1'] = df['Default Rate']
        return df

def check_default_rate(df: pd.DataFrame):

    mapping = {
        'medicare' : 'MCR',
        'medicaid' : 'MCD',
        'billed charge' : 'BC',
        'allowable charge' : 'AC',
        'allowed charge' : 'AC',
        'average wholesale price' : 'AWP'
    }

    def replace_substring(string):
        for k, v in mapping.items():
            string = re.sub(r'\b' + k + r'\b', v, string, flags=re.IGNORECASE)
        return string

    def safe_extract(pattern, text):
        match = re.search(pattern, text)
        return match.group(0) if match else ''

    df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
    df_relevant = df.loc[(df['Default Rate Corrected Step 1'].isna()==False) & (df['Default Rate Corrected Step 1'] != '') & ((df['Default Term']=='') | (df['Default Term']!=df['Default Term'])), :].copy()

    if not df_relevant.empty:
        pct_pattern = r'\d+\% of '
        reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price)'
        df_relevant.loc[:, 'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Corrected Step 1'].apply(lambda x: 
                                                                                         safe_extract(pct_pattern, x.lower()) + safe_extract(reimb_pattern, x.lower())
                                                                                        )
        df_relevant.loc[:,'Default Rate Shortened'] = df_relevant.loc[:, 'Default Rate Shortened'].apply(replace_substring)
        df_grouped = df_relevant.groupby(['Contract Name', 'Attachment/Exhibit', 'Default Rate Shortened'], as_index = False)[r'If rate is % of Payor or MCR [STANDARD]'].agg(lambda x: list(set(x.dropna().to_list()))).reset_index()
        df_grouped = df_grouped.rename(columns = {r'If rate is % of Payor or MCR [STANDARD]' : 'Rate Standard List'})
        try:
            df_grouped['Remove Default Rate?'] = df_grouped.apply(lambda row: any([x.lower() in row['Default Rate Shortened'].lower() for x in row['Rate Standard List']]), axis=1)
            df = df.merge(df_grouped, how = 'left', on = ['Contract Name', 'Attachment/Exhibit'])
            df['Default Rate Corrected Step 2'] = df['Default Rate Corrected Step 1']
            df.loc[df['Remove Default Rate?'] == True, 'Default Rate Corrected Step 2'] = ''

        except ValueError:
            pass

        #df.drop(columns = ['Default Rate Shortened'], inplace=True)
    return df

def populate_default_term(df, text_dict):

    file_df = df.loc[(df['Default Rate Corrected Step 2'].isna()==False) & (df['Default Rate Corrected Step 2'] != '') & ((df['Default Term'].isna()==True) | (df['Default Term'] == '')), :]
    
    file_df = file_df.dropna(subset=['Page_Num', 'Attachment/Exhibit'])

    if not file_df.empty:
        exhibits = file_df['Attachment/Exhibit'].unique()
        for exhibit in exhibits:
            start_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Page_Num'].min()
            
            end_page = file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Pages'].min() + 1

            extracted_text = ''

            for page in range(int(round(start_page)), int(round(end_page))):
            
                exhibit_text = text_dict[f'{page:.0f}']

                def_rate = file_df.loc[file_df['Page_Num'] == start_page, 'Default Rate Corrected Step 2']

                if isinstance(def_rate, pd.Series):
                    def_rate = def_rate.min()
                elif not isinstance(def_rate, str):
                    def_rate = str(def_rate)
                
                sentences = exhibit_text.split('.')

                pct_pattern = r'\d+\%'
            
                reimb_pattern = r'(medicare|allowed charge|allowable charge|billed charge|medicaid|average wholesale price|fee schedule)'
                                
                if (def_rate == def_rate) and (def_rate != '') and (re.search(pct_pattern, def_rate)) and (re.search(reimb_pattern, def_rate.lower())):
                    def_pct = re.search(pct_pattern, def_rate).group(0)
                    def_reimb = re.search(reimb_pattern, def_rate.lower()).group(0)
                    for sentence in sentences:
                        sentence = sentence.replace('\n', ' ')                        
                        if (def_pct in sentence) and (def_reimb in sentence.lower()):
                            extracted_text = sentence
                            break
                        
                    if extracted_text != '':
                        break

            file_df.loc[file_df['Attachment/Exhibit'] == exhibit, 'Default Term Corrected Step 3'] = extracted_text.strip()                    
        file_df = file_df[['Attachment/Exhibit', 'Default Term Corrected Step 3']]
        file_df.drop_duplicates(inplace=True)
        df = df.merge(file_df, how='left', on=['Attachment/Exhibit'])
    else:
        file_df['Default Term Corrected Step 3'] = file_df['Default Term']
        file_df = file_df[['Page_Num', 'Attachment/Exhibit', 'Default Term Corrected Step 3']]
        df = df.merge(file_df, how = 'left', on = ['Page_Num', 'Attachment/Exhibit'])

    df['Default Term Corrected Step 3'] = df.apply(lambda x: x['Default Term'] if (x['Default Term Corrected Step 3'] == '') or (x['Default Term Corrected Step 3'] != x['Default Term Corrected Step 3']) else x['Default Term Corrected Step 3'], axis = 1)

    for col in ['Contains Only','All Checks','Default Match?','index', 'Default Rate Corrected Step 1','Default Rate Shortened','Rate Standard List','Remove Default Rate?']:
        if col in df.columns:
            df.drop(col, axis=1, inplace=True)
        else:
            pass

    df.rename(columns={'Default Rate Corrected Step 2' : 'Default Rate Corrected', 'Default Term Corrected Step 3' : 'Default Term Corrected'}, inplace=True)

    return df


#### HEALTH PLAN STATE ####
def clean_healthplan_state(
    df: pd.DataFrame,
    contract_name: str,
    text_dict: dict[str, str],
) -> pd.DataFrame:

    if not isinstance(df, pd.DataFrame):
        # raise TypeError(f"Expected a dataframe, got {type(df).__name__}")
        return df

    if contract_name is None:
        # raise ValueError(
        #     f"Expected 'contract_name' parameter, got {contract_name}"
        # )
        return df

    if "Health Plan State" not in df.columns:
        # raise KeyError(
        #     f"Column 'Health Plan State' not found in DataFrame. Available columns are: {list(df.columns)}"
        # )
        return df

    health_plan_state = df['Health Plan State'].unique().tolist()[0]
    
    answer = state_check(health_plan_state, text_dict)
    
    df["Health Plan State_corrected"] = answer
    
    return df

#### LINE OF BUSINESS ####
def clean_lob(df, text_dict):

    df['Line of Business'] = df['Line of Business'].str.upper()
    incorrect_map = {"MMP PLAN": "", "COMPLETE": "", "ALL OTHER HEALTH PLANS": "", "NAN" : ""}
    df['Line of Business'] = df['Line of Business'].replace(incorrect_map)
    df['LOB_Correct'] = df['Line of Business']
    
    match_pattern = r'|'.join(valid.VALID_LOBS)

    for index, row in df.iterrows():
        # Issue 1: Missing
        if utils.is_empty(row['Line of Business']) and pd.notna(row['Page_Num']):  # Check if 'Line of Business' is missing

            # Prepare to search text
            page_text = text_dict.get(str(int(row['Page_Num'])), "")
            next_page_text = text_dict.get(str(int(row['Page_Num']) + 1), "")  # Assuming page_num is a numerical index
            second_page_text = text_dict.get(str(int(row['Page_Num']) + 2), "")

            # First check in or near exhibit
            exhibit_index = page_text.find(str(row['Attachment/Exhibit']).strip())
            if exhibit_index != -1 and pd.notna(row['Attachment/Exhibit']):
                search_area = page_text[exhibit_index:exhibit_index + 200 + len(row['Attachment/Exhibit'])]
                matches = re.findall(match_pattern, search_area, re.IGNORECASE)
                if matches:
                    df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                    continue

            # Then check on entire page
            matches = re.findall(match_pattern, page_text, re.IGNORECASE)
            if matches:
                df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                continue

            # Then check on next page
            matches = re.findall(match_pattern, next_page_text, re.IGNORECASE)
            if matches:
                df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
                continue
            
            df.at[index, 'LOB_Correct'] = 'NO MATCH FOUND - REVIEW'

        # Issue 2: When Payer..., Medicare and/not Medicaid
        elif pd.notna(row["Service Type"]):
            service = str(row['Service Type']).lower()
            if 'where' in service and ('payer' in service or 'payor' in service):
                # Check Exhibit first
                matches = re.findall(match_pattern, str(row['Attachment/Exhibit']), re.IGNORECASE)
                if matches:
                    df.at[index, 'LOB_Correct'] = ', '.join(set([m.upper() for m in matches]))
            else:
                df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()
        
        # Non-Issue
        else:
            df.at[index, 'LOB_Correct'] = str(row['Line of Business']).upper()

    return df


#### Rate Standard ####
def reimb_methodology_fix(rm: str):

    d = {'FULL_METHODOLOGY' : rm}
    prompt = BOTTOM_UP_METHODOLOGY_BREAKOUT(d)
    answer = claude_funcs.invoke_claude(
        prompt, config.MODEL_ID_CLAUDE35_SONNET, '', max_tokens=512
    )
    answer_dict = dict_operations.secondary_string_to_dict(answer, '')
    d.update(answer_dict)

    # Ensure all bottom up methodology keys are present
    for key in [
        "LESSER",
        "RATE_STANDARD",
        "RATE_SHORT",
        "FLAT_FEE_STANDARD",
        "LESSER_RATE",
        "NOT_TO_EXCEED",
    ]:
        if key not in d.keys():
            d[key] = ""

    # SHORT_METHODOLOGY
    if "%" in d["RATE_STANDARD"]:
        d["SHORT_METHODOLOGY"] = get_short_methodology(d["RATE_STANDARD"])
    elif "$" in d["FLAT_FEE_STANDARD"]:
        d["SHORT_METHODOLOGY"] = "Flat Fee"
    else:
        d["SHORT_METHODOLOGY"] = "N/A"

    return d

def clean_rate_standard(df: pd.DataFrame) -> pd.DataFrame:

    df_relevant = df.loc[(df['Reimb. Methodology'].isna() == False) & (df[r'If rate is % of Payor or MCR [STANDARD]'].isna() == True) & (df['FLAT FEE'].isna() == True), :]

    if not df_relevant.empty:
        rm_list = df_relevant['Reimb. Methodology'].unique()

        dict_list = []

        for rm in rm_list:

            d = reimb_methodology_fix(rm)

            dict_list.append(d)

        results_df = pd.DataFrame.from_records(dict_list)

        results_df = results_df.add_suffix('_fixed')

        df = pd.merge(df, results_df, how = 'left', right_on = 'FULL_METHODOLOGY_fixed', left_on = 'Reimb. Methodology')

    return df


#### LESSER ####
def clean_lesser(df, text_dict):
    df['Lesser_Flag'] = ""
    # For Exhibit
    all_dfs = []
    for page_num, page_df in df.groupby('Page_Num', dropna=False):
        if pd.notna(page_num) and str(int(page_num)) in text_dict.keys():
            contract_page = str(int(page_num))
            try:
                contract_text = '\n'.join([text_dict[str(int(contract_page)+i)] for i in range(2)])
            except:
                contract_text = text_dict[str(int(contract_page))]

            output_lesser_count = (page_df['Lesser of Logic language, included (Y/N)'] == 'Y').sum()
            row_count = page_df.shape[0]
            page_lesser_count = len(re.findall(r"lesser|lessor", contract_text, re.IGNORECASE))
            # Likely Incorrect
            if output_lesser_count == 1 and row_count > 1 and page_lesser_count == 1:
                page_df.loc[:, 'Lesser_Flag'] = 'Likely Incorrect'
            # Likely Correct
            elif page_lesser_count == output_lesser_count == row_count:
                page_df.loc[:, 'Lesser_Flag'] = 'Likely Correct'
            # Other
            else:
                page_df.loc[:, 'Lesser_Flag'] = "No Flag"
        else:
            page_df.loc[:, 'Lesser_Flag'] = "No Flag"
        
        all_dfs.append(page_df)
    
    try:
        final_df = pd.concat(all_dfs, ignore_index=True)
    except:
        final_df = df.copy()

    # Second Column
    final_df['Service-Methodology'] = final_df['Service Type'].astype(str) + ' - ' + final_df['Reimb. Methodology'].astype(str)
    return final_df


#### PROVIDER TYPE LEVEL 2 ####
def clean_provider_type_2(df):
    valid_lists = {
        'PROF': [
            "Physician", "Primary Care Provider", "Dual Capacity Physician",
            "Mid-Level Practicioner", "Specialist", "OB/GYN", "Behavioral Health",
            "Therapist", "Surgery", "Independent RHC", "Ambulance Services",
            "Home Health", "SNF"
        ],
        'ANC': [
            "DME", "Durable Medical Equipment", "Radiology", "Lab", "Home Health",
            "Hospice", "Dialysis", "PT/OT/ST", "Urgent Care", "Home Infusion",
            "Ambulatory Surgery Center", "ASC", "Ambulance Services", "Home Health"
        ],
        'FAC': [
            "Skilled Nursing Facility", "Ambulatory Surgery Center", "ASC",
            "Hospital", "Clinic", "Rural Health Clinic", "FQHC", "Long Term Care",
            "Community Mental Health Center"
        ]
    }
    result_df = df.copy()
    result_df['PROV_TYPE_LEVEL_2_Corrected'] = result_df['Provider Type - Level 2']
    unique_exhibits = df['Attachment/Exhibit'].unique()
    for exhibit in unique_exhibits:
        if pd.isna(exhibit):
            continue
        match, _ = check_field_for_matches(exhibit, valid_lists)
        if match:
            exhibit_mask = (result_df['Attachment/Exhibit'] == exhibit) & (result_df['PROV_TYPE_LEVEL_2_Corrected'].isna())
            if exhibit_mask.any():
                result_df.loc[exhibit_mask, 'PROV_TYPE_LEVEL_2_Corrected'] = match
    blank_mask = result_df['PROV_TYPE_LEVEL_2_Corrected'].isna()
    for idx, row in result_df[blank_mask].iterrows():
        for field in df.columns:
            if field != 'PROV_TYPE_LEVEL_2_Corrected':
                match, _ = check_field_for_matches(row[field], valid_lists)
                if match:
                    result_df.at[idx, 'PROV_TYPE_LEVEL_2_Corrected'] = match
                    break
    return result_df
 
#### IRS ####
def clean_irs(abc_df, filename, text_dict ):
    """Gets the answers for fixed IRS, TIN Others and merges them with clean output
    Args:
        abc_df(df): Clean input which is grouped on 'Contract Name'
        filename: Name of the contract
        text_dict (dict): Dictionary keyed by string page-number and valued by actual textract output page
    Returns:
        merged_df(df): Left joined [abc_df, irs_hotfix_df] on 'Contract Name' with additional '_corrected' columns 
    """
    
    # gets the irs hotfixes in
    irs_answers = irs_hotfix(filename, text_dict)
    irs_df = pd.DataFrame([irs_answers])
    
    # adds corrected suffix for the hotfixes
    irs_df.columns = [col + '_corrected' for col in irs_df.columns]

    # join on 'Contract Name'
    irs_df["Contract Name"] = str(filename)
    df_merged = pd.merge(abc_df, irs_df, on='Contract Name', indicator=True, how='left')
    df_merged = clean_output_postprocess(df_merged)

    # overwrite the corrected columns with clean output if it already has an answer
    df_merged['PROV_GROUP_TIN_corrected'] = df_merged['dummy'].fillna(df_merged['PROV_GROUP_TIN_corrected'])

    if 'dummy_other' in df_merged.columns:
        df_merged['PROV_TIN_OTHER_corrected'] = df_merged['dummy_other'].fillna(df_merged['PROV_TIN_OTHER_corrected'])

    if 'dummy_other_signatory' in df_merged.columns:
        df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'] = df_merged['dummy_other_signatory'].fillna(df_merged['PROV_TIN_GROUP_SIGNATORY_corrected'])

    df_merged = df_merged.rename(columns={'PROV_GROUP_TIN_corrected': 'IRS_corrected'})
    
    df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]
    return df_merged


##### CONTRACT EFFECTIVE DATE #####
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str:
    # perform smart chunking    
    page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
                                                                        include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"],
                                                                        exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"],
                                                                        )
    
    context = "\n".join(text_dict[page] for page in page_list)
    prompt = prompts.get_effective_date_prompt(context)    # get effective date - this is for both meridian and non-meridian
    
    try:
        claude_answer_raw = claude_funcs.invoke_claude(
                    prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
        claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]

    except Exception as e:
        raise
    
    # meridian special case
    if is_meridian:
        if claude_answer_extracted == "N/A":
            # perform smart chunking with signature date pages included
            
            included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
            
            included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
            
            excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
            
            page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
                                                                            include_keywords = included_keywords,
                                                                            exclude_keywords = excluded_keywords
                                                                            )

            context = "\n".join(text_dict[page] for page in page_list)
            prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
        
            try:
                claude_answer_raw = claude_funcs.invoke_claude(
                            prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
                claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
            except Exception as e:
                raise
    
    return claude_answer_extracted

def clean_contract_effective_date(
    df: pd.DataFrame,
    file_name: str,
    text_dict: dict[str, str]
) -> pd.DataFrame:
    """Apply `contract_effective_date_fix` to a grouped input dataframe, `df`.  A new column
        is added.

    Args:
        df (pd.DataFrame): Grouped dataframe for one contract.  Can have multiple rows because
            of B-fields.  Contract effective date is an AC-field, so one per document.
        file_name (str): Filename of the contract
        text_dict (dict[str, str]): String-keyed dictionary valued by contract text

    Raises:
        TypeError: If a non-dataframe is passed in as `df`
        KeyError: If 'Contract Effective Date' is not found in the input `df`

    Returns:
        pd.DataFrame: A grouped dataframe with an extra column, 
            `Contract Effective Date_corrected`
    """
    
    if not isinstance(df, pd.DataFrame):
        raise TypeError(f"Expected a dataframe, got {type(df).__name__}")

    if "Contract Effective Date" not in df.columns:
        raise KeyError(
            f"Column 'Contract Effective Date' not found in DataFrame. Available columns are: {list(df.columns)}"
        )
    
    contract_effective_date = df['Contract Effective Date'].dropna().unique().tolist()[0] if not df['Contract Effective Date'].dropna().empty else None # previous run's contract effective date (if any)
    
    payer_name = df["PAYER NAME"].dropna().unique().tolist()
    
    if len(payer_name) == 0:
        payer_name = ""
    else:
        payer_name = payer_name[0]
  
    is_meridian = any(re.search(r'\b' + keyword + r'\b', payer_name, re.IGNORECASE) for keyword in ['meridian'])
    
    if utils.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing
        try:
            answer = contract_effective_date_fix(file_name, text_dict,is_meridian)
        except Exception as e:
            print(f"Error processing {file_name}, got error: {e}")
    else:
        answer = contract_effective_date
    
    if answer == "N/A":
        df["Contract Effective Date_corrected"] = answer
    else:
        try:
            corrected_date = utils.convert_to_us_date_format(answer)  # apply date formatting fix
        except utils.InvalidDateException as e:
            corrected_date = "N/A"
            
        df["Contract Effective Date_corrected"] = corrected_date

    return df


#### NPI ####
def clean_npi(abc_df, filename, text_dict, top_sheet_dict ):
    npi_answers = npi_hotfix(text_dict=text_dict)
    if npi_answers["NPI"] == "N/A" and npi_answers["NPI_other"] == "N/A":
            npi_answers = npi_hotfix(top_sheet_dict)
    
    npi_df = pd.DataFrame([npi_answers])
    npi_df.columns = [col + '_corrected' for col in npi_df.columns]
    npi_df["Contract Name"] = str(filename)
    
    df_merged = pd.merge(abc_df, npi_df, on='Contract Name', how='left')
    df_merged = npi_post_process(df_merged)

    df_merged['NPI_corrected'] = df_merged['dummy'].fillna(df_merged['NPI_corrected'])
    if 'dummy_other' in df_merged.columns:
        df_merged['NPI_other_corrected'] = df_merged['dummy_other'].fillna(df_merged['NPI_other_corrected'])
    df_merged = df_merged.loc[:, ~df_merged.columns.str.contains('dummy')]

    return df_merged


#### TERM GROUP ####
def clean_term_group(df: pd.DataFrame, 
                     contract_name: str, 
                     text_dict: dict[str, str], 
                     ac_chunks: dict[str, str]) -> pd.DataFrame:
    """this function takes previously generated file as dataframe and one contract name as input. If value for term is null
    it runs the term group prompts and sets term clause_corrected, auto-renewal_corrected and termination date_columns
    """
    df["Term Clause_corrected"] = df['Term Clause']
    df["Contract Auto-Renewal Indicator_corrected"] = df['Contract Auto-Renewal Indicator']
    df["Termination Date_corrected"] = df['Termination Date']

    # identify contracts where term is blank
    term_df = df[(df['Term Clause'].isna())|(df['Term Clause'].str.len() < 10)]
    term_df['Filename'] = term_df['Contract Name']
    contract_list = term_df['Filename'].to_list()
    contract_list = list(set(contract_list))

    # get term group fields and corresponding prompts
    fields = keywords.GROUPED_KEYWORD_MAPPINGS['term_group']["fields"]
    fields.sort()
    questions = {}
    for field in fields:
        questions[field] = prompts.AC_DICT[field]

    # process the contract
    if contract_name in contract_list:
        ac_answers_dict = {}
        context = ac_chunks['term_group']
        context = context[0 : min(100000, len(context)) - 1]
        prompt = prompts.AC_MULTI_FIELD_TEMPLATE(context, questions)
        field_group_answer_raw = claude_funcs.invoke_claude(
            prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192
        )
        field_group_answer_raw = field_group_answer_raw.replace("_DATE", "_DT")
        field_group_answer_dict = ac_funcs.json_parsing_search(field_group_answer_raw, fields)

        for field in fields:
            field_answer = field_group_answer_dict[field]
            ac_answers_dict[field] = field_answer

        ac_df = pd.DataFrame([ac_answers_dict])
        ac_df = postprocessing_funcs.clean_term_clause(ac_df)
        ac_df = postprocessing_funcs.clean_auto_renewal_ind(ac_df)

        if 'TERM_CLAUSE' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Term Clause_corrected'] = ac_df['TERM_CLAUSE'].iloc[0]
        if 'CONTRACT_AUTO_RENEWAL_IND' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Contract Auto-Renewal Indicator_corrected'] = ac_df['CONTRACT_AUTO_RENEWAL_IND'].iloc[0]
        if 'CONTRACT_TERMINATION_DT' in ac_df.columns:
            df.loc[df['Contract Name'] == contract_name, 'Termination Date_corrected'] = ac_df['CONTRACT_TERMINATION_DT'].iloc[0]

    return df

#### Agreement Name ####
# ensure that prompt for CONTRACT_TITLE is updated and df contains field Agreement Name (Contract Title)_corrected
def clean_agreement_name(df: pd.DataFrame,
                     contract_name: str,
                     text_dict: dict[str, str]) -> pd.DataFrame:
    """this function first runs agreement name prompt on first 5 pages.
    If agreement name is not found, it runs on the context selected based on keywords.
    """

    contract_title_chunk_dict =  dict(itertools.islice(text_dict.items(), 5))
    question = prompts.AC_DICT["CONTRACT_TITLE"]

    context = "\n".join(
            [contract_title_chunk_dict[str(page_num)] for page_num in contract_title_chunk_dict.keys()]
        )
    
    prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
    prompt_answer_raw = claude_funcs.invoke_claude(
                prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
    
    if prompt_answer_raw == "N/A":
        agreement_keywords = ["AGREEMENT", "AMENDMENT", "ADDENDUM", "PROVIDER", "CONTRACT", "MEMORANDUM", "LETTER", "REQUEST", "EFFECTIVE"]
        page_list = ac_smart_chunking.chunk_hierarchical(
            text_dict, agreement_keywords, False
        )
        context = "\n".join(
                [text_dict[str(page_num)] for page_num in page_list]
            )
        prompt = prompts.AC_SINGLE_FIELD_TEMPLATE(context, question)
        prompt_answer_raw = claude_funcs.invoke_claude(
                    prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
        
    df.loc[df['Contract Name'] == contract_name, 'Agreement Name (Contract Title)_corrected'] = prompt_answer_raw

    return df