doczyai-pipelines/archive/scripts/adhoc/cnc_prov2_effdate.py

import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import os
import time

import utils
import postprocessing_funcs
import claude_funcs
import config
from utils import is_empty
import re
import valid


def clean_prov_2(df):
    valid_types = valid.select_valid_prov_2(df['Provider Type'])
    target_rows = df[df['Provider Type - Level 2'].apply(is_empty)]

    def find_exact_match(text):
        if pd.isna(text) or text == '':
            return None

        words = re.findall(r'\b[\w/]+(?:[-\s][\w/]+)*\b', text)
        for i in range(len(words)):
            for j in range(i+1, len(words)+1):
                phrase = ' '.join(words[i:j])
                if phrase in valid_types:  # Case-sensitive matching
                    return phrase
        return None

    for index, row in target_rows.iterrows():
        match = None

        if not is_empty(row['Service Type']):
            match = find_exact_match(str(row['Service Type']))
            if match:
                df.at[index, 'Provider Type - Level 2'] = match
                continue

        if not is_empty(row['Attachment/Exhibit']):
            match = find_exact_match(str(row['Attachment/Exhibit']))
            if match:
                df.at[index, 'Provider Type - Level 2'] = match
                continue

        exhibit_rows = df[df['Attachment/Exhibit'] == row['Attachment/Exhibit']]
        if not exhibit_rows.empty:
            for _, exhibit_row in exhibit_rows.iterrows():
                if not is_empty(exhibit_row['Provider Type - Level 2']):
                    match = find_exact_match(str(exhibit_row['Provider Type - Level 2']))
                    if match:
                        df.at[index, 'Provider Type - Level 2'] = match
                        break
                elif not is_empty(exhibit_row['Service Type']):
                    match = find_exact_match(str(exhibit_row['Service Type']))
                    if match:
                        df.at[index, 'Provider Type - Level 2'] = match
                        break

    # Final check to ensure no invalid values were assigned
    invalid_assignments = df[
        (~df['Provider Type - Level 2'].isin(valid_types)) & 
        (~df['Provider Type - Level 2'].apply(is_empty))
    ]
    if not invalid_assignments.empty:
        df.loc[invalid_assignments.index, 'Provider Type - Level 2'] = ''

    return df

def get_new_effective_date(unique_contract_names):
    new_dates = {}
    for contract_name in unique_contract_names:
        if contract_name+'.txt' in input_dict.keys():
            contract_text = input_dict[contract_name+'.txt']
            
            try:
                prompt = f"""### Contract Start ### {contract_text} ### Contract End ###
    Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. 
    If there is no clear effective date, return the date from the signature page.
    Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
    """
                date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
                print(date_answer)
                new_dates[contract_name] = date_answer
            except:
                prompt = f"""### Contract Start ### {contract_text[0:200000]} ### Contract End ###
    Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. 
    If there is no clear effective date, return the date from the signature page.
    Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
    """
                date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
                new_dates[contract_name] = date_answer
    return new_dates




#################### Process Starts Here ###################
abc = pd.read_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
# null_counts = abc.isnull().sum()
# print(null_counts)
# quit()

input_dict = utils.read_input('data_cnc/batch3A')

# Clean Prov 2
abc_grouped = abc.groupby('Contract Name')
abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped])

# Effective Date
contains_meridian = abc_clean['PAYER NAME'].str.contains('meridian', case=False, na=False)

# Use your utility function to identify empty dates
empty_dates = abc_clean['Contract Effective Date'].apply(utils.is_empty)

# Combine filters to find the relevant 'Contract Names'
relevant_contracts = abc_clean[contains_meridian & empty_dates]['Contract Name'].unique()

# Get new effective dates for these contracts
new_effective_dates = get_new_effective_date(relevant_contracts)

# Apply the new effective dates to the DataFrame
for contract_name, new_date in new_effective_dates.items():
    # Find rows with this 'Contract Name' where dates need replacing
    condition = (abc_clean['Contract Name'] == contract_name) & contains_meridian & empty_dates
    abc_clean.loc[condition, 'Contract Effective Date'] = new_date


abc_clean.to_csv('output_consolidated/CNC-3-RERUN-DRAFT7.csv')

print("ABC Full Final (after column renaming)")
print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}")
print(abc_clean.shape)
print(list(abc_clean.columns))



# abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
# print(abc.shape)
# print(len(abc['Contract Effective Date'].unique()))

# date_mapping = pd.read_csv('output_consolidated/CNC-1-RERUN-DRAFT5.csv')
# unique_date_mapping = date_mapping.drop_duplicates(subset='Contract Name', keep='first')
# contract_dates_dict = dict(zip(unique_date_mapping['Contract Name'], unique_date_mapping['Contract Effective Date']))
# print(contract_dates_dict)

# abc_postprocessed = clean_prov_2(abc)
# abc_postprocessed['Contract Effective Date'] = abc_postprocessed['Contract Name'].map(contract_dates_dict)

# print(abc_postprocessed.shape)
# print(len(abc_postprocessed['Contract Effective Date'].unique()))
# print(list(abc_postprocessed['Contract Effective Date'].unique()))

# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv')