import pandas as pd pd.set_option('display.max_columns', None) pd.set_option('display.max_rows', None) import numpy as np from concurrent.futures import ThreadPoolExecutor import os import time import utils import postprocessing_funcs import claude_funcs import config from utils import is_empty import re import valid def clean_prov_2(df): valid_types = valid.select_valid_prov_2(df['Provider Type']) target_rows = df[df['Provider Type - Level 2'].apply(is_empty)] def find_exact_match(text): if pd.isna(text) or text == '': return None words = re.findall(r'\b[\w/]+(?:[-\s][\w/]+)*\b', text) for i in range(len(words)): for j in range(i+1, len(words)+1): phrase = ' '.join(words[i:j]) if phrase in valid_types: # Case-sensitive matching return phrase return None for index, row in target_rows.iterrows(): match = None if not is_empty(row['Service Type']): match = find_exact_match(str(row['Service Type'])) if match: df.at[index, 'Provider Type - Level 2'] = match continue if not is_empty(row['Attachment/Exhibit']): match = find_exact_match(str(row['Attachment/Exhibit'])) if match: df.at[index, 'Provider Type - Level 2'] = match continue exhibit_rows = df[df['Attachment/Exhibit'] == row['Attachment/Exhibit']] if not exhibit_rows.empty: for _, exhibit_row in exhibit_rows.iterrows(): if not is_empty(exhibit_row['Provider Type - Level 2']): match = find_exact_match(str(exhibit_row['Provider Type - Level 2'])) if match: df.at[index, 'Provider Type - Level 2'] = match break elif not is_empty(exhibit_row['Service Type']): match = find_exact_match(str(exhibit_row['Service Type'])) if match: df.at[index, 'Provider Type - Level 2'] = match break # Final check to ensure no invalid values were assigned invalid_assignments = df[ (~df['Provider Type - Level 2'].isin(valid_types)) & (~df['Provider Type - Level 2'].apply(is_empty)) ] if not invalid_assignments.empty: df.loc[invalid_assignments.index, 'Provider Type - Level 2'] = '' return df def get_new_effective_date(unique_contract_names): new_dates = {} for contract_name in unique_contract_names: if contract_name+'.txt' in input_dict.keys(): contract_text = input_dict[contract_name+'.txt'] try: prompt = f"""### Contract Start ### {contract_text} ### Contract End ### Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. If there is no clear effective date, return the date from the signature page. Return the date converted to YYYY-MM-DD format, with no other commentary or explanation. """ date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128) print(date_answer) new_dates[contract_name] = date_answer except: prompt = f"""### Contract Start ### {contract_text[0:200000]} ### Contract End ### Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. If there is no clear effective date, return the date from the signature page. Return the date converted to YYYY-MM-DD format, with no other commentary or explanation. """ date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128) new_dates[contract_name] = date_answer return new_dates #################### Process Starts Here ################### abc = pd.read_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv') # null_counts = abc.isnull().sum() # print(null_counts) # quit() input_dict = utils.read_input('data_cnc/batch3A') # Clean Prov 2 abc_grouped = abc.groupby('Contract Name') abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped]) # Effective Date contains_meridian = abc_clean['PAYER NAME'].str.contains('meridian', case=False, na=False) # Use your utility function to identify empty dates empty_dates = abc_clean['Contract Effective Date'].apply(utils.is_empty) # Combine filters to find the relevant 'Contract Names' relevant_contracts = abc_clean[contains_meridian & empty_dates]['Contract Name'].unique() # Get new effective dates for these contracts new_effective_dates = get_new_effective_date(relevant_contracts) # Apply the new effective dates to the DataFrame for contract_name, new_date in new_effective_dates.items(): # Find rows with this 'Contract Name' where dates need replacing condition = (abc_clean['Contract Name'] == contract_name) & contains_meridian & empty_dates abc_clean.loc[condition, 'Contract Effective Date'] = new_date abc_clean.to_csv('output_consolidated/CNC-3-RERUN-DRAFT7.csv') print("ABC Full Final (after column renaming)") print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}") print(abc_clean.shape) print(list(abc_clean.columns)) # abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv') # print(abc.shape) # print(len(abc['Contract Effective Date'].unique())) # date_mapping = pd.read_csv('output_consolidated/CNC-1-RERUN-DRAFT5.csv') # unique_date_mapping = date_mapping.drop_duplicates(subset='Contract Name', keep='first') # contract_dates_dict = dict(zip(unique_date_mapping['Contract Name'], unique_date_mapping['Contract Effective Date'])) # print(contract_dates_dict) # abc_postprocessed = clean_prov_2(abc) # abc_postprocessed['Contract Effective Date'] = abc_postprocessed['Contract Name'].map(contract_dates_dict) # print(abc_postprocessed.shape) # print(len(abc_postprocessed['Contract Effective Date'].unique())) # print(list(abc_postprocessed['Contract Effective Date'].unique())) # abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv')