Alex Galarce
3740588efa
Refactor/daip2-9 code refactor * add missing import * refactor ac_smart_chunking.py * update tests * removed old and unused imports * removed outdated import * forgot to import re * refactor bottom up funcs * remove unused imports from file_processing.py * refactor dict_operations.py * remove unused import from conditional_funcs.py * replace top_down_funcs with one_to_n_funcs and remove top_down_funcs.py * remove duplicate import from one_to_n_funcs.py * refactor all utils.py imports * refactor error handling by removing last code in utils and integrating InvalidDateException into postprocessing_funcs * refactor import in claude_funcs.py to use string_funcs directly and (hopefully) resolve circular import * refactor regex_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor hotfix_helper_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor postprocess.py to remove unused imports * add numpy import to string_funcs * fix tests after utils refactor * move adhoc from tests to scripts * remove unused imports * remove scripts from mypy checking * isort * Merged main into refactor/daip-2-9-code-refactor Approved-by: Katon Minhas
158 lines
6.4 KiB
Python
158 lines
6.4 KiB
Python
|
|
import pandas as pd
|
|
pd.set_option('display.max_columns', None)
|
|
pd.set_option('display.max_rows', None)
|
|
import numpy as np
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import os
|
|
import time
|
|
|
|
import utils
|
|
import postprocessing_funcs
|
|
import claude_funcs
|
|
import config
|
|
from utils import is_empty
|
|
import re
|
|
import valid
|
|
|
|
|
|
def clean_prov_2(df):
|
|
valid_types = valid.select_valid_prov_2(df['Provider Type'])
|
|
target_rows = df[df['Provider Type - Level 2'].apply(is_empty)]
|
|
|
|
def find_exact_match(text):
|
|
if pd.isna(text) or text == '':
|
|
return None
|
|
|
|
words = re.findall(r'\b[\w/]+(?:[-\s][\w/]+)*\b', text)
|
|
for i in range(len(words)):
|
|
for j in range(i+1, len(words)+1):
|
|
phrase = ' '.join(words[i:j])
|
|
if phrase in valid_types: # Case-sensitive matching
|
|
return phrase
|
|
return None
|
|
|
|
for index, row in target_rows.iterrows():
|
|
match = None
|
|
|
|
if not is_empty(row['Service Type']):
|
|
match = find_exact_match(str(row['Service Type']))
|
|
if match:
|
|
df.at[index, 'Provider Type - Level 2'] = match
|
|
continue
|
|
|
|
if not is_empty(row['Attachment/Exhibit']):
|
|
match = find_exact_match(str(row['Attachment/Exhibit']))
|
|
if match:
|
|
df.at[index, 'Provider Type - Level 2'] = match
|
|
continue
|
|
|
|
exhibit_rows = df[df['Attachment/Exhibit'] == row['Attachment/Exhibit']]
|
|
if not exhibit_rows.empty:
|
|
for _, exhibit_row in exhibit_rows.iterrows():
|
|
if not is_empty(exhibit_row['Provider Type - Level 2']):
|
|
match = find_exact_match(str(exhibit_row['Provider Type - Level 2']))
|
|
if match:
|
|
df.at[index, 'Provider Type - Level 2'] = match
|
|
break
|
|
elif not is_empty(exhibit_row['Service Type']):
|
|
match = find_exact_match(str(exhibit_row['Service Type']))
|
|
if match:
|
|
df.at[index, 'Provider Type - Level 2'] = match
|
|
break
|
|
|
|
# Final check to ensure no invalid values were assigned
|
|
invalid_assignments = df[
|
|
(~df['Provider Type - Level 2'].isin(valid_types)) &
|
|
(~df['Provider Type - Level 2'].apply(is_empty))
|
|
]
|
|
if not invalid_assignments.empty:
|
|
df.loc[invalid_assignments.index, 'Provider Type - Level 2'] = ''
|
|
|
|
return df
|
|
|
|
def get_new_effective_date(unique_contract_names):
|
|
new_dates = {}
|
|
for contract_name in unique_contract_names:
|
|
if contract_name+'.txt' in input_dict.keys():
|
|
contract_text = input_dict[contract_name+'.txt']
|
|
|
|
try:
|
|
prompt = f"""### Contract Start ### {contract_text} ### Contract End ###
|
|
Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'.
|
|
If there is no clear effective date, return the date from the signature page.
|
|
Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
|
|
"""
|
|
date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
|
|
print(date_answer)
|
|
new_dates[contract_name] = date_answer
|
|
except:
|
|
prompt = f"""### Contract Start ### {contract_text[0:200000]} ### Contract End ###
|
|
Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'.
|
|
If there is no clear effective date, return the date from the signature page.
|
|
Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
|
|
"""
|
|
date_answer = claude_funcs.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128)
|
|
new_dates[contract_name] = date_answer
|
|
return new_dates
|
|
|
|
|
|
|
|
|
|
#################### Process Starts Here ###################
|
|
abc = pd.read_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
|
|
# null_counts = abc.isnull().sum()
|
|
# print(null_counts)
|
|
# quit()
|
|
|
|
input_dict = utils.read_input('data_cnc/batch3A')
|
|
|
|
# Clean Prov 2
|
|
abc_grouped = abc.groupby('Contract Name')
|
|
abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped])
|
|
|
|
# Effective Date
|
|
contains_meridian = abc_clean['PAYER NAME'].str.contains('meridian', case=False, na=False)
|
|
|
|
# Use your utility function to identify empty dates
|
|
empty_dates = abc_clean['Contract Effective Date'].apply(utils.is_empty)
|
|
|
|
# Combine filters to find the relevant 'Contract Names'
|
|
relevant_contracts = abc_clean[contains_meridian & empty_dates]['Contract Name'].unique()
|
|
|
|
# Get new effective dates for these contracts
|
|
new_effective_dates = get_new_effective_date(relevant_contracts)
|
|
|
|
# Apply the new effective dates to the DataFrame
|
|
for contract_name, new_date in new_effective_dates.items():
|
|
# Find rows with this 'Contract Name' where dates need replacing
|
|
condition = (abc_clean['Contract Name'] == contract_name) & contains_meridian & empty_dates
|
|
abc_clean.loc[condition, 'Contract Effective Date'] = new_date
|
|
|
|
|
|
abc_clean.to_csv('output_consolidated/CNC-3-RERUN-DRAFT7.csv')
|
|
|
|
print("ABC Full Final (after column renaming)")
|
|
print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}")
|
|
print(abc_clean.shape)
|
|
print(list(abc_clean.columns))
|
|
|
|
|
|
|
|
# abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
|
|
# print(abc.shape)
|
|
# print(len(abc['Contract Effective Date'].unique()))
|
|
|
|
# date_mapping = pd.read_csv('output_consolidated/CNC-1-RERUN-DRAFT5.csv')
|
|
# unique_date_mapping = date_mapping.drop_duplicates(subset='Contract Name', keep='first')
|
|
# contract_dates_dict = dict(zip(unique_date_mapping['Contract Name'], unique_date_mapping['Contract Effective Date']))
|
|
# print(contract_dates_dict)
|
|
|
|
# abc_postprocessed = clean_prov_2(abc)
|
|
# abc_postprocessed['Contract Effective Date'] = abc_postprocessed['Contract Name'].map(contract_dates_dict)
|
|
|
|
# print(abc_postprocessed.shape)
|
|
# print(len(abc_postprocessed['Contract Effective Date'].unique()))
|
|
# print(list(abc_postprocessed['Contract Effective Date'].unique()))
|
|
|
|
# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv') |