doczyai-pipelines/fieldExtraction/scripts/adhoc/cnc_prov2_effdate.py

import pandas as pd

pd.set_option("display.max_columns", None)
pd.set_option("display.max_rows", None)
import numpy as np
from concurrent.futures import ThreadPoolExecutor
import os
import time

import utils
import postprocessing_funcs
import claude_funcs
import config
from utils import is_empty
import re
import valid


def clean_prov_2(df):
    valid_types = valid.select_valid_prov_2(df["Provider Type"])
    target_rows = df[df["Provider Type - Level 2"].apply(is_empty)]

    def find_exact_match(text):
        if pd.isna(text) or text == "":
            return None

        words = re.findall(r"\b[\w/]+(?:[-\s][\w/]+)*\b", text)
        for i in range(len(words)):
            for j in range(i + 1, len(words) + 1):
                phrase = " ".join(words[i:j])
                if phrase in valid_types:  # Case-sensitive matching
                    return phrase
        return None

    for index, row in target_rows.iterrows():
        match = None

        if not is_empty(row["Service Type"]):
            match = find_exact_match(str(row["Service Type"]))
            if match:
                df.at[index, "Provider Type - Level 2"] = match
                continue

        if not is_empty(row["Attachment/Exhibit"]):
            match = find_exact_match(str(row["Attachment/Exhibit"]))
            if match:
                df.at[index, "Provider Type - Level 2"] = match
                continue

        exhibit_rows = df[df["Attachment/Exhibit"] == row["Attachment/Exhibit"]]
        if not exhibit_rows.empty:
            for _, exhibit_row in exhibit_rows.iterrows():
                if not is_empty(exhibit_row["Provider Type - Level 2"]):
                    match = find_exact_match(
                        str(exhibit_row["Provider Type - Level 2"])
                    )
                    if match:
                        df.at[index, "Provider Type - Level 2"] = match
                        break
                elif not is_empty(exhibit_row["Service Type"]):
                    match = find_exact_match(str(exhibit_row["Service Type"]))
                    if match:
                        df.at[index, "Provider Type - Level 2"] = match
                        break

    # Final check to ensure no invalid values were assigned
    invalid_assignments = df[
        (~df["Provider Type - Level 2"].isin(valid_types))
        & (~df["Provider Type - Level 2"].apply(is_empty))
    ]
    if not invalid_assignments.empty:
        df.loc[invalid_assignments.index, "Provider Type - Level 2"] = ""

    return df


def get_new_effective_date(unique_contract_names):
    new_dates = {}
    for contract_name in unique_contract_names:
        if contract_name + ".txt" in input_dict.keys():
            contract_text = input_dict[contract_name + ".txt"]

            try:
                prompt = f"""### Contract Start ### {contract_text} ### Contract End ###
    Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. 
    If there is no clear effective date, return the date from the signature page.
    Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
    """
                date_answer = claude_funcs.invoke_claude(
                    prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128
                )
                print(date_answer)
                new_dates[contract_name] = date_answer
            except:
                prompt = f"""### Contract Start ### {contract_text[0:200000]} ### Contract End ###
    Above is a contract. What is the contract effective date mentioned in any of the following locations: the signatory section, the preamble of the agreement, or the start of the amendment? Look for phrases such as /'This amendment is effective/'. 
    If there is no clear effective date, return the date from the signature page.
    Return the date converted to YYYY-MM-DD format, with no other commentary or explanation.
    """
                date_answer = claude_funcs.invoke_claude(
                    prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 128
                )
                new_dates[contract_name] = date_answer
    return new_dates


#################### Process Starts Here ###################
abc = pd.read_csv("output_consolidated/CNC-3-RERUN-DRAFT6.csv")
# null_counts = abc.isnull().sum()
# print(null_counts)
# quit()

input_dict = utils.read_input("data_cnc/batch3A")

# Clean Prov 2
abc_grouped = abc.groupby("Contract Name")
abc_clean = pd.concat([clean_prov_2(group) for name, group in abc_grouped])

# Effective Date
contains_meridian = abc_clean["PAYER NAME"].str.contains(
    "meridian", case=False, na=False
)

# Use your utility function to identify empty dates
empty_dates = abc_clean["Contract Effective Date"].apply(utils.is_empty)

# Combine filters to find the relevant 'Contract Names'
relevant_contracts = abc_clean[contains_meridian & empty_dates][
    "Contract Name"
].unique()

# Get new effective dates for these contracts
new_effective_dates = get_new_effective_date(relevant_contracts)

# Apply the new effective dates to the DataFrame
for contract_name, new_date in new_effective_dates.items():
    # Find rows with this 'Contract Name' where dates need replacing
    condition = (
        (abc_clean["Contract Name"] == contract_name) & contains_meridian & empty_dates
    )
    abc_clean.loc[condition, "Contract Effective Date"] = new_date


abc_clean.to_csv("output_consolidated/CNC-3-RERUN-DRAFT7.csv")

print("ABC Full Final (after column renaming)")
print(f"Unique Filenames: {len(abc_clean['Contract Name'].unique())}")
print(abc_clean.shape)
print(list(abc_clean.columns))


# abc = pd.read_excel('output_consolidated/CNC-3-RERUN-DRAFT6.csv')
# print(abc.shape)
# print(len(abc['Contract Effective Date'].unique()))

# date_mapping = pd.read_csv('output_consolidated/CNC-1-RERUN-DRAFT5.csv')
# unique_date_mapping = date_mapping.drop_duplicates(subset='Contract Name', keep='first')
# contract_dates_dict = dict(zip(unique_date_mapping['Contract Name'], unique_date_mapping['Contract Effective Date']))
# print(contract_dates_dict)

# abc_postprocessed = clean_prov_2(abc)
# abc_postprocessed['Contract Effective Date'] = abc_postprocessed['Contract Name'].map(contract_dates_dict)

# print(abc_postprocessed.shape)
# print(len(abc_postprocessed['Contract Effective Date'].unique()))
# print(list(abc_postprocessed['Contract Effective Date'].unique()))

# abc_postprocessed.to_csv('output_consolidated/CNC-1-RERUN-DRAFT7.csv')