doczyai-pipelines/fieldExtraction/scripts/adhoc/carveouts.py

import pandas as pd
from difflib import SequenceMatcher
import re
import time

import prompts
import claude_funcs


def get_closest_substring_match(val, valid_values):
    """Returns the first match from valid_values using a case-insensitive substring match."""
    if pd.isna(val):
        return None
    val = val.strip().upper()
    for valid_value in valid_values:
        if valid_value.upper() in val:
            return valid_value
    return None


def get_best_carveout_from_claude(carveout_list, service):
    print("using claude to get best carveout...")
    prompt = f"Given the service '{service}', please choose the best matching carveout from the following list: {', '.join(carveout_list)}. ONLY SELECT ONE FROM THE LIST. DO NOT RETURN A SENTENCE"
    while True:
        try:
            response = claude_funcs.invoke_claude_3(prompt, max_tokens=100)
            return response.strip()
        except Exception as e:
            print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
            time.sleep(60)


def check_prov_type_similarity(prov_type, carveout):
    print("checking similarity between prov_type and carveout with claude...")
    prompt = f"Do the provider type '{prov_type}' and the carveout '{carveout}' mean the same thing or are they very similar? ONLY ANSWER WITH 'True' OR 'False'"
    while True:
        try:
            response = claude_funcs.invoke_claude_3(prompt, max_tokens=10)
            return response.strip()
        except Exception as e:
            print(f"Error occurred: {e}. Waiting for 60 seconds before retrying...")
            time.sleep(60)


def label_services(filepath, carveout_list, output_filepath):
    # Read the CSV file
    df = pd.read_csv(filepath)

    # Select the first 50 rows of the DataFrame
    df = df.head(500)

    # Define primary service terms
    primary_terms = [
        "Covered Services",
        "Inpatient Services",
        "Outpatient Services",
        "Physician Services",
        "Inpatient",
        "Outpatient",
        "Physician",
        "Medical",
        "Surgical",
        "Diagnostic",
        "Therapeutic",
    ]

    # Initialize columns for the results
    df["carveout_matched"] = ""
    df["label"] = ""
    df["IS_CARVEOUT"] = ""

    # Initialize a nested dictionary to track occurrences for each Filename and TD_LOB
    occurrence_tracker = {}

    carveout_count = 0

    # Iterate through the rows in the DataFrame
    for index, row in df.iterrows():
        service = str(row["SERVICE"])
        prov_type = str(row["PROV_TYPE"]) if pd.notna(row["PROV_TYPE"]) else ""
        td_lob = row["TD_LOB"]
        filename = row["Filename"]

        # Initialize the nested dictionary for the filename and TD_LOB if not present
        if filename not in occurrence_tracker:
            occurrence_tracker[filename] = {}
        if td_lob not in occurrence_tracker[filename]:
            occurrence_tracker[filename][td_lob] = {term: 0 for term in primary_terms}

        # Check if the service is a primary or similar to a primary
        primary = get_closest_substring_match(service, primary_terms)
        if primary:
            # Ensure the primary term is initialized in the occurrence tracker
            if primary not in occurrence_tracker[filename][td_lob]:
                occurrence_tracker[filename][td_lob][primary] = 0

            # Determine the label based on the count of this primary term for the given Filename and TD_LOB
            count = occurrence_tracker[filename][td_lob][primary]
            if count == 0:
                df.at[index, "label"] = "primary"
            elif count == 1:
                df.at[index, "label"] = "secondary"
            elif count == 2:
                df.at[index, "label"] = "tertiary"
            else:
                df.at[index, "label"] = "additional"
            occurrence_tracker[filename][td_lob][primary] += 1
            df.at[index, "IS_CARVEOUT"] = "N"
        else:
            # If PROV_TYPE is blank, continue with carveout determination
            best_carveout_response = get_best_carveout_from_claude(
                carveout_list, service
            )
            matched_carveout = best_carveout_response  # Use the model response directly

            df.at[index, "carveout_matched"] = matched_carveout

            if matched_carveout:
                # Check similarity between PROV_TYPE and carveout
                prov_type_similarity_score = SequenceMatcher(
                    None, prov_type, matched_carveout
                ).ratio()
                if prov_type_similarity_score < 0.8:
                    prov_type_similarity = check_prov_type_similarity(
                        prov_type, matched_carveout
                    )
                else:
                    prov_type_similarity = "True"

                if re.search(r"\b(True|Yes)\b", prov_type_similarity, re.IGNORECASE):
                    if matched_carveout not in occurrence_tracker[filename][td_lob]:
                        print(
                            "claude returned a match between provider type and service"
                        )
                        occurrence_tracker[filename][td_lob][matched_carveout] = 0
                    count = occurrence_tracker[filename][td_lob][matched_carveout]
                    if count == 0:
                        df.at[index, "label"] = "primary"
                    elif count == 1:
                        df.at[index, "label"] = "secondary"
                    elif count == 2:
                        df.at[index, "label"] = "tertiary"
                    else:
                        df.at[index, "label"] = "additional"
                    occurrence_tracker[filename][td_lob][matched_carveout] += 1
                    df.at[index, "IS_CARVEOUT"] = "N"
                else:
                    df.at[index, "IS_CARVEOUT"] = "Y"
            else:
                df.at[index, "IS_CARVEOUT"] = "Y"

            carveout_count += 1

            # Save the updated DataFrame to a new CSV file after every 10 carveouts
            if carveout_count % 10 == 0:
                df.to_csv(output_filepath, index=False)
                print(f"Saved progress after processing {carveout_count} carveouts.")

    # Final save of the updated DataFrame to a new CSV file
    df.to_csv(output_filepath, index=False)


label_services("service.csv", prompts.get_carveout_list(), "carveouts50.csv")