doczyai-pipelines/src/pipelines/shared/postprocessing/postprocessing_funcs.py

import ast
import hashlib
import json
import logging
import os
import re
from datetime import datetime

import pandas as pd

from src.utils import string_utils
from src.utils import json_utils

# Determine the base directory for the project
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Path to the mappings folder
MAPPINGS_DIR = os.path.join(BASE_DIR, "crosswalk", "mappings")


def rename_columns(df):
    """
    Use the provided mapping to rename columns in the data

    Args:
        df (pd.DataFrame): The input DataFrame with columns to be renamed.
    Returns:
        pd.DataFrame: The DataFrame with renamed columns.
    """
    # Check if the DataFrame is empty, return it as is if it is
    if df.empty:
        return df

    # If not empty, proceed with column renaming
    rename_map = {
        "PROCEDURE_CD": "CPT4_PROC_CD",
        "PROCEDURE_CD_DESC": "CPT4_PROC_CD_DESC",
    }
    # Rename columns using the mapping
    df = df.rename(columns=rename_map)
    return df


def normalize_indicator_field(value: str) -> str:
    """
    Normalize the indicator field value to 'Y' or 'N'.

    Explicitly handles empty strings, None, NaN, and blank values by setting them to 'N'.

    Args:
        value (str): The input value from an _IND field.
    Returns:
        str: 'Y' if the value is 'Y', otherwise 'N'.
    """
    # Handle None, NaN, and empty values
    if value is None or (isinstance(value, float) and pd.isna(value)):
        return "N"

    # Handle empty strings and whitespace-only strings
    if isinstance(value, str):
        value_stripped = value.strip()
        if not value_stripped:
            return "N"
        if value_stripped.upper() == "Y":
            return "Y"

    # Everything else becomes 'N'
    return "N"


def clean_na_values(df: pd.DataFrame) -> pd.DataFrame:
    """
    Remove cells containing ONLY "N/A", "['N/A']", "[]", "['[]']", "UNKNOWN", or similar placeholder values.

    This function removes placeholder values only when they are the sole value in a cell.
    If a placeholder value is part of a list with other non-placeholder values, it is kept.
    Empty list representations (e.g. "[]", "['[]']") are replaced with blank to avoid
    clutter in the output.

    Args:
        df (pd.DataFrame): Input DataFrame to clean.

    Returns:
        pd.DataFrame: DataFrame with placeholder values removed from cells where they are the only value.
    """
    if df.empty:
        return df

    df_cleaned = df.copy()

    # Patterns to match placeholder values (case-insensitive)
    # Include [] and ['[]'] so empty list representations are blanked instead of shown
    placeholder_patterns = [
        r"^N/A$",
        r"^NA$",
        r"^NaN$",
        r"^UNKNOWN$",
        r"^None$",
        r"^null$",
        r"^\['N/A'\]$",
        r"^\[\"N/A\"\]$",
        r"^\[N/A\]$",
        r"^\[\]$",  # Empty list "[]"
        r"^\['\[\]'\]$",  # Python repr of list containing "[]"
        r"^\[\"\[\]\"\]$",  # JSON list containing "[]"
    ]

    def is_placeholder_only(value):
        """Check if value is only a placeholder."""
        # Handle None, NaN, and pd.NA
        if value is None:
            return True
        try:
            if pd.isna(value):
                return True
        except (TypeError, ValueError):
            # pd.isna might raise TypeError for some types, continue checking
            pass

        if isinstance(value, str):
            value_stripped = value.strip()
            if not value_stripped:
                return True

            # Try to parse as JSON list first
            if value_stripped.startswith("[") and value_stripped.endswith("]"):
                try:
                    parsed_list = json.loads(value_stripped)
                    if isinstance(parsed_list, list):
                        # If list contains only placeholder values, consider it placeholder
                        if len(parsed_list) == 0:
                            return True
                        all_placeholders = True
                        for item in parsed_list:
                            item_str = str(item).strip()
                            is_placeholder = False
                            for pattern in placeholder_patterns:
                                if re.match(pattern, item_str, re.IGNORECASE):
                                    is_placeholder = True
                                    break
                            if not is_placeholder and item_str:
                                all_placeholders = False
                                break
                        return all_placeholders
                except (json.JSONDecodeError, ValueError):
                    # If JSON parsing fails, continue with string matching
                    pass

            # Check against placeholder patterns for string values
            for pattern in placeholder_patterns:
                if re.match(pattern, value_stripped, re.IGNORECASE):
                    return True

        # Handle list types (Python lists)
        if isinstance(value, list):
            if len(value) == 0:
                return True
            # If list contains only placeholder values, consider it placeholder
            all_placeholders = True
            for item in value:
                item_str = str(item).strip()
                is_placeholder = False
                for pattern in placeholder_patterns:
                    if re.match(pattern, item_str, re.IGNORECASE):
                        is_placeholder = True
                        break
                if not is_placeholder and item_str:
                    all_placeholders = False
                    break
            return all_placeholders

        return False

    # Apply cleaning to all columns
    for col in df_cleaned.columns:
        df_cleaned[col] = df_cleaned[col].apply(
            lambda x: "" if is_placeholder_only(x) else x
        )

    return df_cleaned


def format_rate_fields_with_commas(value: str) -> str:
    """
    Format the rate with commas and two decimal places.

    Args:
        value (str): A string representing a numeric value.
    Returns:
        str: The formatted rate string.
    """
    if string_utils.is_empty(value):
        return ""

    try:
        numeric_value = float(value)
        return f"{numeric_value:,.2f}"
    except ValueError:
        return ""


def normalize_currency(value: str) -> str:
    """
    Normalize currency-like strings by removing $ and commas.

    Args:
        value (str): A string representing a currency value.
    Returns:
        str: Normalized numeric string or empty string if invalid.
    """
    if string_utils.is_empty(value):
        return ""

    if isinstance(value, (int, float)) and not pd.isna(value):
        return str(value)

    if not isinstance(value, str):
        return ""

    text = value.strip()
    if text.upper() in {"N/A", "NA", "NAN"}:
        return ""

    text = text.replace("$", "").replace(",", "").strip()
    return text


def flatten_singleton_string_list(list_str: str) -> str:
    """Take in a list within a string, and if it has a single element, return that element.
    If it has multiple elements, return the list as a string.

    Args:
        list_str (str): A string representation of a list.

    Returns:
        str: The single element if the list has one element, otherwise the list as a string.
    """
    if string_utils.is_empty(list_str):
        return ""
    try:
        # Convert the string to a list using ast.literal_eval
        list_obj = ast.literal_eval(list_str)
        # Check if list_obj is actually a list
        if not isinstance(list_obj, list):
            return list_str
        # If the list has one element, return that element
        if len(list_obj) == 1:
            return list_obj[0]
        # Otherwise, return the list as a string
        return ", ".join(list_obj)
    except (ValueError, SyntaxError):
        return list_str


def validate_and_reformat_date(date_str: str) -> str:
    """
    Validate if a string is in YYYY/MM/DD format or reformat it to YYYY/MM/DD if possible.

    Strips leading/trailing pipes (e.g. |01/01/2022|) before parsing so date-only
    content is validated and reformatted.

    Args:
        date_str (str): The input date string.

    Returns:
        str: The reformatted date string if valid or reformatted, otherwise the original string.
    """
    if not isinstance(date_str, str):
        return date_str

    # Strip pipes and whitespace so |01/01/2022| becomes 01/01/2022 before parsing.
    date_str = date_str.strip().strip("|").strip()
    if not date_str:
        return ""

    try:
        # Check if the date is already in YYYY/MM/DD format
        datetime.strptime(date_str, "%Y/%m/%d")
        return date_str
    except ValueError:
        # Attempt to reformat the date from known formats (date-only and datetime).
        known_formats = [
            "%Y-%m-%d",
            "%m/%d/%Y",
            "%d-%b-%Y",
            "%d/%m/%Y",
            "%Y-%m-%d %H:%M:%S",
            "%Y-%m-%d %H:%M:%S.%f",
        ]
        for fmt in known_formats:
            try:
                return datetime.strptime(date_str, fmt).strftime("%Y/%m/%d")
            except ValueError:
                continue
    return date_str


def normalize_auto_renewal_term(text: str) -> str:
    """
    Normalize the auto-renewal term to a standard format.

    Args:
        text (str): The input auto-renewal term.

    Returns:
        str: The normalized auto-renewal term.
    """
    if not text or not isinstance(text, str):
        return ""

    # Convert to lowercase and strip whitespace
    text = text.lower().strip()

    # Remove numbers enclosed in parentheses (e.g., "(1) ", "(12) ")
    text = re.sub(r"\(\d+\) ", "", text)

    # Define replacements
    replacements = {
        r"\bone\b": "1",
        r"\btwo\b": "2",
        r"\bthree\b": "3",
        r"\btwelve\b": "12",
        r"year to year": "1 year",
        r"12 months": "1 year",
        r"\btwelve months\b": "1 year",
        r"\bone year\b": "1 year",
        r"\bone-year\b": "1 year",
        r"\b1 year\b": "1 year",
        r"\bmonth to month\b": "1 month",
    }

    # Apply replacements
    for pattern, replacement in replacements.items():
        text = re.sub(pattern, replacement, text)

    # Remove any remaining parentheses or extra spaces
    text = text.strip().replace("(", "").replace(")", "")

    text = text.replace("-", " ")
    if text == "year":
        text = "1 year"

    return text


def normalize_cpt_fields(value):
    """
    Normalize code values to a list format.

    Args:
        value (Any): Input value which could be a list, string, or other types.

    Returns:
        list: A normalized list of string codes.
    """
    if not value or (isinstance(value, float) and pd.isna(value)):  # Handle NaN values
        return ""  # Empty string

    # First normalize to a list
    if isinstance(value, (list, tuple)):  # If already a list or tuple
        result = [str(v).strip() for v in value]
    elif isinstance(value, str):  # If it's a string
        value = value.strip()
        if value.startswith("[") and value.endswith(
            "]"
        ):  # String representation of a list
            try:
                # Parse it first
                parsed_list = ast.literal_eval(value)
                result = []
                # Process each item in the parsed list
                for item in parsed_list:
                    item_str = str(item).strip()
                    # If the item contains commas, split it into multiple items
                    if "," in item_str:
                        result.extend([str(v).strip() for v in item_str.split(",")])
                    else:
                        result.append(item_str)
            except (ValueError, SyntaxError):
                result = [value]  # If parsing fails, treat it as a single value
        elif "," in value:  # If it's a comma-separated string
            # Split by comma and create a list
            result = [str(v).strip() for v in value.split(",")]
        elif "-" in value:
            # Range format
            result = [value]
        else:  # Single value
            result = [value]
    elif isinstance(value, (float, int)):  # If it's a number
        result = [str(int(value)).strip()]
    else:
        result = [str(value).strip()]  # Default case for other types

    return json.dumps(result)  # Return JSON list string


def generate_reimb_ids(df: pd.DataFrame) -> pd.DataFrame:
    """Generates REIMB_ID and REIMB_LESSER_OF_ID for each row in the DataFrame based on
        the FILE_NAME and other fields.
    The REIMB_ID is constructed using the following format:
    {FILE_NAME}_exh_pg_{EXHIBIT_PAGE}_{index}_{hash_value}

    the REIMB_LESSER_OF_ID is just the {hash_value} itself.

    Where:
    - FILE_NAME is the name of the file (without extension).
    - EXHIBIT_PAGE is the page number of the exhibit.
    - index is the index of the row in the group.
    - hash_value is the first 8 characters of the MD5 hash of a concatenated string of relevant fields.
    The relevant fields are SERVICE_TERM, REIMB_TERM, PROGRAM, PRODUCT, NETWORK, and LOB.

    Args:
        df (pd.DataFrame): Input DataFrame containing the columns FILE_NAME, EXHIBIT_PAGE, SERVICE_TERM, REIMB_TERM, PROGRAM, PRODUCT, NETWORK, and LOB.
        The FILE_NAME column should contain the name of the file (with or without extension).
        The EXHIBIT_PAGE column should contain the page number of the exhibit.

    Returns:
        pd.DataFrame: DataFrame with generated REIMB_IDs and REIMB_LESSER_OF_IDs for each row.
    """
    # First sort to ensure consistent ordering
    df = (
        df.sort_values(by=["FILE_NAME", "EXHIBIT_PAGE"])
        if "EXHIBIT_PAGE" in df.columns
        else df.sort_values(by=["FILE_NAME"])
    )

    # Create a temporary copy to avoid SettingWithCopyWarning
    df_temp = df.copy()

    # Process each file+exhibit_page group to reset counter for each exhibit page
    group_cols = (
        ["FILE_NAME", "EXHIBIT_PAGE"] if "EXHIBIT_PAGE" in df.columns else ["FILE_NAME"]
    )
    for group_key, indices in df.groupby(group_cols).groups.items():
        if isinstance(group_key, tuple):
            filename, exhibit_page = group_key
            exhibit_page = str(exhibit_page).zfill(3)
        else:
            filename = group_key
            exhibit_page = "000"
        clean_filename = str(filename).split("/")[-1].replace(".", "_")

        for i, idx in enumerate(indices):
            row = df.loc[idx]

            # Get fields to include in hash
            service_term = str(row.get("SERVICE_TERM", ""))
            reimb_term = str(row.get("REIMB_TERM", ""))
            program = str(row.get("PROGRAM", ""))
            product = str(row.get("PRODUCT", ""))
            network = str(row.get("NETWORK", ""))
            lob = str(row.get("LOB", ""))

            # Create hash string
            hash_string = f"{clean_filename}_{service_term}_{reimb_term}_{program}_{product}_{network}_{lob}"
            hash_value = hashlib.md5(hash_string.encode()).hexdigest()[:8]

            # Generate REIMB_ID and REIMB_LESSER_OF_ID
            df_temp.at[idx, "REIMB_ID"] = (
                f"{clean_filename}_exh_pg_{exhibit_page}_{i+1:03d}_{hash_value}"
            )
            df_temp.at[idx, "REIMB_LESSER_OF_ID"] = hash_value

    return df_temp


def reorder_columns(
    df: pd.DataFrame, field_format_mapping: dict[str, str]
) -> pd.DataFrame:
    """
    Reorders the columns of the DataFrame based on the given column order.
    Steps:
    1. Adds any missing columns from `column_order` to the DataFrame at once, filled with empty strings.
    2. Reorders the columns of the DataFrame to match `column_order`.
    3. Appends any columns in the DataFrame that are not in `column_order` to the end.

    Args:
        df (pd.DataFrame): The input DataFrame.
        column_order (list[str]): The desired column order.

    Returns:
        pd.DataFrame: The reordered DataFrame.
    """
    # Get a list of column names, preserving order
    column_order = list(field_format_mapping.keys())

    # Create a copy to avoid fragmentation
    df_copy = df.copy()

    # Find missing columns from column_order
    missing_columns = [col for col in column_order if col not in df_copy.columns]

    # Add all missing columns at once using a dictionary
    if missing_columns:
        # Create a dictionary of empty columns
        empty_cols = {col: [""] * len(df_copy) for col in missing_columns}
        # Add all missing columns at once with pd.concat
        df_copy = pd.concat(
            [df_copy, pd.DataFrame(empty_cols, index=df_copy.index)], axis=1
        )

    # Reorder columns to match column_order
    final_column_order = [col for col in column_order if col in df_copy.columns]

    # Return the DataFrame with reordered columns
    return df_copy[final_column_order]


def auto_renewal(df: pd.DataFrame) -> pd.DataFrame:
    # Normalize the 'AUTO_RENEWAL_TERM' column
    if "AUTO_RENEWAL_TERM" in df.columns:
        df["AUTO_RENEWAL_TERM"] = df["AUTO_RENEWAL_TERM"].apply(
            normalize_auto_renewal_term
        )
    return df


def update_termination_date_for_conditions(df: pd.DataFrame) -> pd.DataFrame:
    """
    Updates the 'AARETE_DERIVED_TERMINATION_DT' column based on specific conditions
    involving 'TERMINATION_DT', 'AUTO_RENEWAL_IND', and 'AARETE_DERIVED_TERMINATION_DT'.

    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with modified 'AARETE_DERIVED_TERMINATION_DT' values.
    """

    if (
        "TERMINATION_DT" in df.columns
        and "AUTO_RENEWAL_IND" in df.columns
        and "AARETE_DERIVED_TERMINATION_DT" in df.columns
    ):
        # Replace Y with 9999/12/31
        df.loc[df["AUTO_RENEWAL_IND"] == "Y", "AARETE_DERIVED_TERMINATION_DT"] = (
            "9999/12/31"
        )

        # Replace "N" and "year to year" with 9999/12/31
        df.loc[
            (df["TERMINATION_DT"] == "year to year")
            & (df["AUTO_RENEWAL_IND"] == "N")
            & (df["AARETE_DERIVED_TERMINATION_DT"] == "N/A"),
            "AARETE_DERIVED_TERMINATION_DT",
        ] = "9999/12/31"

    return df


def standardize_reimb_method_and_fee_schedule(
    df: pd.DataFrame, VALID_UNIT_OF_MEASURE: list
) -> pd.DataFrame:
    """
    Standardizes the 'AARETE_DERIVED_REIMB_METHOD' column and updates the
    'AARETE_DERIVED_FEE_SCHEDULE' column based on specific values.

    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with standardized columns.
    """
    if (
        "AARETE_DERIVED_REIMB_METHOD" in df.columns
        and "AARETE_DERIVED_FEE_SCHEDULE" in df.columns
    ):
        # Standardize 'AARETE_DERIVED_REIMB_METHOD' values
        # Copy AWP, ASP, WAC to AARETE_DERIVED_FEE_SCHEDULE column for the corresponding row
        medrx_fee_schedule_values = ["AWP", "ASP", "WAC"]
        df.loc[
            df["AARETE_DERIVED_REIMB_METHOD"].isin(medrx_fee_schedule_values),
            "AARETE_DERIVED_FEE_SCHEDULE",
        ] = df["AARETE_DERIVED_REIMB_METHOD"]

        # Convert AARETE_DERIVED_REIMB_METHOD values [AWP, ASP, WAC] to 'MedRx'
        df["AARETE_DERIVED_REIMB_METHOD"] = df["AARETE_DERIVED_REIMB_METHOD"].replace(
            {"AWP": "MedRx", "ASP": "MedRx", "WAC": "MedRx"}
        )

    if "AARETE_DERIVED_REIMB_METHOD" in df.columns:

        # Remove rows where 'AARETE_DERIVED_REIMB_METHOD' is 'DOFR' but 'REIMB_TERM' does not contain 'DOFR'
        mask = (df["AARETE_DERIVED_REIMB_METHOD"] == "DOFR") & (
            ~df["REIMB_TERM"].str.contains("DOFR", na=False)
        )
        df = df[~mask]

    # standardize the 'UNIT_OF_MEASURE' column
    if "UNIT_OF_MEASURE" in df.columns:
        df["UNIT_OF_MEASURE"] = df["UNIT_OF_MEASURE"].str.replace(
            "Per Encounter", "Per Visit"
        )

        # remove cases of Per Member Per Month (PMPM)
        df = df[~df["UNIT_OF_MEASURE"].isin(["Per Member Per Month (PMPM)"])]

        # Enhanced logic: Set UNIT_OF_MEASURE as 'Per Unit' for Flat Rate reimbursement method with empty UNIT_OF_MEASURE
        # This catches all edge cases including NaN, None, empty strings, etc.
        if "AARETE_DERIVED_REIMB_METHOD" in df.columns:
            flat_rate_mask = df["AARETE_DERIVED_REIMB_METHOD"] == "Flat Rate"
            empty_uom_mask = (
                df["UNIT_OF_MEASURE"].isna()
                | df["UNIT_OF_MEASURE"].eq("")
                | df["UNIT_OF_MEASURE"].apply(string_utils.is_empty)
            )
            df.loc[flat_rate_mask & empty_uom_mask, "UNIT_OF_MEASURE"] = "Per Unit"

        # Ensure UNIT_OF_MEASURE is one of the valid values
        df["UNIT_OF_MEASURE"] = df["UNIT_OF_MEASURE"].apply(
            lambda x: x if x in VALID_UNIT_OF_MEASURE else ""
        )

    return df


def blank_uom_for_default_flat_rate_cc(df: pd.DataFrame) -> pd.DataFrame:
    """
    CC output only: blank UNIT_OF_MEASURE when DEFAULT_IND='Y' and method is flat rate.

    Ensures default flat-rate reimbursements are not shown as per-unit in CC output.
    Dashboard output is not modified (call this only in contract_config_postprocess).
    """
    if not all(
        c in df.columns
        for c in ("DEFAULT_IND", "AARETE_DERIVED_REIMB_METHOD", "UNIT_OF_MEASURE")
    ):
        return df
    default_ind_y = df["DEFAULT_IND"].astype(str).str.strip().str.upper() == "Y"
    flat_rate_method = (
        df["AARETE_DERIVED_REIMB_METHOD"].astype(str).str.strip().str.lower()
        == "flat rate"
    )
    df.loc[default_ind_y & flat_rate_method, "UNIT_OF_MEASURE"] = ""
    return df


def process_patient_age_range(df):
    """
    Process the 'PATIENT_AGE_RANGE' column into 'PATIENT_AGE_MIN' and 'PATIENT_AGE_MAX'.
    If a single number is provided, both min and max are set to that number.
    Remove PATIENT_AGE_RANGE column.

    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with 'PATIENT_AGE_MIN' and 'PATIENT_AGE_MAX' columns.
    """
    if "PATIENT_AGE_RANGE" in df.columns:
        try:
            # Create temporary columns for processing
            df["PATIENT_AGE_MIN"] = df["PATIENT_AGE_RANGE"]
            df["PATIENT_AGE_MAX"] = df["PATIENT_AGE_RANGE"]

            # Handle hyphenated ranges
            mask = df["PATIENT_AGE_RANGE"].str.contains("-", na=False)
            if mask.any():
                temp_split = df.loc[mask, "PATIENT_AGE_RANGE"].str.split(
                    "-", expand=True
                )
                df.loc[mask, "PATIENT_AGE_MIN"] = temp_split[0]
                df.loc[mask, "PATIENT_AGE_MAX"] = temp_split[1]

        except Exception as e:
            # If any error occurs, keep the original values
            pass

        # Remove the original PATIENT_AGE_RANGE column
        df.drop(columns=["PATIENT_AGE_RANGE"], inplace=True, errors="ignore")

    return df


def add_aarete_derived_amendment_num(df: pd.DataFrame) -> pd.DataFrame:
    """
    Add the 'AARETE_DERIVED_AMENDMENT_NUM' column to the DataFrame.

    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with the 'AARETE_DERIVED_AMENDMENT_NUM' column.
    """

    def derive_amendment_num(value):
        if value is None or (isinstance(value, float) and pd.isna(value)):
            return "0"

        if not isinstance(value, str):
            value = str(value)

        text = value.strip()
        if string_utils.is_empty(text):
            return "0"

        lowered = text.lower()
        if lowered in {"n/a", "na", "nan", "unknown", "unspecified", "none", "null"}:
            return "0"

        if any(token in lowered for token in ["base", "original", "master"]):
            return "N/A"

        numbers = re.findall(r"\b\d+\b", text)
        if numbers:
            return str(int(numbers[-1]))

        return "0"

    if "CONTRACT_AMENDMENT_NUM" in df.columns:
        df["AARETE_DERIVED_AMENDMENT_NUM"] = (
            df["CONTRACT_AMENDMENT_NUM"]
            .str.split("-")
            .str[1]
            .fillna(df["CONTRACT_AMENDMENT_NUM"])
        )

        df["AARETE_DERIVED_AMENDMENT_NUM"] = df["CONTRACT_AMENDMENT_NUM"].apply(
            derive_amendment_num
        )

    return df


def add_aarete_derived_product(df):
    """
    Add AARETE_DERIVED_PRODUCT column to df, based on PRODUCT column.

    The new column should be Title Case of the PRODUCT column value.
    """
    if "PRODUCT" in df.columns:
        df["AARETE_DERIVED_PRODUCT"] = df["PRODUCT"]
    return df


def fill_empty_reimb_pct_rate(df):
    """
    Fill 'REIMB_PCT_RATE' based on reimbursement method:
    - Set 'REIMB_PCT_RATE' to '100' for rows where:
      - 'REIMB_PCT_RATE', 'REIMB_FEE_RATE', and 'REIMB_CONVERSION_FACTOR' are all empty.
    - Set 'REIMB_PCT_RATE' to 'N/A' for rows where:
      - 'AARETE_DERIVED_REIMB_METHOD' contains specific grouper.
    """
    required_cols = [
        "REIMB_PCT_RATE",
        "REIMB_FEE_RATE",
        "REIMB_CONVERSION_FACTOR",
        "AARETE_DERIVED_REIMB_METHOD",
    ]

    if all(col in df.columns for col in required_cols):
        # Fill with '100' when all three rate fields are empty
        pct_empty = string_utils.is_empty(df["REIMB_PCT_RATE"])
        fee_empty = string_utils.is_empty(df["REIMB_FEE_RATE"])
        conversion_empty = string_utils.is_empty(df["REIMB_CONVERSION_FACTOR"])
        applicable_methods = ["Billed Charges", "Fee Schedule", "Case Rate", "Cost"]
        method_applicable = df["AARETE_DERIVED_REIMB_METHOD"].isin(applicable_methods)

        fill_mask = pct_empty & fee_empty & conversion_empty & method_applicable
        df.loc[fill_mask, "REIMB_PCT_RATE"] = "100"

        # Set to N/A for specific reimbursement methods
        na_methods = ["Per Diem", "Flat Rate", "Per Visit"]

        na_mask = df["AARETE_DERIVED_REIMB_METHOD"].isin(na_methods)
        df.loc[na_mask, "REIMB_PCT_RATE"] = "N/A"

    return df


def remove_redundant_reimb_info(df):
    """
    Checks if "REIMB_" fields are the same as their corresponding 1:1 fields.
    If they are the same, replace the "REIMB_" values with ""

    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with redundant reimbursement field values removed.
    """
    COLUMN_PAIRS = [
        ("REIMB_EFFECTIVE_DT", "AARETE_DERIVED_EFFECTIVE_DT"),
        ("REIMB_TERMINATION_DT", "AARETE_DERIVED_TERMINATION_DT"),
        ("REIMB_PROV_NAME", "PROV_GROUP_NAME_FULL"),
    ]
    for column_tuple in COLUMN_PAIRS:
        reimb_col = column_tuple[0]
        one_to_one_col = column_tuple[1]
        if all(col in df.columns for col in [reimb_col, one_to_one_col]):
            mask = df[reimb_col] == df[one_to_one_col]
            df.loc[mask, reimb_col] = ""
    return df


def replace_provider_name_from_hybrid_smart_chunking(df):
    """
    Replaces PROV_GROUP_NAME_FULL with PROVIDER_NAME for rows where PROVIDER_NAME is not empty/N/A,
    then removes the PROVIDER_NAME column.

    This function is used in postprocessing after the Hybrid Smart Chunking provider name extraction.
    The RAG method may produce a more accurate provider name than the initial extraction,
    so we use it to override PROV_GROUP_NAME_FULL when available.

    Args:
        df (pd.DataFrame): DataFrame containing contract data

    Returns:
        pd.DataFrame: DataFrame with PROV_GROUP_NAME_FULL updated and PROVIDER_NAME removed
    """
    # Check if both columns exist
    if "PROVIDER_NAME" not in df.columns:
        logging.debug(
            "PROVIDER_NAME column not found in DataFrame, skipping replacement"
        )
        return df

    if "PROV_GROUP_NAME_FULL" not in df.columns:
        logging.debug("PROV_GROUP_NAME_FULL column not found in DataFrame")
        # Just remove PROVIDER_NAME if it exists
        df = df.drop(columns=["PROVIDER_NAME"])
        return df

    # Create a mask for rows where PROVIDER_NAME should replace PROV_GROUP_NAME_FULL
    # Replace when PROVIDER_NAME is not empty
    mask = ~string_utils.is_empty(df["PROVIDER_NAME"], pd_mask=True)

    # Count how many rows will be updated
    num_replacements = mask.sum()

    if num_replacements > 0:
        # Replace PROV_GROUP_NAME_FULL with PROVIDER_NAME for matching rows
        df.loc[mask, "PROV_GROUP_NAME_FULL"] = df.loc[mask, "PROVIDER_NAME"]
        logging.debug(
            f"Replaced PROV_GROUP_NAME_FULL with PROVIDER_NAME for {num_replacements} rows"
        )
    else:
        logging.debug("No valid PROVIDER_NAME values to replace PROV_GROUP_NAME_FULL")

    # Remove the PROVIDER_NAME column
    df = df.drop(columns=["PROVIDER_NAME"])

    return df


def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
    """Deduplicate provider fields in the DataFrame.

    This function removes PROV_GROUP_* values from the PROV_OTHER_* columns if they are already present.
    In addition, it deduplicates PROV_OTHER_* values.

    E.g. If PROV_GROUP_TIN is ["123456789"] and PROV_OTHER_TIN is ["123456789", "987654321", "987654321"],
    then PROV_OTHER_TIN will be updated to ["987654321"].


    Args:
        df (pd.DataFrame): The input DataFrame.

    Returns:
        pd.DataFrame: The updated DataFrame with deduplicated provider fields.
    """
    provider_columns = [
        "PROV_GROUP_TIN",
        "PROV_GROUP_NPI",
        "PROV_GROUP_NAME_FULL",
        "PROV_OTHER_TIN",
        "PROV_OTHER_NPI",
        "PROV_OTHER_NAME_FULL",
    ]

    if not all(col in df.columns for col in provider_columns):
        return df

    for index, row in df.iterrows():
        group_values = {
            "PROV_OTHER_TIN": row.get("PROV_GROUP_TIN"),
            "PROV_OTHER_NPI": row.get("PROV_GROUP_NPI"),
            "PROV_OTHER_NAME_FULL": row.get("PROV_GROUP_NAME_FULL"),
        }

        for other_col, group_vals in group_values.items():
            other_vals = row.get(other_col)

            if not other_vals:
                # None, NaN, or empty list
                df.at[index, other_col] = []
                continue

            # Normalize other_vals to a list if it's a string
            if isinstance(other_vals, str):
                other_vals = [other_vals]

            # Normalize group values (list or empty)
            group_vals = group_vals or []

            # Remove GROUP values
            filtered = [
                v for v in other_vals if v not in group_vals and v and v != "UNKNOWN"
            ]

            # Deduplicate while preserving order
            seen = set()
            deduped = []
            for v in filtered:
                if v not in seen:
                    seen.add(v)
                    deduped.append(v)

            df.at[index, other_col] = deduped

    return df


def update_grouper_base_rate_and_grouper_pct_rate(df):
    """
    Update GROUPER_PCT_RATE and GROUPER_BASE_RATE columns based on grouper conditions.

    Creates new columns and populates them when:
    - AARETE_DERIVED_REIMB_METHOD is 'Grouper', OR
    - GROUPER_TYPE is not empty (using string_utils.is_empty) AND not empty array/empty array string, OR
    - GROUPER_CD is not empty (using string_utils.is_empty) AND not empty array/empty array string

    Args:
        df: DataFrame to update

    Returns:
        DataFrame with updated GROUPER_PCT_RATE and GROUPER_BASE_RATE columns
    """
    # Required columns for the operation
    required_cols = [
        "AARETE_DERIVED_REIMB_METHOD",
        "REIMB_PCT_RATE",
        "REIMB_CONVERSION_FACTOR",
        "GROUPER_TYPE",
        "GROUPER_CD",
    ]

    # Check if all required columns exist
    if not all(col in df.columns for col in required_cols):
        # Initialize empty columns and return if required columns are missing
        df["GROUPER_PCT_RATE"] = ""
        df["GROUPER_BASE_RATE"] = ""
        return df

    # Create grouper condition mask
    grouper_mask = (df["AARETE_DERIVED_REIMB_METHOD"] == "Grouper") & (
        (~string_utils.is_empty(df["GROUPER_TYPE"]) & df["GROUPER_TYPE"].ne("[]"))
        | (~string_utils.is_empty(df["GROUPER_CD"]) & df["GROUPER_CD"].ne("[]"))
    )

    # Initialize new columns with empty strings
    df["GROUPER_PCT_RATE"] = ""
    df["GROUPER_BASE_RATE"] = ""

    # Update values where grouper condition is met
    df.loc[grouper_mask, "GROUPER_PCT_RATE"] = df.loc[grouper_mask, "REIMB_PCT_RATE"]
    df.loc[grouper_mask, "GROUPER_BASE_RATE"] = df.loc[
        grouper_mask, "REIMB_CONVERSION_FACTOR"
    ]

    return df


def attach_sid_column(df: pd.DataFrame) -> pd.DataFrame:
    """
    Ensures the given AARETE_DERIVED_SID columns exist in the DataFrame, fills missing values,
    and creates a unique 'AARETE_DERIVED_SID' identifier column — while keeping all original columns.

    Steps:
    1. Verify that all columns listed in sid_cols exist in the DataFrame.
       - If any are missing, create them and fill with ''.
    2. Concatenate all sid_cols values with '__' as a separator to form 'AARETE_DERIVED_SID'.
    3. If any value in sid_cols is blank or NaN, replace it with 'xx' in the concatenation.
    4. Add the 'AARETE_DERIVED_SID' column to the DataFrame.

    Parameters
    ----------
    df : pd.DataFrame
        Input DataFrame containing the source columns.
    Returns
    -------
    pd.DataFrame
        DataFrame with all original columns plus a new 'AARETE_DERIVED_SID' column.
    """
    sid_cols = [
        "CLIENT_NAME",
        "PAYER_NAME",
        "PROV_GROUP_TIN",
        "DOCUMENT_TYPE",
        "AARETE_DERIVED_AMENDMENT_NUM",
        "AARETE_DERIVED_CLAIM_TYPE_CD",
        "AARETE_DERIVED_LOB",
        "AARETE_DERIVED_EFFECTIVE_DT",
    ]

    # Make a copy to avoid mutating the original DataFrame
    df = df.copy()
    # Ensure all SID columns exist; create missing ones filled with 'xx'
    for col in sid_cols:
        if col not in df.columns:
            df[col] = ""

    # Create the concatenated SID column (replace blanks/NaN only here)
    if df.empty:
        df["AARETE_DERIVED_SID"] = pd.Series(dtype=str)
    else:
        sid_df = df[sid_cols].fillna("").astype(str)
        df["AARETE_DERIVED_SID"] = sid_df.apply(
            lambda row: "__".join(val if val.strip() else "xx" for val in row),
            axis=1,
            result_type="reduce",
        )

    return df


def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame:
    """
    Add a derived indicator column to the DataFrame indicating whether all signatory lines are signed.

    Creates column:
        AARETE_DERIVED_SIGNATORY_COMPLETE_IND = 'Y' if
            ((NUM_AVAILABLE_SIGNATORY_LINE_COUNT <= NUM_SIGNED_SIGNATORY_LINE_COUNT) and
            (NUM_SIGNED_SIGNATORY_LINE_COUNT >= 1))
        else 'N'.

    Parameters
    ----------
    df : pd.DataFrame
        Input DataFrame containing the required columns.

    Returns
    -------
    pd.DataFrame
        DataFrame with the new indicator column added.
    """

    try:
        required_cols = [
            "NUM_AVAILABLE_SIGNATORY_LINE_COUNT",
            "NUM_SIGNED_SIGNATORY_LINE_COUNT",
        ]

        # Validate required columns
        for col in required_cols:
            if col not in df.columns:
                return df

        # Ensure consistent numeric types
        for col in required_cols:
            df[col] = pd.to_numeric(df[col], errors="coerce")

        # Create indicator column
        df["AARETE_DERIVED_SIGNATORY_COMPLETE_IND"] = (
            (
                df["NUM_AVAILABLE_SIGNATORY_LINE_COUNT"].le(
                    df["NUM_SIGNED_SIGNATORY_LINE_COUNT"]
                )
            )
            & (df["NUM_SIGNED_SIGNATORY_LINE_COUNT"].ge(1))
        ).map({True: "Y", False: "N"})

        return df

    except Exception:
        # Return df unchanged on any exception
        return df


def standardize_file_name(file_name: str) -> str:
    """
    Standardizes the given file name by applying a series of regex replacements
    to normalize terms related to time periods.

    Args:
        file_name (str): The original file name string.

    Returns:
        str: The standardized file name string.
    """

    # remove .txt from the end if present
    file_name = re.sub(r"\.txt$", "", file_name, flags=re.IGNORECASE)

    return file_name


def fill_claim_type_from_title(df: pd.DataFrame) -> pd.DataFrame:
    """
    Fill empty AARETE_DERIVED_CLAIM_TYPE_CD values using two strategies:
    1. First, fill from other rows in the same file that have populated values (mode within file)
    2. Then, infer from CONTRACT_TITLE using keyword matching

    This is a postprocessing safety net to ensure claim type is populated even if
    earlier extraction steps failed to set it.

    Args:
        df (pd.DataFrame): Input DataFrame with FILE_NAME, CONTRACT_TITLE and AARETE_DERIVED_CLAIM_TYPE_CD columns.

    Returns:
        pd.DataFrame: DataFrame with AARETE_DERIVED_CLAIM_TYPE_CD filled where possible.
    """
    if "AARETE_DERIVED_CLAIM_TYPE_CD" not in df.columns:
        df["AARETE_DERIVED_CLAIM_TYPE_CD"] = ""

    # Step 1: Fill from other rows in the same file (mode within file)
    if "FILE_NAME" in df.columns:
        for file_name in df["FILE_NAME"].unique():
            file_mask = df["FILE_NAME"] == file_name
            file_rows = df.loc[file_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"]

            # Get non-empty values for this file, normalizing lists to strings
            non_empty_values = []
            for v in file_rows:
                if not string_utils.is_empty(v):
                    # Convert lists to their first element if present
                    if isinstance(v, list) and len(v) > 0:
                        non_empty_values.append(v[0])
                    elif not isinstance(v, list):
                        non_empty_values.append(v)

            if non_empty_values:
                # Use mode (most common value) to fill empty rows in this file
                mode_value = max(set(non_empty_values), key=non_empty_values.count)
                empty_mask = file_mask & df["AARETE_DERIVED_CLAIM_TYPE_CD"].apply(
                    string_utils.is_empty
                )
                df.loc[empty_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"] = mode_value

    # Step 2: If still empty, infer from CONTRACT_TITLE
    if "CONTRACT_TITLE" not in df.columns:
        return df

    # Professional indicators -> M
    professional_keywords = [
        "PHYSICIAN",
        "PROFESSIONAL",
        "PROVIDER SERVICE",
        "PROVIDER GROUP",
        "PROVIDER AGREEMENT",
        "PARTICIPATING PROVIDER",
        "MEDICAL GROUP",
        "PRACTITIONER",
    ]

    # Ancillary indicators -> M
    ancillary_keywords = [
        "ANCILLARY",
        "HOME HEALTH",
        "DME",
        "DURABLE MEDICAL",
        "LABORATORY",
        "LAB SERVICE",
        "RADIOLOGY",
        "DIALYSIS",
        "IMAGING",
        "BEHAVIORAL",
        "MENTAL HEALTH",
        "TELEMEDICINE",
        "TELEHEALTH",
        "SURGERY CENTER",
        "AMBULATORY SURGICAL",
        "SKILLED NURSING",
        "REHAB",
    ]

    # Institutional/Hospital/Facility indicators -> H
    institutional_keywords = [
        "HOSPITAL",
        "INSTITUTIONAL",
        "FACILITY",
        "INPATIENT",
    ]

    def infer_from_title(row):
        # If already has a value, keep it
        current_val = row.get("AARETE_DERIVED_CLAIM_TYPE_CD", "")
        if not string_utils.is_empty(current_val):
            return current_val

        title = str(row.get("CONTRACT_TITLE", "")).upper()
        if not title:
            return ""

        # Check Professional keywords
        for kw in professional_keywords:
            if kw in title:
                return "M"

        # Check Ancillary keywords
        for kw in ancillary_keywords:
            if kw in title:
                return "M"

        # Check Institutional keywords
        for kw in institutional_keywords:
            if kw in title:
                return "H"

        return ""

    df["AARETE_DERIVED_CLAIM_TYPE_CD"] = df.apply(infer_from_title, axis=1)

    return df


def _is_empty_list_placeholder(item) -> bool:
    """Return True if item is "[]", "['[]']", or similar empty-list representation."""
    if item is None or (isinstance(item, float) and pd.isna(item)):
        return True
    s = str(item).strip()
    return s in ("[]", "['[]']", '["[]"]') or s == ""


def _strip_wrapping_single_quotes(s: str) -> str:
    """Remove surrounding single quotes so \"'NV'\" becomes \"NV\"."""
    s = s.strip()
    if len(s) >= 2 and s[0] == "'" and s[-1] == "'":
        return s[1:-1]
    return s


def format_as_json_list(val):
    """
    Format a value as a JSON list string. Returns blank for empty lists
    instead of "[]" to avoid clutter in output.
    """
    # 1. Handle actual lists or Nulls
    if isinstance(val, list):
        # Flatten nested lists before processing
        flattened = []
        for i in val:
            if i is None or _is_empty_list_placeholder(i):
                continue
            # If item is a list itself, extend (flatten)
            if isinstance(i, list):
                flattened.extend(
                    [
                        _strip_wrapping_single_quotes(str(x).strip())
                        for x in i
                        if x is not None and not _is_empty_list_placeholder(x)
                    ]
                )
            else:
                flattened.append(_strip_wrapping_single_quotes(str(i).strip()))

        return json.dumps(flattened) if flattened else ""
    if pd.isna(val) or val == "" or val == "[]":
        return ""

    # 2. Cleanup & Parsing
    text = str(val).strip()

    # Try to parse as JSON if it looks like a list
    if text.startswith("[") and text.endswith("]"):
        try:
            parsed = json.loads(text)
            if isinstance(parsed, list):
                filtered = [
                    _strip_wrapping_single_quotes(str(i).strip())
                    for i in parsed
                    if i is not None and not _is_empty_list_placeholder(i)
                ]
                return json.dumps(filtered) if filtered else ""
        except (json.JSONDecodeError, ValueError):
            text = text.strip("[]")  # Strip brackets if it's a "fake" list like ['A']

    # 3. Pipe-delimited string (e.g. CHIP|MMC from crosswalk/derived fields)
    if "|" in text:
        items = [
            _strip_wrapping_single_quotes(i.strip())
            for i in text.split("|")
            if i.strip()
        ]
        return json.dumps(items) if items else ""

    # 4. Handle comma-separated or single string; keep only letters, digits, comma,
    # space, apostrophe, hyphen (e.g. "Children's / Medicaid-Medicare (MM)" ->
    # "Children's Medicaid-Medicare MM")
    clean_text = re.sub(r"[^a-zA-Z0-9, ' -]+", "", text)
    items = [i.strip() for i in clean_text.split(",") if i.strip()]
    items = [
        _strip_wrapping_single_quotes(re.sub(r"\s+", " ", i).strip())
        for i in items
        if i
    ]

    return json.dumps(items) if items else ""


def format_prov_info_json(val) -> str:
    """
    Format PROV_INFO_JSON cell as a valid JSON string (list of objects).

    Delegates to json_utils.format_prov_info_json after normalizing pandas NaN.
    Returns "[]" for empty; otherwise valid JSON string.
    """
    if isinstance(val, float) and pd.isna(val):
        val = None
    return json_utils.format_prov_info_json(val)