From f412754167c414d4f7697804b5fe0e792fc00957 Mon Sep 17 00:00:00 2001 From: Siddhant Medar Date: Wed, 28 Jan 2026 20:18:11 +0000 Subject: [PATCH] Merged in feature/generic-parent-child (pull request #852) Feature/generic parent child * Refactor parent_child module with generic hierarchy lineage - Add generic_hierarchy_lineage.py for core parent-child mapping - Add generic_hierarchy_lineage_qc.py for QC validation engine - Add bcbs_generic_hierarchy_lineage.py for BCBS-specific processing - Add column_mapper.py for automatic column detection - Rename parent_child_preprocessing.py to generic_hierarchy_lineage_preprocessing.py - Remove old parent_child_mapping.py (replaced by generic_hierarchy_lineage.py) - Fix import paths from src.generic_hierarchy_lineage to src.parent_child - Replace hardcoded BCBS xwalk path with configurable parameter - Remove global warning suppression * Fix code review issues in parent_child module - Fix import paths from src.generic_hierarchy_lineage to src.parent_child - Replace hardcoded BCBS crosswalk path with configurable xwalk_path param - Remove global warnings.filterwarnings('ignore') suppressions - Change bare except clauses to except Exception with logging - Add explicit client= CLI argument with backward compatible fallback - Convert print statements to logging in bcbs_generic_hierarchy_lineage.py * Add tests for parent_child module and fix JSON path resolution - Add 55 unit tests covering grouping keys, parent identification, child ranking, and QC utilities - Use pathlib to resolve JSON config paths relative to module location, allowing tests to run from any directory * WIP: Refactor parent_child module structure * Extract constants to centralized location for parent_child module - Create src/constants/parent_child/ with generic.py and bcbs.py - Update pipeline.py to use centralized constants (ASSIGNMENT_NO_PARENT, tier values, grouping key config) - Update bcbsnc/mapping.py to use BCBS-specific constants - Update imports in __main__.py and bcbsnc/__init__.py - Pass client parameter from runner.py and saas/main.py * Fix code formatting in parent_child module * Merge remote-tracking branch 'origin/main' into feature/generic-parent-child * Fix client detection and output path in parent_child module * Fix typo in numeric_mappings.json filename and update logging * Add type hints to parent_child module * Merged main into feature/generic-parent-child Approved-by: Katon Minhas --- ...ric_mappigs.json => numeric_mappings.json} | 0 src/constants/parent_child/__init__.py | 25 + src/constants/parent_child/bcbs.py | 103 + src/constants/parent_child/generic.py | 72 + src/parent_child/__init__.py | 18 + src/parent_child/__main__.py | 180 ++ src/parent_child/bcbsnc/__init__.py | 5 + src/parent_child/bcbsnc/mapping.py | 1035 ++++++++ src/parent_child/column_mapper.py | 266 ++ src/parent_child/main.py | 56 - src/parent_child/parent_child_mapping.py | 890 ------- src/parent_child/pipeline.py | 1127 +++++++++ ...hild_preprocessing.py => preprocessing.py} | 408 ++- src/parent_child/qc.py | 2251 +++++++++++++++++ src/pipelines/runner.py | 2 +- src/pipelines/saas/main.py | 9 +- src/tests/test_parent_child.py | 641 +++++ 17 files changed, 6080 insertions(+), 1008 deletions(-) rename src/constants/mappings/{numeric_mappigs.json => numeric_mappings.json} (100%) create mode 100644 src/constants/parent_child/__init__.py create mode 100644 src/constants/parent_child/bcbs.py create mode 100644 src/constants/parent_child/generic.py create mode 100644 src/parent_child/__main__.py create mode 100644 src/parent_child/bcbsnc/__init__.py create mode 100644 src/parent_child/bcbsnc/mapping.py create mode 100644 src/parent_child/column_mapper.py delete mode 100644 src/parent_child/main.py delete mode 100644 src/parent_child/parent_child_mapping.py create mode 100644 src/parent_child/pipeline.py rename src/parent_child/{parent_child_preprocessing.py => preprocessing.py} (53%) create mode 100644 src/parent_child/qc.py create mode 100644 src/tests/test_parent_child.py diff --git a/src/constants/mappings/numeric_mappigs.json b/src/constants/mappings/numeric_mappings.json similarity index 100% rename from src/constants/mappings/numeric_mappigs.json rename to src/constants/mappings/numeric_mappings.json diff --git a/src/constants/parent_child/__init__.py b/src/constants/parent_child/__init__.py new file mode 100644 index 0000000..3cce00e --- /dev/null +++ b/src/constants/parent_child/__init__.py @@ -0,0 +1,25 @@ +"""Parent-child module constants.""" + +from src.constants.parent_child.generic import ( + GROUPING_KEY_SEPARATOR, + GROUPING_KEY_PREFIXES, + TIER_ALL_IDENTIFIERS, + TIER_TWO_IDENTIFIERS, + TIER_ONE_IDENTIFIER, + TIER_NO_IDENTIFIERS, + ASSIGNMENT_NO_PARENT, + DATE_FORMAT, + LEGAL_SUFFIXES, +) + +__all__ = [ + "GROUPING_KEY_SEPARATOR", + "GROUPING_KEY_PREFIXES", + "TIER_ALL_IDENTIFIERS", + "TIER_TWO_IDENTIFIERS", + "TIER_ONE_IDENTIFIER", + "TIER_NO_IDENTIFIERS", + "ASSIGNMENT_NO_PARENT", + "DATE_FORMAT", + "LEGAL_SUFFIXES", +] diff --git a/src/constants/parent_child/bcbs.py b/src/constants/parent_child/bcbs.py new file mode 100644 index 0000000..f3d20b5 --- /dev/null +++ b/src/constants/parent_child/bcbs.py @@ -0,0 +1,103 @@ +""" +BCBS-specific constants for parent-child module. + +These constants are used by the BCBSNC-specific mapping logic. +""" + +# Crosswalk configuration +XWALK_SHEET_NAME = "File Name to System Crosswalk" + +# Column Names - Original (from source data) +COL_CONTRACT_NAME_ORIG = "File Name" +COL_AGREEMENT_NAME_ORIG = "Contract Name" +COL_MULTIPLE_IRS_ORIG = "Multiple IRS Names" +COL_PAGE_COUNT_ORIG = "Total Page Count" +COL_LOB_ORIG = "Line of Business" +COL_PROVIDER_TYPE_ORIG = "Provider Type" +COL_PROVIDER_TYPE_L2_ORIG = "Provider Type - Level 2" +COL_CONTRACT_EFF_DATE = "Contract Effective Date" + +# Column Names - Standardized (after processing) +COL_CONTRACT_NAME = "Contract Name" +COL_AGREEMENT_NAME = "Agreement_Name (Contract Title)" +COL_MULTIPLE_IRS = "MULTIPLE_IRS_NAMES" +COL_PAGES = "Pages" +COL_LOB = "Line of Business" +COL_CONSOLIDATED_LOB = "consolidated_line_of_business" +COL_CONSOLIDATED_PROVIDER = "consolidated_provider_type" +COL_FIXED_CONTRACT_NAME = "fixed_contract_name" +COL_IRS_NAME = "irs_name" +COL_FOLDER = "Folder" +COL_IRS_GROUP = "IRS_Group" +COL_CONTRACT_TITLE_CLEAN = "contract_title_clean" +COL_FINAL_EFF_DATE = "final_effective_date" +COL_EXTRACTED_LOB = "extracted_lob" +COL_IS_PARENT = "is_parent" +COL_PARENT_CHILD_FLAG = "parent_child_flag" +COL_COMBINED_RANK = "combined_rank" +COL_NAMING_COL = "naming_col" +COL_ASSIGNED_PARENT = "assigned_parent" +COL_PROVIDER_TYPE_COUNT = "provider_type_count" +COL_EFF_DATE_RANK = "effective_date_rank" +COL_FINAL_RANK = "final_rank" +COL_CHILD_RANK = "child_rank" +COL_CHILD_INDEX = "child_index" + +# Crosswalk columns +XWALK_COL_FILE_NAME = "File Name" +XWALK_COL_FOLDER = "Folder" + +# Fields to consolidate +FIELDS_TO_CONSOLIDATE = ["Line of Business", "Provider Type"] + +# File extensions regex pattern +FILE_EXTENSIONS_PATTERN = r"\.(pdf|txt|msg|xlsx|docx|tif|mht|doc|htm|xps)$" + +# LOB Search Terms +LOB_SEARCH_TERMS = [ + "MCR", + "Medicare", + "MCD", + "Medicaid", + "COMM", + "Commercial", + "Commercial Exchange", + "MA", + "Medicare Advantage", +] + +# LOB Mapping Dictionary +LOB_MAPPING = { + "medicare": "mcr", + "medicare advantage": "mcr", + "ma": "mcr", + "medicaid": "mcd", + "commercial": "comm", +} + +# Non-parent Keywords (contracts with these are children) +NON_PARENT_KEYWORDS = [ + "tracking sheet", + "letter", + "notice", + "memo", + "form", + "term sheet", + "request", +] + +# Date extraction regex patterns +DATE_PATTERNS = [ + r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", # 3-27-2015, 10/15/14 + r"(\d{1,2}[.]\d{1,2}[.]\d{2,4})", # 10.01.24, 07.01.2018 + r"(\d{1,2}[_]\d{1,2}[_]\d{2,4})", # 7_1_17, 01_01_2022 + r"(\d{1,2}[~]\d{1,2}[~]\d{2,4})", # 08~01~18 + r"(\d{1,2}\s+\d{1,2}\s+\d{2,4})", # 3 27 2015 + r"(\d{4}\s+\d{4})", # 0415 0515 (month-year chunks) + r"(\d{8})", # 01012025 (MMDDYYYY) + r"(\d{6})", # 120618 (MMDDYY) + r"([A-Za-z]+ ?\d{1,2},?\s*\d{2,4})", # June1, 2016 +] + +# Special IRS Group Replacements +IRS_GROUP_REPLACEMENTS = {"cape": "cumberland"} diff --git a/src/constants/parent_child/generic.py b/src/constants/parent_child/generic.py new file mode 100644 index 0000000..7bbc815 --- /dev/null +++ b/src/constants/parent_child/generic.py @@ -0,0 +1,72 @@ +""" +Generic constants for parent-child module. + +These constants are used across the generic pipeline (non-client-specific). +""" + +# Grouping key configuration +GROUPING_KEY_SEPARATOR = "|" +GROUPING_KEY_PREFIXES = { + "TIN": "TIN:", + "NPI": "NPI:", + "NAME": "NAME:", +} + +# Tier levels (based on number of identifiers present) +TIER_ALL_IDENTIFIERS = 1 # All 3: TIN + NPI + NAME +TIER_TWO_IDENTIFIERS = 2 # Any 2 identifiers +TIER_ONE_IDENTIFIER = 3 # Only 1 identifier +TIER_NO_IDENTIFIERS = 0 # No identifiers (immediate orphan) + +# Assignment status +ASSIGNMENT_NO_PARENT = "no_parent" + +# Date format +DATE_FORMAT = "%m/%d/%Y" + +# Legal entity suffixes to remove during cleaning +LEGAL_SUFFIXES = [ + "inc", + "inc.", + "llc", + "l.l.c.", + "corp", + "corp.", + "co", + "co.", + "ltd", + "ltd.", + "lp", + "llp", + "pty", + "company", + "corporation", + "incorporated", + "limited", + "dba", + "d/b/a", +] + +# Pluralization rules for text normalization +PLURAL_TO_SINGULAR = { + "centers": "center", + "plans": "plan", + "services": "service", + "systems": "system", + "solutions": "solution", + "associates": "associate", +} + +# File extensions to remove from contract names +FILE_EXTENSIONS = [ + ".txt", + ".pdf", + ".msg", + ".xlsx", + ".docx", + ".tif", + ".mht", + ".doc", + ".htm", + ".xps", +] diff --git a/src/parent_child/__init__.py b/src/parent_child/__init__.py index e69de29..9b5e05b 100644 --- a/src/parent_child/__init__.py +++ b/src/parent_child/__init__.py @@ -0,0 +1,18 @@ +""" +Parent-Child Contract Hierarchy Module. + +Public API: + main - Main module (backward compatible) + ColumnMapper - Auto-detection of column names +""" + +# Re-export main module for backward compatibility +# This allows: from src.parent_child import main +from src.parent_child import __main__ as main + +from src.parent_child.column_mapper import ColumnMapper + +__all__ = [ + "main", + "ColumnMapper", +] diff --git a/src/parent_child/__main__.py b/src/parent_child/__main__.py new file mode 100644 index 0000000..1ef203a --- /dev/null +++ b/src/parent_child/__main__.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +from src.parent_child.qc import qc_main +import src.config as config +from src.config import get_arg_value +from src.parent_child.preprocessing import ( + parent_child_preprocessing, +) +from src.parent_child.pipeline import parent_child_mapping +from src.parent_child.column_mapper import ColumnMapper +from src.parent_child.bcbsnc import bcbs_main +import os +import re +import pandas as pd +from src.utils import io_utils +import logging +from datetime import datetime +from typing import Optional + +# Optional command-line arguments +BCBSNC_XWALK_PATH = get_arg_value("bcbsnc_xwalk_path", None) +# Explicit client override (takes precedence over string matching) +# Supported values: bcbs, caresource, molina, cnc, clover_health +CLIENT_ARG = get_arg_value("client", None) + + +def extract_client_name(file_path: str) -> str: + """ + Extract client name from input file path for output naming. + + Args: + file_path: Input file path (e.g., 'Clover Health_132.xlsx' or full path) + + Returns: + Cleaned client name (e.g., 'clover_health') + """ + # Get just the filename without extension + filename = os.path.basename(file_path) + name_without_ext = os.path.splitext(filename)[0] + + # Remove trailing numbers and underscores (e.g., '_132' from 'Clover Health_132') + cleaned_name = re.sub(r"[_\s]*\d+$", "", name_without_ext) + + # Convert to lowercase and replace spaces with underscores + cleaned_name = cleaned_name.lower().strip().replace(" ", "_") + + # Remove any double underscores + cleaned_name = re.sub(r"_+", "_", cleaned_name) + + return cleaned_name if cleaned_name else "output" + + +def write_pc_excel( + mapped_df: pd.DataFrame, + pc_df: Optional[pd.DataFrame], + output_dir: str, + output_name: str, +) -> str: + """ + Write parent-child mapping results to Excel with multiple sheets. + + Args: + mapped_df: Main dataframe with PC flags merged back + pc_df: Detailed PC mapping dataframe + output_dir: Directory to write to + output_name: Client-based output name for filename + """ + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join( + output_dir, f"{output_name}_generic_hierarchy_lineage_output.xlsx" + ) + + with pd.ExcelWriter(output_path, engine="openpyxl") as writer: + mapped_df.to_excel(writer, sheet_name="Main_Output", index=False) + if pc_df is not None: + pc_df.to_excel(writer, sheet_name="PC_Details", index=False) + + logging.info(f"Excel file written to: {output_path}") + return output_path + + +def main(doczy_output_for_pc: str, client: Optional[str] = None) -> int: + """ + Main entry point for parent-child mapping. + + Args: + doczy_output_for_pc: Path to input file (local or S3) + client: Optional client identifier. If provided, takes precedence over + CLI argument and path-based detection. + Supported values: bcbs, caresource, molina, cnc, clover_health + + Returns: + Number of rows processed + """ + # Extract client name from input file for output naming + client_name = extract_client_name(doczy_output_for_pc) + + # Generate timestamp for this run + run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{client_name}" + + try: + # Read input file + df_read = io_utils.read_DataFrame(doczy_output_for_pc) + logging.info(f"Original shape: {df_read.shape}") + logging.info(f"Original columns: {df_read.columns.tolist()}") + + # Determine client: function param > CLI arg > path-based detection + if client: + client = client.lower() + logging.info(f"Client specified via parameter: {client}") + elif CLIENT_ARG: + client = CLIENT_ARG.lower() + logging.info(f"Client specified via CLI argument: {client}") + else: + # Fallback: detect client from file path (backward compatibility) + path_lower = doczy_output_for_pc.lower() + if "bcbsnc" in path_lower: + client = "bcbsnc" + elif "caresource" in path_lower: + client = "caresource" + else: + client = "molina" + logging.info(f"Client detected from path: {client}") + + # BCBS-specific processing + if client == "bcbsnc": + logging.info("Using BCBSNC specialized reader.") + mapped_df = df_read + pc_df = bcbs_main(df_read, xwalk_path=BCBSNC_XWALK_PATH) + else: + # Auto-detect column mappings + logging.info("=" * 80) + logging.info("Starting automatic column detection...") + logging.info("=" * 80) + col_mapper = ColumnMapper(df_read) + col_mapper.log_mapping_summary() + + # Preprocess with column mapper - returns cleaned_df and original_df + cleaned_df, original_df = parent_child_preprocessing(df_read, col_mapper) + logging.info( + f"Unique contracts in the output: {cleaned_df['FILE_NAME'].nunique()}" + ) + + # Perform parent-child mapping - returns original_df_with_pc and pc_df + original_df_with_pc, pc_df = parent_child_mapping(cleaned_df, original_df) + + # Save detailed results + output_dir = os.path.join("outputs/parent_child", run_timestamp) + pc_df1 = qc_main(pc_df, client) + + output_filename = f"{client_name}_generic_hierarchy_lineage_output.xlsx" + if config.WRITE_PC_TO_S3: + # Write locally first, then upload to S3 + write_pc_excel(df_read, pc_df1, output_dir, client_name) + # TODO: Add S3 upload logic if needed + logging.info(f"PC results saved: {run_timestamp}/{output_filename}") + else: + write_pc_excel(df_read, pc_df1, output_dir, client_name) + logging.info(f"PC results saved locally: {output_dir}/{output_filename}") + + # TODO: check with dev + return pc_df.shape[0] if pc_df is not None else mapped_df.shape[0] + except Exception as e: + logging.error(f"Parent child mapping failed due to {e}", exc_info=True) + raise + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + # Generate timestamp for this run + run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}" + + if config.DOCZY_OUTPUT_FOR_PC: + results = main(config.DOCZY_OUTPUT_FOR_PC) + logging.info(f"PC Mapping complete. Processed {results} files.") + else: + logging.warning("No input file specified in config.DOCZY_OUTPUT_FOR_PC") diff --git a/src/parent_child/bcbsnc/__init__.py b/src/parent_child/bcbsnc/__init__.py new file mode 100644 index 0000000..07a6150 --- /dev/null +++ b/src/parent_child/bcbsnc/__init__.py @@ -0,0 +1,5 @@ +"""BCBSNC-specific parent-child processing.""" + +from src.parent_child.bcbsnc.mapping import bcbs_main + +__all__ = ["bcbs_main"] diff --git a/src/parent_child/bcbsnc/mapping.py b/src/parent_child/bcbsnc/mapping.py new file mode 100644 index 0000000..f31c4c0 --- /dev/null +++ b/src/parent_child/bcbsnc/mapping.py @@ -0,0 +1,1035 @@ +""" +BCBS Parent-Child Mapping at Scale +Converts notebook logic to production Python script with centralized constants. +""" + +import os +import logging +import pandas as pd +import numpy as np +import re +import string +from collections import defaultdict +import warnings +from datetime import datetime +from dateutil import parser + +from src.constants.parent_child.bcbs import ( + XWALK_SHEET_NAME, + COL_CONTRACT_NAME_ORIG, + COL_AGREEMENT_NAME_ORIG, + COL_MULTIPLE_IRS_ORIG, + COL_PAGE_COUNT_ORIG, + COL_LOB_ORIG, + COL_PROVIDER_TYPE_ORIG, + COL_PROVIDER_TYPE_L2_ORIG, + COL_CONTRACT_EFF_DATE, + COL_CONTRACT_NAME, + COL_AGREEMENT_NAME, + COL_MULTIPLE_IRS, + COL_PAGES, + COL_LOB, + COL_CONSOLIDATED_LOB, + COL_CONSOLIDATED_PROVIDER, + COL_FIXED_CONTRACT_NAME, + COL_IRS_NAME, + COL_FOLDER, + COL_IRS_GROUP, + COL_CONTRACT_TITLE_CLEAN, + COL_FINAL_EFF_DATE, + COL_EXTRACTED_LOB, + COL_IS_PARENT, + COL_PARENT_CHILD_FLAG, + COL_COMBINED_RANK, + COL_NAMING_COL, + COL_ASSIGNED_PARENT, + COL_PROVIDER_TYPE_COUNT, + COL_EFF_DATE_RANK, + COL_FINAL_RANK, + COL_CHILD_RANK, + COL_CHILD_INDEX, + XWALK_COL_FILE_NAME, + XWALK_COL_FOLDER, + FIELDS_TO_CONSOLIDATE, + FILE_EXTENSIONS_PATTERN, + LOB_SEARCH_TERMS, + LOB_MAPPING, + NON_PARENT_KEYWORDS, + DATE_PATTERNS, + IRS_GROUP_REPLACEMENTS, +) + +logger = logging.getLogger(__name__) + +# Default fallback path - relative to this module's location +_DEFAULT_XWALK_PATH = os.path.join( + os.path.dirname(__file__), "BCBSNC_Folder_Xwalk.xlsx" +) + +# Alias for backward compatibility +FILE_EXTENSIONS = FILE_EXTENSIONS_PATTERN + +# Output columns (final export) +OUTPUT_COLUMNS = [ + COL_CONTRACT_NAME, + COL_FIXED_CONTRACT_NAME, + COL_CONTRACT_TITLE_CLEAN, + "raw_extract", + "fixed_raw_extract", + COL_FINAL_EFF_DATE, + COL_FOLDER, + COL_IRS_GROUP, + COL_CONSOLIDATED_PROVIDER, + COL_EXTRACTED_LOB, + COL_ASSIGNED_PARENT, + COL_IS_PARENT, + COL_PARENT_CHILD_FLAG, + COL_COMBINED_RANK, + COL_NAMING_COL, + COL_LOB, + COL_PROVIDER_TYPE_ORIG, + COL_CONTRACT_EFF_DATE, + "File Name", + "Provider Type_Consolidated", +] + +# ============================================================================ +# NLTK Setup +# ============================================================================ + +import nltk +import logging + +try: + nltk.download("stopwords", quiet=True) + from nltk.corpus import stopwords + + STOP_WORDS = stopwords.words("english") +except Exception as e: + STOP_WORDS = [] + logging.warning(f"NLTK stopwords not available: {e}") + +# ============================================================================ +# Configuration +# ============================================================================ + +pd.set_option("display.max_colwidth", None) +warnings.filterwarnings("ignore") + + +# ============================================================================ +# UTILITY FUNCTIONS +# ============================================================================ + + +def consolidate_B_fields(contract_name_col, df, fields_to_consolidate): + """ + Consolidate specified fields for each contract into unique, lowercased lists. + + Parameters + ---------- + contract_name_col : str + Name of the contract column to group by + df : pandas.DataFrame + DataFrame containing contract data + fields_to_consolidate : list of str + List of column names to consolidate + + Returns + ------- + pandas.DataFrame + DataFrame with consolidated columns + """ + agg_dict = { + field: (lambda x: [", ".join(x.dropna().astype(str))]) + for field in fields_to_consolidate + } + grouped = df.groupby(contract_name_col).agg(agg_dict).reset_index() + + for field in fields_to_consolidate: + new_col = f'consolidated_{field.lower().replace(" ", "_")}' + grouped[new_col] = grouped[field].apply( + lambda x: ", ".join( + sorted( + set( + item.strip().lower() + for sublist in x + for item in sublist.split(",") + if item.strip() != "" and item.strip().lower() != "nan" + ) + ) + ) + ) + grouped = grouped.drop(columns=[field]) + + cols = [contract_name_col] + [ + f'consolidated_{field.lower().replace(" ", "_")}' + for field in fields_to_consolidate + ] + return grouped[cols] + + +def remove_blue_cross_blue_shield(text): + """Remove Blue Cross Blue Shield references from text""" + if pd.isna(text): + return text + pattern = re.compile( + r""" + (blue\s*cross(\s*and\s*blue\s*shield)? + |blue\s*shield) + (\s*of)? + (\s*north\s*carolina|\s*nc)? + ,?\s* + """, + flags=re.IGNORECASE | re.VERBOSE, + ) + new_text = pattern.sub("", text) + return new_text.strip() + + +def clean_prefix(name): + """Clean name prefix before inc/llc or d/b/a""" + if pd.isna(name): + return None + name = str(name).lower().strip() + + inc_pos = name.find("inc") + llc_pos = name.find("llc") + + if inc_pos != -1 or llc_pos != -1: + cutoff_pos = min(pos for pos in [inc_pos, llc_pos] if pos != -1) + return name[:cutoff_pos].strip().rstrip(",") + elif "d/b/a" in name: + return re.sub(r",?\s+d/b/a.*$", "", name).strip() + else: + return name + + +def preprocess_irs(name): + """Remove common suffixes from IRS name""" + name = str(name).lower() + suffixes = ["inc", "ltd", "llc", "corp", "corporation", "incorporated"] + for s in suffixes: + name = name.replace(s, "") + return " ".join(name.split()) + + +def remove_bcbsnc(text): + """Remove BCBSNC references""" + if pd.isna(text): + return text + return re.sub(r"bcbsnc,?\s*", "", text, flags=re.IGNORECASE).strip() + + +def clean_string(text): + """Clean string to alphanumeric with single spaces""" + cleaned_text = re.sub(r"[^a-zA-Z0-9]", " ", text) + cleaned_text = re.sub(r"\s+", " ", cleaned_text) + cleaned_text = cleaned_text.strip() + return cleaned_text + + +def extract_date(text): + """Extract first matching date pattern from text""" + if pd.isna(text) or not text: + return None + text = str(text) + for pattern in DATE_PATTERNS: + match = re.search(pattern, text) + if match: + return match.group(0) + return None + + +def parse_date(date_str): + """Parse extracted date string into standard format""" + if not date_str: + return None + try: + # Case: MMYY MMYY (0415 0515) + if re.fullmatch(r"\d{4}\s+\d{4}", date_str): + parts = date_str.split() + results = [] + for part in parts: + mm, yy = int(part[:2]), int(part[2:]) + yyyy = 2000 + yy if yy < 50 else 1900 + yy + dt = datetime(yyyy, mm, 1) + results.append(dt.strftime("%m/%d/%Y")) + return " & ".join(results) + + # Case: MMDDYY + if re.fullmatch(r"\d{6}", date_str): + mm, dd, yy = int(date_str[:2]), int(date_str[2:4]), int(date_str[4:6]) + yyyy = 2000 + yy if yy < 50 else 1900 + yy + dt = datetime(yyyy, mm, dd) + return dt.strftime("%m/%d/%Y") + + # Case: MMDDYYYY + if re.fullmatch(r"\d{8}", date_str): + mm, dd, yyyy = int(date_str[:2]), int(date_str[2:4]), int(date_str[4:8]) + dt = datetime(yyyy, mm, dd) + return dt.strftime("%m/%d/%Y") + + # Normalize separators + clean_date = re.sub(r"[-._~]", "/", date_str) + dt = parser.parse(clean_date, dayfirst=False, yearfirst=False) + return dt.strftime("%m/%d/%Y") + + except Exception: + return None + + +def clean_and_parse(val): + """Clean and parse date value into datetime""" + try: + cleaned = str(val) + cleaned = re.sub(r"[\u200b\r\n\x00]", "", cleaned).strip() + return pd.to_datetime(cleaned, errors="coerce") + except Exception: + return pd.NaT + + +def clean_text(text): + """Remove non-alphabetic characters""" + cleaned_text = re.sub(r"[^a-zA-Z\s]", " ", text) + cleaned_text = re.sub(r"\s+", " ", cleaned_text).strip() + return cleaned_text + + +def clean_line_of_business_column(df, col_name=COL_LOB): + """Clean and deduplicate LOB column""" + if col_name in df.columns: + + def clean_lob(val): + if pd.isna(val): + return val + items = [item.strip().lower() for item in str(val).split(",")] + unique_items = sorted(set(items)) + return ", ".join(unique_items) + + df[col_name] = df[col_name].apply(clean_lob) + return df + + +def map_comma_separated_values(df, col_name, mapping_dict): + """Map comma-separated values using dictionary""" + if col_name in df.columns: + + def map_values(val): + if pd.isna(val): + return val + items = [item.strip().lower() for item in str(val).split(",")] + mapped = [mapping_dict.get(item, item) for item in items] + return ", ".join(mapped) + + df[col_name] = df[col_name].apply(map_values) + return df + + +def add_flag_if_words_present(df, cols_to_search, words, flag_col="flag"): + """Add boolean flag if any word is present in specified columns""" + pattern = "|".join([rf"\b{w}\b" for w in words]) + df[flag_col] = ( + df[cols_to_search] + .astype(str) + .apply( + lambda row: row.str.contains(pattern, case=False, regex=True).any(), axis=1 + ) + ) + return df + + +def replace_single_counts(df, group_col, value_col): + """Replace values with count 1 with most frequent value in group""" + value_counts = ( + df.groupby(group_col)[value_col].value_counts().rename("count").reset_index() + ) + single_counts = value_counts[value_counts["count"] == 1] + most_frequent = value_counts.loc[value_counts.groupby(group_col)["count"].idxmax()] + most_frequent = most_frequent.set_index(group_col)[value_col].to_dict() + + replace_map = {} + for _, row in single_counts.iterrows(): + group = row[group_col] + old_value = row[value_col] + if group in most_frequent: + replace_map[(group, old_value)] = most_frequent[group] + + for (group, old_val), new_val in replace_map.items(): + df.loc[(df[group_col] == group) & (df[value_col] == old_val), value_col] = ( + new_val + ) + + return df + + +def is_missing_lob(lob): + """Check if LOB value is missing""" + return pd.isnull(lob) or str(lob).strip().upper() in ["NA", ""] + + +# ============================================================================ +# PARENT-CHILD ASSIGNMENT FUNCTIONS +# ============================================================================ + + +def assign_parents(df): + """Assign each child to a parent within same Folder + IRS_Group""" + df[COL_ASSIGNED_PARENT] = None + + for (folder, group_id), group in df.groupby([COL_FOLDER, COL_IRS_GROUP]): + parents = group[group[COL_IS_PARENT]].sort_values(COL_FINAL_EFF_DATE) + children = group[~group[COL_IS_PARENT]].sort_values(COL_FINAL_EFF_DATE) + + for _, child in children.iterrows(): + candidate = None + child_title = str(child[COL_CONTRACT_TITLE_CLEAN]).lower().strip() + logger.debug( + f"Evaluating child: '{child[COL_FIXED_CONTRACT_NAME]}' | effective date: {child[COL_FINAL_EFF_DATE]} | LOB: {child[COL_EXTRACTED_LOB]}" + ) + + # Rule 0: Professional logic + if "professional" in child_title: + # 0a. Parent title fully contained in child title + subset_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & ( + parents[COL_CONTRACT_TITLE_CLEAN].apply( + lambda pt: str(pt).lower().strip() in child_title + and len(str(pt).strip()) > 0 + ) + ) + ].sort_values(COL_FINAL_EFF_DATE, ascending=False) + if not subset_matches.empty: + candidate = subset_matches.iloc[0] + logger.debug( + f"Assigned by PROFESSIONAL subset match to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # 0b. Parent title contains "professional" + if candidate is None: + prof_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & ( + parents[COL_CONTRACT_TITLE_CLEAN].str.contains( + "professional", case=False, na=False + ) + ) + ].sort_values(COL_FINAL_EFF_DATE, ascending=False) + if not prof_matches.empty: + candidate = prof_matches.iloc[0] + logger.debug( + f"Assigned by PROFESSIONAL keyword to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Rule 1: LOB + Provider Type match + if ( + candidate is None + and not is_missing_lob(child[COL_EXTRACTED_LOB]) + and pd.notna(child[COL_CONSOLIDATED_PROVIDER]) + and str(child[COL_CONSOLIDATED_PROVIDER]).strip() != "" + ): + lob_pt_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & (parents[COL_EXTRACTED_LOB] == child[COL_EXTRACTED_LOB]) + & (parents[COL_CONSOLIDATED_PROVIDER].notna()) + & (parents[COL_CONSOLIDATED_PROVIDER].str.strip() != "") + & ( + parents[COL_CONSOLIDATED_PROVIDER] + == child[COL_CONSOLIDATED_PROVIDER] + ) + ].sort_values(COL_FINAL_EFF_DATE, ascending=False) + + if not lob_pt_matches.empty: + candidate = lob_pt_matches.iloc[0] + logger.debug( + f"Assigned by LOB + Provider Type match to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Rule 2: Title substring match + if candidate is None and is_missing_lob(child[COL_EXTRACTED_LOB]): + title_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & ( + parents.apply( + lambda p: str(p[COL_CONTRACT_TITLE_CLEAN]).lower().strip() + in child_title, + axis=1, + ) + ) + ].sort_values(COL_FINAL_EFF_DATE, ascending=False) + if not title_matches.empty: + candidate = title_matches.iloc[0] + logger.debug( + f"Assigned by title match to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Rule 3: Consolidated Provider Type logic + if candidate is None: + if ( + not pd.isna(child[COL_CONSOLIDATED_PROVIDER]) + and str(child[COL_CONSOLIDATED_PROVIDER]).strip() != "" + ): + provider_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & (parents[COL_EXTRACTED_LOB] == child[COL_EXTRACTED_LOB]) + & ( + parents[COL_CONSOLIDATED_PROVIDER] + == child[COL_CONSOLIDATED_PROVIDER] + ) + ].sort_values(COL_FINAL_EFF_DATE, ascending=False) + if not provider_matches.empty: + candidate = provider_matches.iloc[0] + logger.debug( + f"Assigned by Provider Type match to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + else: + # Fallback to max provider_type_count + eligible_parents = parents[ + parents[COL_EXTRACTED_LOB] == child[COL_EXTRACTED_LOB] + ] + if not pd.isna(child[COL_FINAL_EFF_DATE]): + eligible_parents = eligible_parents[ + eligible_parents[COL_FINAL_EFF_DATE] + <= child[COL_FINAL_EFF_DATE] + ] + + if not eligible_parents.empty: + max_count = eligible_parents[COL_PROVIDER_TYPE_COUNT].max() + top_candidates = eligible_parents[ + eligible_parents[COL_PROVIDER_TYPE_COUNT] == max_count + ] + + if ( + top_candidates[COL_CONSOLIDATED_PROVIDER] == "facility" + ).any(): + facility_candidates = top_candidates[ + top_candidates[COL_CONSOLIDATED_PROVIDER] == "facility" + ] + candidate = facility_candidates.sort_values( + COL_FINAL_EFF_DATE, ascending=False + ).iloc[0] + else: + candidate = top_candidates.sort_values( + COL_FINAL_EFF_DATE, ascending=False + ).iloc[0] + + logger.debug( + f"Assigned by Provider Type Count fallback to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Rule 4: Latest eligible COMMERCIAL parent + if candidate is None: + if pd.isna(child[COL_FINAL_EFF_DATE]): + comm_matches = parents[ + parents[COL_EXTRACTED_LOB].str.contains( + "comm", case=False, na=False + ) + ] + else: + comm_matches = parents[ + (parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE]) + & ( + parents[COL_EXTRACTED_LOB].str.contains( + "comm", case=False, na=False + ) + ) + ] + comm_matches = comm_matches.sort_values( + COL_FINAL_EFF_DATE, ascending=False + ) + if not comm_matches.empty: + candidate = comm_matches.iloc[0] + logger.debug( + f"Assigned by commercial fallback (Latest) to parent: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Rule 5: Fallback to Latest parent + if candidate is None: + if pd.isna(child[COL_FINAL_EFF_DATE]): + eligible = parents + else: + eligible = parents[ + parents[COL_FINAL_EFF_DATE] <= child[COL_FINAL_EFF_DATE] + ] + eligible = eligible.sort_values(COL_FINAL_EFF_DATE, ascending=False) + if not eligible.empty: + candidate = eligible.iloc[0] + logger.debug( + f"Assigned by Latest parent fallback: '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank: {candidate[COL_FINAL_RANK]}" + ) + + # Final assignment + if candidate is not None: + df.loc[df.index == child.name, COL_ASSIGNED_PARENT] = candidate[ + COL_FINAL_RANK + ] + logger.debug( + f"Final assignment: child '{child[COL_FIXED_CONTRACT_NAME]}' → parent '{candidate[COL_FIXED_CONTRACT_NAME]}' | rank {candidate[COL_FINAL_RANK]}" + ) + else: + logger.debug( + f"No eligible parent found for child: '{child[COL_FIXED_CONTRACT_NAME]}'" + ) + + return df + + +def rank_children(df): + """Assign child ranks under their assigned parent""" + df[COL_CHILD_RANK] = None + df[COL_CHILD_INDEX] = None + + for (folder, group_id), group in df.groupby([COL_FOLDER, COL_IRS_GROUP]): + children = group[~group[COL_IS_PARENT]].sort_values(COL_FINAL_EFF_DATE) + + rank_map = {} + orphan_counter = 0 + + for _, child in children.iterrows(): + parent_rank = child[COL_ASSIGNED_PARENT] + + if pd.notnull(parent_rank): + rank_map[parent_rank] = rank_map.get(parent_rank, 0) + 1 + child_idx = rank_map[parent_rank] + df.loc[df.index == child.name, COL_CHILD_RANK] = ( + f"{parent_rank}.{child_idx}" + ) + df.loc[df.index == child.name, COL_CHILD_INDEX] = child_idx + else: + orphan_counter += 1 + df.loc[df.index == child.name, COL_CHILD_RANK] = f"0.{orphan_counter}" + df.loc[df.index == child.name, COL_CHILD_INDEX] = orphan_counter + + return df + + +# ============================================================================ +# MAIN PROCESSING PIPELINE +# ============================================================================ + + +def bcbs_main(df_read, xwalk_path=None): + """ + Main processing pipeline for BCBS parent-child mapping. + + Args: + df_read: Input DataFrame with contract data + xwalk_path: Path to BCBSNC crosswalk Excel file. If None, uses default + location (src/parent_child/BCBSNC_Folder_Xwalk.xlsx) + + Returns: + DataFrame with parent-child mappings + + Raises: + FileNotFoundError: If crosswalk file not found at specified or default path + """ + # Resolve crosswalk path with fallback + if xwalk_path is None: + xwalk_path = _DEFAULT_XWALK_PATH + + if not os.path.exists(xwalk_path): + raise FileNotFoundError( + f"BCBS crosswalk file not found: {xwalk_path}\n" + f"Please provide the path via xwalk_path parameter or place the file at: {_DEFAULT_XWALK_PATH}" + ) + + # Step 1: Load B fields data + b_fields = df_read + + # Step 2: Rename columns to standardized names + logger.info("Renaming columns...") + b_fields = b_fields.rename( + columns={ + COL_AGREEMENT_NAME_ORIG: COL_AGREEMENT_NAME, + COL_CONTRACT_NAME_ORIG: COL_CONTRACT_NAME, + COL_MULTIPLE_IRS_ORIG: COL_MULTIPLE_IRS, + COL_PAGE_COUNT_ORIG: COL_PAGES, + } + ) + + # Step 3: Consolidate fields + logger.info("Consolidating fields...") + consolidated_fields = consolidate_B_fields( + COL_CONTRACT_NAME, b_fields, FIELDS_TO_CONSOLIDATE + ) + + # Step 4: Merge consolidated fields + df = pd.merge(b_fields, consolidated_fields, on=COL_CONTRACT_NAME, how="left") + df = df.drop_duplicates(COL_CONTRACT_NAME) + df = df[ + [ + COL_CONTRACT_NAME, + COL_AGREEMENT_NAME, + COL_CONTRACT_EFF_DATE, + COL_MULTIPLE_IRS, + COL_PAGES, + COL_CONSOLIDATED_LOB, + COL_CONSOLIDATED_PROVIDER, + COL_LOB, + COL_PROVIDER_TYPE_ORIG, + ] + ] + + # Step 5: Fix and standardize filename + logger.info("Standardizing filenames...") + df[COL_FIXED_CONTRACT_NAME] = ( + df[COL_CONTRACT_NAME] + .str.lower() + .str.replace(r"filename: |" + FILE_EXTENSIONS, "", regex=True) + .str.strip() + ) + + # Step 6: Create IRS_Name column + logger.info("Creating IRS name column...") + df[COL_IRS_NAME] = df[COL_MULTIPLE_IRS].fillna(df[COL_FIXED_CONTRACT_NAME]) + + # Step 7: Load and process crosswalk + logger.info(f"Processing crosswalk from: {xwalk_path}") + xwalk = pd.read_excel(xwalk_path, sheet_name=XWALK_SHEET_NAME) + xwalk[COL_FIXED_CONTRACT_NAME] = ( + xwalk[XWALK_COL_FILE_NAME] + .str.lower() + .str.replace(r"filename: |" + FILE_EXTENSIONS, "", regex=True) + .str.strip() + ) + xwalk = xwalk.drop_duplicates([COL_FIXED_CONTRACT_NAME]) + + # Step 8: Merge with crosswalk + logger.info(f"df shape before merge: {df.shape}") + df = pd.merge( + df, + xwalk[[COL_FIXED_CONTRACT_NAME, XWALK_COL_FOLDER]], + on=COL_FIXED_CONTRACT_NAME, + how="left", + ) + logger.info(f"df shape after merge: {df.shape}") + + # Step 9: Process IRS names + logger.info("Processing IRS names...") + subset = df.sort_values(by=COL_IRS_NAME, ascending=True) + subset["irs_without_bcbs"] = subset[COL_IRS_NAME].apply( + remove_blue_cross_blue_shield + ) + subset["irs_prefix"] = subset["irs_without_bcbs"].apply(clean_prefix) + subset["irs_prefix_clean"] = subset["irs_prefix"].apply(preprocess_irs) + subset["irs_prefix_clean"] = subset["irs_prefix_clean"].apply(remove_bcbsnc) + subset["irs_prefix_clean"] = subset["irs_prefix_clean"].apply( + lambda x: " ".join([word for word in x.split() if word not in STOP_WORDS]) + ) + subset["irs_prefix_clean"] = subset["irs_prefix_clean"].apply(clean_string) + subset[COL_IRS_GROUP] = subset["irs_prefix_clean"].apply( + lambda x: ( + " ".join(x.split()[:2]) + if pd.notna(x) and "onslow" in x.lower() + else (x.split(" ")[0] if pd.notna(x) else None) + ) + ) + + # Step 10: Extract effective dates from filename + logger.info("Extracting effective dates...") + subset["raw_extract"] = subset[COL_FIXED_CONTRACT_NAME].apply(extract_date) + subset["fixed_raw_extract"] = subset["raw_extract"].apply(parse_date) + subset[COL_FINAL_EFF_DATE] = ( + subset[COL_CONTRACT_EFF_DATE] + .where(subset[COL_CONTRACT_EFF_DATE].fillna("").astype(str).str.strip() != "") + .fillna(subset["fixed_raw_extract"]) + ) + subset[COL_FINAL_EFF_DATE] = subset[COL_FINAL_EFF_DATE].apply(clean_and_parse) + + # Step 11: Create filling column for IRS group matching + logger.info("Creating filling column...") + subset["filling"] = ( + subset[COL_FIXED_CONTRACT_NAME].fillna("") + + " " + + subset[COL_AGREEMENT_NAME].fillna("") + + " " + + subset[COL_IRS_NAME].fillna("") + ) + subset["filling"] = ( + subset["filling"] + .str.replace("Filename: ", "", regex=True) + .str.replace(".txt", "", regex=True) + ) + subset["filling"] = subset["filling"].astype(str).str.lower() + subset["filling"] = subset["filling"].apply(clean_text) + + # Step 12: Process IRS group matching + logger.info("Processing IRS group matching...") + value_counts_IRS_Group = subset[COL_IRS_GROUP].value_counts() + values_to_modify = value_counts_IRS_Group[value_counts_IRS_Group > 1].index + mask = subset[COL_IRS_GROUP].isin(values_to_modify) + subset.loc[mask, "filling"] = np.nan + + subset["filling"] = subset["filling"].fillna("") + unique_a_values = subset[COL_IRS_GROUP].unique() + + def find_all_matching_values(text_b, unique_values): + matches = [] + for val_a in unique_values: + if val_a == "" or val_a.lower() == "health": + continue + if val_a in text_b: + if val_a == "cape": + matches.append("cumberland") + else: + matches.append(val_a) + return ", ".join(matches) + + subset["match_found"] = subset["filling"].apply( + lambda x: find_all_matching_values(x, set(unique_a_values)) + ) + + def remove_IRS_Group_matches(matching_values, IRS_Group_value): + matching_set = set( + [val.strip() for val in matching_values.split(",") if val.strip()] + ) + IRS_Group_set = ( + set([IRS_Group_value]) + if pd.notna(IRS_Group_value) and IRS_Group_value != "" + else set() + ) + final_set = matching_set - IRS_Group_set + return ", ".join(final_set) + + subset["match_found"] = subset.apply( + lambda row: remove_IRS_Group_matches(row["match_found"], row[COL_IRS_GROUP]), + axis=1, + ) + + subset["match_found"] = subset["match_found"].replace("", np.nan) + subset.loc[subset["match_found"].notna(), COL_IRS_GROUP] = subset["match_found"] + + subset = replace_single_counts(subset.copy(), COL_FOLDER, COL_IRS_GROUP) + subset[COL_IRS_GROUP] = subset[COL_IRS_GROUP].replace(IRS_GROUP_REPLACEMENTS) + + # Step 13: Clean contract title + logger.info("Cleaning contract titles...") + subset[COL_CONTRACT_TITLE_CLEAN] = ( + subset[COL_AGREEMENT_NAME] + .fillna("") + .str.lower() + .str.replace(r"[^a-z0-9\s]", " ", regex=True) + .str.replace(r"\s+", " ", regex=True) + .str.strip() + ) + subset[COL_CONTRACT_TITLE_CLEAN].replace("", pd.NA, inplace=True) + + # Step 14: Clean LOB column + logger.info("Cleaning LOB column...") + subset = clean_line_of_business_column(subset) + + # Step 15: Extract LOB from title/filename + logger.info("Extracting LOB...") + search_terms_lower = [str.lower(i) for i in LOB_SEARCH_TERMS] + patterns = [ + re.compile( + rf"(?:(?<=^)|(?<=[^a-z0-9]|_)){re.escape(term)}(?:(?=$)|(?=[^a-z0-9]|_))", + re.IGNORECASE, + ) + for term in search_terms_lower + ] + + def find_exact_match(row): + text_title = row[COL_CONTRACT_TITLE_CLEAN] + text_name = row[COL_FIXED_CONTRACT_NAME] + + if pd.notnull(text_title) and str(text_title).strip() != "": + t = text_title.lower() + for pattern, term in zip(patterns, search_terms_lower): + if pattern.search(t): + return term + + if pd.notnull(text_name) and str(text_name).strip() != "": + t = text_name.lower() + for pattern, term in zip(patterns, search_terms_lower): + if pattern.search(t): + return term + + return None + + subset[COL_EXTRACTED_LOB] = subset.apply(find_exact_match, axis=1) + subset[COL_EXTRACTED_LOB] = ( + subset[COL_EXTRACTED_LOB] + .replace("", pd.NA) + .fillna(subset[COL_CONSOLIDATED_LOB]) + ) + + subset = map_comma_separated_values(subset, COL_EXTRACTED_LOB, LOB_MAPPING) + + # Step 16: Fill extracted LOB + logger.info("Filling extracted LOB...") + subset["filled_extracted_lob"] = subset[COL_EXTRACTED_LOB].replace("", "comm") + subset.loc[ + subset["filled_extracted_lob"].str.contains("comm,"), "filled_extracted_lob" + ] = "comm" + subset.loc[ + subset["filled_extracted_lob"].str.contains("commercial-"), + "filled_extracted_lob", + ] = "comm" + subset.loc[ + subset["filled_extracted_lob"].str.contains("mcd, mcr"), "filled_extracted_lob" + ] = "mcd" + + subset["original_lob"] = subset[COL_EXTRACTED_LOB] + subset[COL_EXTRACTED_LOB] = subset["filled_extracted_lob"] + + # Step 17: Count provider types + subset[COL_PROVIDER_TYPE_COUNT] = ( + subset[COL_CONSOLIDATED_PROVIDER].str.split(",").str.len() + ) + + # Step 18: Identify non-parents + logger.info("Identifying non-parents...") + subset[COL_FINAL_EFF_DATE] = pd.to_datetime( + subset[COL_FINAL_EFF_DATE], errors="coerce" + ) + + cols_to_search = [COL_FIXED_CONTRACT_NAME, COL_CONTRACT_TITLE_CLEAN] + non_parents = add_flag_if_words_present( + subset, cols_to_search, NON_PARENT_KEYWORDS, flag_col="has_non_parent_keywords" + ) + subset = non_parents[~non_parents["has_non_parent_keywords"]] + logger.info(f"Shape after removing non-parents: {subset.shape}") + + # Step 19: Rank files on effective date (Parent identification step 1) + logger.info("Ranking by effective date...") + subset[COL_EFF_DATE_RANK] = subset.groupby([COL_FOLDER, COL_IRS_GROUP])[ + COL_FINAL_EFF_DATE + ].rank(method="dense", ascending=True) + + subset[COL_IS_PARENT] = False + + # Case 1: effective_date_rank == 1 AND title is not null + subset.loc[ + (subset[COL_EFF_DATE_RANK] == 1) & (subset[COL_CONTRACT_TITLE_CLEAN].notna()), + COL_IS_PARENT, + ] = True + + # Case 2: agreement in title but not amendment/addendum AND title not null + mask_case2 = ( + (subset[COL_EFF_DATE_RANK] != 1) + & (subset[COL_CONTRACT_TITLE_CLEAN].notna()) + & (subset[COL_CONTRACT_TITLE_CLEAN].str.contains(r"\bagreement\b", na=False)) + & (~subset[COL_CONTRACT_TITLE_CLEAN].str.contains(r"\bamendment\b", na=False)) + & (~subset[COL_CONTRACT_TITLE_CLEAN].str.contains(r"\baddendum\b", na=False)) + ) + subset.loc[mask_case2, COL_IS_PARENT] = True + + # Step 20: LOB-based parent identification (step 2) + logger.info("Identifying parents by LOB...") + mask_lob_no_amendment = ( + (subset[COL_IS_PARENT] == False) + & (subset[COL_EXTRACTED_LOB].notna()) + & (subset[COL_CONTRACT_TITLE_CLEAN].notna()) + & (~subset[COL_CONTRACT_TITLE_CLEAN].str.contains(r"\bamendment\b", na=False)) + & (~subset[COL_CONTRACT_TITLE_CLEAN].str.contains(r"\baddendum\b", na=False)) + ) + subset.loc[mask_lob_no_amendment, COL_IS_PARENT] = True + + # Exclude amendments/addendums from parents + mask_amend_addendum_title = subset[COL_CONTRACT_TITLE_CLEAN].str.contains( + r"\bamendment\b|\baddendum\b", na=False + ) + subset.loc[mask_amend_addendum_title, COL_IS_PARENT] = False + + mask_amend_addendum_filename = ( + subset[COL_FIXED_CONTRACT_NAME] + .str.lower() + .str.contains(r"\bamendment\b|\baddendum\b", na=False) + ) + subset.loc[mask_amend_addendum_filename, COL_IS_PARENT] = False + + logger.info(f"Parent count: {subset[COL_IS_PARENT].sum()}") + + # Step 21: Rank parents + logger.info("Ranking parents...") + mask_parent = subset[COL_IS_PARENT] == True + parent_subset = subset[mask_parent].copy() + + parent_subset["has_commercial"] = parent_subset[COL_EXTRACTED_LOB].apply( + lambda x: pd.notnull(x) and "comm" in str(x).lower() + ) + + def rank_group(df): + commercial_df = df[df["has_commercial"]].sort_values(COL_FINAL_EFF_DATE) + non_commercial_df = df[~df["has_commercial"]].sort_values(COL_FINAL_EFF_DATE) + ranked_df = pd.concat([commercial_df, non_commercial_df], ignore_index=False) + ranked_df[COL_FINAL_RANK] = range(1, len(ranked_df) + 1) + return ranked_df + + ranked_parents = parent_subset.groupby( + [COL_FOLDER, COL_IRS_GROUP], group_keys=False + ).apply(rank_group) + subset = subset.merge( + ranked_parents[[COL_FINAL_RANK]], left_index=True, right_index=True, how="left" + ) + subset = subset.sort_values(COL_FOLDER) + + # Step 22: Add back non-parents + logger.info("Adding back non-parents...") + non_parents_only = non_parents[non_parents["has_non_parent_keywords"] == True] + non_parents_only[COL_EFF_DATE_RANK] = "" + non_parents_only[COL_IS_PARENT] = False + non_parents_only[COL_FINAL_RANK] = np.nan + + subset = pd.concat([subset, non_parents_only]) + logger.info(f"Final shape: {subset.shape}") + + # Step 23: Assign parents to children + logger.info("Assigning parents to children...") + subset = assign_parents(subset) + + # Step 24: Rank children + logger.info("Ranking children...") + subset = rank_children(subset) + + # Step 25: Create combined ranks + logger.info("Creating combined ranks...") + subset[COL_COMBINED_RANK] = subset[COL_FINAL_RANK].fillna(subset[COL_CHILD_RANK]) + subset["combined_index"] = subset[COL_FINAL_RANK].fillna(subset[COL_CHILD_INDEX]) + + # Step 26: Create parent/child/orphan flags + logger.info("Creating flags...") + subset[COL_PARENT_CHILD_FLAG] = subset[COL_IS_PARENT].map( + {True: "Parent", False: "Child"} + ) + + subset.loc[ + (subset[COL_IS_PARENT] == False) + & (subset[COL_COMBINED_RANK].astype(str).str.count(r"\.") == 1), + COL_PARENT_CHILD_FLAG, + ] = "Orphan" + + # Step 27: Create naming column + subset[COL_NAMING_COL] = ( + subset[COL_FOLDER].replace({None: "xx", np.nan: "xx", "": "xx"}).astype(str) + + "_" + + subset[COL_IRS_GROUP] + .replace({None: "xx", np.nan: "xx", "": "xx"}) + .astype(str) + + "_" + + subset[COL_PARENT_CHILD_FLAG] + .replace({None: "xx", np.nan: "xx", "": "xx"}) + .astype(str) + + "_" + + subset["combined_index"].apply( + lambda x: ( + "xx" + if pd.isnull(x) or x == "" + else (str(int(x)) if float(x) == int(x) else str(x)) + ) + ) + ) + + # Step 28: Add File Name and Provider Type_Consolidated columns + subset["File Name"] = subset[COL_CONTRACT_NAME] + subset["Provider Type_Consolidated"] = subset[COL_CONSOLIDATED_PROVIDER] + + # Step 29: Select output columns and export + subset = subset[OUTPUT_COLUMNS] + subset = subset.sort_values(COL_FOLDER) + logger.info(f"Total records: {len(subset)}") + logger.info(f"Parents: {subset[COL_IS_PARENT].sum()}") + logger.info(f"Children: {(~subset[COL_IS_PARENT]).sum()}") + return subset diff --git a/src/parent_child/column_mapper.py b/src/parent_child/column_mapper.py new file mode 100644 index 0000000..c837ca5 --- /dev/null +++ b/src/parent_child/column_mapper.py @@ -0,0 +1,266 @@ +""" +Column Mapper Utility for Generic Parent-Child Processing + +This module auto-detects column names from input DataFrames to make the +parent-child processing pipeline work across different data sources without +requiring JSON/YAML configuration files. +""" + +import pandas as pd +import re +import logging +from typing import Dict, Optional, List + + +class ColumnMapper: + """ + Automatically detects and maps column names from input DataFrame to + standardized internal column names used in processing. + """ + + # Define patterns for each required field + COLUMN_PATTERNS = { + "FILE_NAME": [ + r"file_name", + r"^file.*name$", + r"^contract.*name$", + r"^document.*name$", + r"^filename$", + r"^doc.*name$", + ], + "CONTRACT_TITLE": [ + r"^agreement_name.*contract.*title.*$", + r"^agreement.*name$", + r"^contract.*title$", + r"^title$", + r"^contract.*name.*title$", + r"^agreement.*title$", + r"^doc.*title$", + ], + "PAYER_NAME": [ + r"^payer.*name$", + r"^payer$", + r"^plan.*name$", + r"^insurance.*name$", + r"^carrier.*name$", + ], + "PROV_GROUP_NAME_FULL": [ + r"^prov.*group.*name.*full$", + r"^irs_name", + r"^provider.*group.*name$", + r"^provider.*name$", + r"^prov.*name$", + r"^group.*name$", + ], + "PROV_GROUP_TIN": [ + r"^prov.*group.*tin$", + r"^irs.*#$", + r"^irs.*number$", + r"^tin$", + r"^tax.*id$", + r"^tax.*identification$", + r"^employer.*id$", + r"^ein$", + ], + "PROV_GROUP_NPI": [ + r"^prov.*group.*npi$", + r"^npi.*10.*digit.*$", + r"^npi$", + r"^national.*provider.*id$", + r"^provider.*npi$", + ], + "EFFECTIVE_DATE": [ + r"aarete_derived_effective_dt", + r"^.*effective.*date$", + r"^eff.*date$", + r"^start.*date$", + r"^begin.*date$", + r"^contract.*effective.*date$", + ], + "TERMINATION_DATE": [ + r"aarete_derived_termination_dt", + r"^.*termination.*date$", + r"^term.*date$", + r"^end.*date$", + r"^expiration.*date$", + r"^expire.*date$", + ], + "LOB": [ + r"aarete_derived_lob", + r"^lob$", + r"^line.*of.*business$", + r"^product.*line$", + r"^business.*line$", + ], + "SERVICE_TERM": [ + r"^service.*term$", + r"^term$", + r"^service.*type$", + r"^service.*category$", + ], + "PAYER_STATE": [ + r"^payer.*state$", + r"^state$", + r"^location.*state$", + r"^plan.*state$", + r"^health.*plan.*state$", + ], + } + + # Optional columns (won't fail if not found) + OPTIONAL_COLUMNS = [ + "PAYER_STATE", + "SERVICE_TERM", + "PROV_GROUP_TIN", + "PROV_GROUP_NPI", + ] + + def __init__(self, df: pd.DataFrame): + """ + Initialize the ColumnMapper with a DataFrame. + + Args: + df: Input DataFrame to analyze + """ + self.df = df + self.column_mapping = {} + self._detect_columns() + + def _normalize_column_name(self, col_name: str) -> str: + """Normalize column name for pattern matching.""" + return col_name.lower().strip().replace(" ", "_") + + def _match_column( + self, patterns: List[str], available_columns: List[str] + ) -> Optional[str]: + """ + Match a column name against a list of regex patterns. + + Args: + patterns: List of regex patterns to match + available_columns: List of available column names + + Returns: + Matched column name or None + """ + for col in available_columns: + normalized_col = self._normalize_column_name(col) + for pattern in patterns: + if re.match(pattern, normalized_col, re.IGNORECASE): + return col + return None + + def _detect_columns(self): + """Auto-detect column mappings from DataFrame.""" + available_columns = self.df.columns.tolist() + logging.info(f"Available columns in input DataFrame: {available_columns}") + + for standard_name, patterns in self.COLUMN_PATTERNS.items(): + matched_col = self._match_column(patterns, available_columns) + + if matched_col: + self.column_mapping[standard_name] = matched_col + logging.info(f"Mapped '{standard_name}' -> '{matched_col}'") + elif standard_name not in self.OPTIONAL_COLUMNS: + logging.warning( + f"Could not find column for '{standard_name}'. Patterns tried: {patterns}" + ) + + # Validate required columns are present + self._validate_required_columns() + + def _validate_required_columns(self): + """Validate that all required columns were found.""" + required_fields = [ + "FILE_NAME", + "CONTRACT_TITLE", + "PAYER_NAME", + "PROV_GROUP_NAME_FULL", + "EFFECTIVE_DATE", + "TERMINATION_DATE", + "LOB", + ] + + missing_fields = [ + field for field in required_fields if field not in self.column_mapping + ] + + if missing_fields: + error_msg = ( + f"Missing required columns: {missing_fields}\n" + f"Available columns: {self.df.columns.tolist()}\n" + f"Successfully mapped: {self.column_mapping}\n\n" + f"Please ensure your input file contains columns that match these patterns:\n" + ) + for field in missing_fields: + patterns = self.COLUMN_PATTERNS.get(field, []) + error_msg += f" {field}: {patterns}\n" + + raise ValueError(error_msg) + + def get(self, standard_name: str) -> Optional[str]: + """ + Get the actual column name for a standard field name. + + Args: + standard_name: Standard field name (e.g., 'FILE_NAME') + + Returns: + Actual column name in DataFrame or None if not found + """ + return self.column_mapping.get(standard_name) + + def get_all_mappings(self) -> Dict[str, str]: + """Get all column mappings.""" + return self.column_mapping.copy() + + def rename_dataframe(self, df: pd.DataFrame) -> pd.DataFrame: + """ + Rename DataFrame columns to standardized names. + + Args: + df: Input DataFrame + + Returns: + DataFrame with standardized column names + """ + reverse_mapping = {v: k for k, v in self.column_mapping.items()} + return df.rename(columns=reverse_mapping) + + def has_column(self, standard_name: str) -> bool: + """ + Check if a column mapping exists. + + Args: + standard_name: Standard field name + + Returns: + True if mapping exists, False otherwise + """ + return standard_name in self.column_mapping + + def get_one_to_n_fields(self) -> List[str]: + """ + Get list of one-to-many fields that should be consolidated. + + Returns: + List of actual column names for one-to-many fields + """ + one_to_n_fields = [] + + if self.has_column("LOB"): + one_to_n_fields.append(self.get("LOB")) + + if self.has_column("SERVICE_TERM"): + one_to_n_fields.append(self.get("SERVICE_TERM")) + + return one_to_n_fields + + def log_mapping_summary(self): + """Log a summary of detected column mappings.""" + logging.info("=" * 60) + logging.info("Column Mapping Summary:") + logging.info("=" * 60) + for standard, actual in sorted(self.column_mapping.items()): + logging.info(f" {standard:25} -> {actual}") + logging.info("=" * 60) diff --git a/src/parent_child/main.py b/src/parent_child/main.py deleted file mode 100644 index 9e5b943..0000000 --- a/src/parent_child/main.py +++ /dev/null @@ -1,56 +0,0 @@ -import src.config as config - -# from src.parent_child.file_reading_utils import read_file_from_path -from src.parent_child.parent_child_preprocessing import parent_child_preprocessing -from src.parent_child.parent_child_mapping import parent_child_mapping -import os -from src.utils import io_utils -import logging -from datetime import datetime - - -def main(doczy_output_for_pc): - # Generate timestamp for this run - run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}" - - try: - df_read = io_utils.read_DataFrame(doczy_output_for_pc) - logging.info(f"original shape: {df_read.shape}") - cleaned_df = parent_child_preprocessing(df_read) - logging.info( - f"Unique contracts in the output: {cleaned_df["FILE_NAME"].nunique()}" - ) - mapped_df = parent_child_mapping(cleaned_df) - - # Save detailed results using same pattern as investment pipeline - if config.WRITE_PC_TO_S3: - io_utils.write_s3( - mapped_df, "Parent_Child_Mapping_", run_timestamp, "parent_child" - ) - logging.info( - f"PC results uploaded to S3: {config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-PC.csv" - ) - else: - io_utils.write_local( - mapped_df, "Parent_Child_Mapping_", run_timestamp, "parent_child" - ) - logging.info( - f"PC results saved locally: {config.CONSOLIDATED_OUTPUT_DIRECTORY}/{run_timestamp}/{config.BATCH_ID}-PC.csv" - ) - return mapped_df.shape[0] - except Exception as e: - logging.error(f"Parent child mapping failed due to {e}") - - -if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - ) - # Generate timestamp for this run - run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}" - - if config.DOCZY_OUTPUT_FOR_PC: - results = main(config.DOCZY_OUTPUT_FOR_PC) - - logging.info(f"PC Mapping complete. Processed {results} files.") diff --git a/src/parent_child/parent_child_mapping.py b/src/parent_child/parent_child_mapping.py deleted file mode 100644 index eb9b8df..0000000 --- a/src/parent_child/parent_child_mapping.py +++ /dev/null @@ -1,890 +0,0 @@ -import warnings - -warnings.filterwarnings("ignore") -import pandas as pd -import re -from src.parent_child.parent_child_preprocessing import ( - clean_effective_date, - clean_termination_date, -) -import numpy as np -import logging -from src.utils import io_utils - - -def create_df_per_tin_presence(df, is_tin_present): - """ - Filters the DataFrame based on the presence or absence of TIN. - - Parameters: - ---------- - df : input DataFrame containing TIN and ICMP-related columns. - is_tin_present : bool - If True, returns rows where 'TIN' is present (not NaN). - If False, returns rows where 'TIN' is missing and 'icmp_part' is present. - - Returns: - ------- - pandas.DataFrame - Filtered DataFrame according to the TIN presence logic. - """ - if is_tin_present: - return df[~df["TIN"].isna()] - else: - return df[df["TIN"].isna() & df["icmp_part"].notna()] - - -def filter_on_exclude_keywords(df, is_tin_present): - """ - Flags rows as 'parent' based on exclusion keywords. - - Parameters: - ---------- - df : input DataFrame containing contract information. - is_tin_present : bool - Flag used to select the appropriate keyword list from `exclude_keywords_dict`. - - Returns: - ------- - updated DataFrame with the 'parent' column set to False - if either 'contract_title_cleaned' or 'fixed_contract_name' - contains any of the exclusion keywords (case-insensitive). - """ - df["parent"] = True - - exclude_keywords_dict = io_utils.convert_json_to_dict( - "constants/mappings/exclude_keywords_for_pc.json" - ) - exclude_keywords_list = exclude_keywords_dict[is_tin_present] - exclude_pattern = ( - r"\b(?:" + "|".join(re.escape(kw) for kw in exclude_keywords_list) + r")\b" - ) - - df["parent"] = ~( - df["contract_title_cleaned"].str.contains(exclude_pattern, case=False, na=False) - | df["contract_name_cleaned"].str.contains( - exclude_pattern, case=False, na=False - ) - ) - return df - - -def filter_on_missing_title(df): - """ - Updates 'parent' flag based on missing or empty contract titles and keywords in FILE_NAME. - - Rules: - 1. If 'contract_title_cleaned' is NaN or empty, default 'parent' to False. - 2. Exception: if 'contract_title_cleaned' is null and 'agmt' in 'contract_name', set parent = True. - 3. Exception to exception: if 'contract_title_cleaned' is null and both 'agmt' and 'letter' in 'contract_name', set parent = False. - """ - df = df.copy() - - # Normalize text columns - contract_name_lower = df["contract_name_cleaned"].astype(str).str.lower() - title_missing = df["contract_title_cleaned"].isna() | df[ - "contract_title_cleaned" - ].astype(str).str.strip().isin(["", "nan"]) - - # Base: set parent False where title is missing - df["parent"] = df["parent"].where(~title_missing, False) - - # Compute final parent mask in one step - # Parent = True if missing title & 'agmt' in name, except if also 'letter' in name - df.loc[ - title_missing - & contract_name_lower.str.contains("agmt") - & ~contract_name_lower.str.contains("letter"), - "parent", - ] = True - - return df - - -def flag_duplicates_within_group(df, grouper_col): - """ - Flags duplicates within each group for rows where parent == True, - based on an exact match of: - - 'contract_title_cleaned' - - 'fixed_effective_date' - - 'payer_name_cleaned' - - NaN values in 'payer_name_cleaned' are treated as equal when both are NaN, - and as different when only one is NaN. - - Non-parent rows are returned unchanged. - """ - df = df.copy() - df["is_duplicate"] = False # default for all rows - - # Work only on parent rows - mask = df["parent"] == True - df_parent = df.loc[mask].copy() - - # Handle NaNs consistently - df_parent["_payer"] = df_parent["payer_name_cleaned"].fillna("__NULL__") - - # Build duplicate key - df_parent["_dup_key"] = ( - df_parent["contract_title_cleaned"].astype(str) - + "||" - + df_parent["fixed_effective_date"].astype(str) - ) - - # Detect duplicates - df_parent["is_duplicate"] = df_parent.groupby(grouper_col)["_dup_key"].transform( - lambda x: x.duplicated(keep=False) - ) - - # Update only parent rows in original df - df.loc[mask, "is_duplicate"] = df_parent["is_duplicate"] - - # Clean temp cols (only in df_parent, since they weren’t in df originally) - return df - - -def rank_parents(df, grouper_col): - """ - Assigns a rank to 'parent' contracts within each group defined by `grouper_col`. - - Ranking logic: - 1. Only rows where `parent == True` are ranked. - 2. Ranking is based on earliest 'fixed_effective_date' first (NaT placed last). - 3. Each parent gets its own rank (duplicates are not merged). - 4. Null dates are ranked last. - """ - df = df.copy() - df["parent_rank"] = np.nan # Initialize - - # Ensure effective_date is datetime - df["_sort_date"] = pd.to_datetime(df["fixed_effective_date"], errors="coerce") - - for group_val, group_df in df.groupby(grouper_col): - parents = group_df[group_df["parent"] == True].copy() - if parents.empty: - continue - - # Replace NaT with a large future date for sorting → ensures null dates go last - parents["_rank_date"] = parents["_sort_date"].fillna(pd.Timestamp.max) - - # Sort and assign unique ranks - parents = parents.sort_values("_rank_date") - ranks = range(1, len(parents) + 1) - - df.loc[parents.index, "parent_rank"] = list(ranks) - - return df.drop(columns=["_sort_date", "_rank_date"], errors="ignore") - - -def flag_multi_parent_with_early_dates( - df: pd.DataFrame, grouper_col: str -) -> pd.DataFrame: - """ - Flags groups where: - 1. There are multiple unique parent ranks. - 2. The group has children. - 3. At least two parents have effective dates earlier than at least one child. - - Parameters - ---------- - df : pd.DataFrame - The DataFrame containing at least: - - 'parent' (bool) - - 'parent_rank' - - 'fixed_effective_date' - - Grouping column specified by `grouper_col` - grouper_col : str - The column name to group by (e.g., 'TIN'). - - Returns - ------- - pd.DataFrame - The input DataFrame with an added column: - 'flag_multi_parent_with_early_dates' containing either the flag value - 'multiple_parents_early_children' or NaN. - - """ - df = df.copy() - df["fixed_effective_date"] = pd.to_datetime( - df["fixed_effective_date"], errors="coerce" - ) - flag_dict = {} - - for group_val, group_df in df.groupby(grouper_col): - parents_df = group_df[group_df["parent"] == True] - children_df = group_df[group_df["parent"] == False] - - # Skip if less than 2 unique parents or no children - unique_parents = parents_df["parent_rank"].dropna().unique() - if len(unique_parents) <= 1 or children_df.empty: - continue - - # Get all child dates - child_dates = children_df["fixed_effective_date"].dropna() - if child_dates.empty: - continue - - # Count parents with earlier dates than any child - count_early_parents = sum( - pd.notna(parent_row["fixed_effective_date"]) - and (child_dates > parent_row["fixed_effective_date"]).any() - for _, parent_row in parents_df.iterrows() - ) - - if count_early_parents >= 2: - flag_dict[group_val] = "multiple_parents_early_children" - - # Map flags back to DataFrame - df["flag_multi_parent_with_early_dates"] = df[grouper_col].map(flag_dict) - return df - - -def assign_parents_with_state_and_payer( - df, grouper_col, flag_col="flag_multi_parent_with_early_dates" -): - """ - Assigns each child contract to a parent contract with strict TIN, state, payer, and date logic. - """ - - df = df.copy() - df["fixed_effective_date"] = pd.to_datetime( - df["fixed_effective_date"], errors="coerce" - ) - - parents = df[df["parent"]].copy() - children = df[~df["parent"]].copy() - assignments = [] - - for idx, child in children.iterrows(): - # Step 1: Filter parents within TIN group - group_parents = parents[parents[grouper_col] == child[grouper_col]].copy() - - if group_parents.empty: - assignments.append(("no_parent", idx)) - continue - - # Step 2: Filter by matching state - state_parents = group_parents[ - group_parents["PAYER_STATE"] == child["PAYER_STATE"] - ] - - if state_parents.empty: - assignments.append(("no_parent", idx)) - continue - - # Step 3: Apply effective date constraint (parent date <= child date) - if pd.notna(child["fixed_effective_date"]): - state_parents = state_parents[ - pd.to_datetime(state_parents["fixed_effective_date"], errors="coerce") - <= child["fixed_effective_date"] - ] - - if state_parents.empty: - assignments.append(("no_parent", idx)) - continue - - # Step 4: Tie-breaker on payer_name - if len(state_parents) > 1: - if pd.notna(child.get("payer_name_cleaned")): - payer_match = state_parents[ - state_parents["payer_name_cleaned"] == child["payer_name_cleaned"] - ] - if len(payer_match) == 1: - assignments.append((payer_match.iloc[0]["parent_rank"], idx)) - continue - elif len(payer_match) > 1: - # Still multiple → pick latest - latest_parent = payer_match.sort_values( - "parent_rank", ascending=False - ).iloc[0] - assignments.append((latest_parent["parent_rank"], idx)) - continue - else: - # No matching payer → no parent - assignments.append(("no_parent", idx)) - continue - else: - # Child payer_name null → no parent - assignments.append(("no_parent", idx)) - continue - else: - # Only one parent left → assign - assignments.append((state_parents.iloc[0]["parent_rank"], idx)) - - children["assigned_parent_rank"] = None - for rank, idx in assignments: - children.at[idx, "assigned_parent_rank"] = rank - - return pd.concat([parents, children]).sort_index() - - -def assign_no_parent_by_title( - df, grouper_col, contract_title_col="contract_title_cleaned" -): - """ - Second-pass assignment for children with 'no_parent' in assigned_parent_rank. - Assigns based on contract title substring match, with same restrictions: - - Must match on grouper_col (e.g., TIN). - - Parent's contract title must be substring of child's title (case-insensitive). - - Parent's effective date must be <= child's date. - - Pick latest parent_rank if multiple. - - Parameters - ---------- - df : pd.DataFrame - Must contain: - - 'assigned_parent_rank' - - 'parent' (bool) - - 'parent_rank' - - 'fixed_effective_date' - - contract_title_col (str) - - grouper_col (str) - grouper_col : str - Column name for grouping (e.g., 'TIN'). - contract_title_col : str - Column name containing the contract title. - - Returns - ------- - pd.DataFrame - Updated DataFrame with some 'no_parent' children now assigned. - """ - # Validate input DataFrame is not empty - if df.empty: - logging.info("Warning: Input DataFrame is empty") - return df.copy() - - # Validate required columns exist - required_cols = [ - "assigned_parent_rank", - "parent", - "parent_rank", - "fixed_effective_date", - contract_title_col, - grouper_col, - ] - - missing_cols = [col for col in required_cols if col not in df.columns] - if missing_cols: - logging.info(f"Available columns: {list(df.columns)}") - raise KeyError(f"Missing required columns: {missing_cols}") - - logging.info(f"Processing {len(df)} rows") - logging.info(f"Using columns - grouper: {grouper_col}, title: {contract_title_col}") - - # Create a copy and ensure we're working with the right data types - df_work = df.copy() - - # Convert date column with explicit error handling - try: - df_work["fixed_effective_date"] = pd.to_datetime( - df_work["fixed_effective_date"], errors="coerce" - ) - except Exception as e: - logging.info(f"Error converting dates: {e}") - return df.copy() - - # Filter parents and children with no parent - parents = df_work[df_work["parent"] == True].copy() - children_no_parent = df_work[df_work["assigned_parent_rank"] == "no_parent"].copy() - - logging.info( - f"Found {len(parents)} parents and {len(children_no_parent)} children with no_parent" - ) - - if len(parents) == 0 or len(children_no_parent) == 0: - logging.info("No work to do - either no parents or no unassigned children") - return df.copy() - - assignments_made = 0 - - # Process each child that needs a parent - for idx, child in children_no_parent.iterrows(): - # Find potential parents in same group - try: - possible_parents = parents[ - parents[grouper_col] == child[grouper_col] - ].copy() - except KeyError as e: - logging.info(f"Error accessing grouper column '{grouper_col}': {e}") - continue - - if len(possible_parents) == 0: - continue - - child_date = child["fixed_effective_date"] - if pd.isna(child_date): - continue # skip if child's date missing - - child_title = child[contract_title_col] - if not isinstance(child_title, str): - continue # skip if child title is not a string - - child_title_lower = child_title.lower() - - # Find parents whose title is substring of child's title (case-insensitive) - def title_match(parent_title): - if not isinstance(parent_title, str): - return False - return parent_title.lower() in child_title_lower - - title_matches = possible_parents[ - possible_parents[contract_title_col].apply(title_match) - ] - - if len(title_matches) == 0: - continue - - # Filter by effective date: parent date <= child date - eligible = title_matches[title_matches["fixed_effective_date"] <= child_date] - - if not eligible.empty: - # Choose parent with latest parent_rank - chosen_parent_rank = eligible.sort_values( - "parent_rank", ascending=False - ).iloc[0]["parent_rank"] - df_work.at[idx, "assigned_parent_rank"] = chosen_parent_rank - assignments_made += 1 - - logging.info(f"Made {assignments_made} new parent assignments") - return df_work - - -def assign_child_ranks(df, grouper_col): - """ - Assigns sequential child ranks to non-parent contracts. - - Ranking logic: - - Parent rank assigned earlier is used as the base. - - Orphans (assigned_parent_rank == 'no_parent') start with base rank "0". - - Children are ordered by 'fixed_effective_date' (null dates last). - - No deduplication: every child gets a unique sequential rank. - - The child rank is always a string (e.g., "1.1", "1.2", "0.1"). - - Returns - ------- - pd.DataFrame - DataFrame with added 'child_rank' and 'combined_rank' columns (as string). - """ - df = df.copy() - - # Separate parents and children - parents = df[df["parent"]].copy() - children = df[~df["parent"]].copy() - - # Assign children to parents (and orphans) - for (grp_val, parent_rank), grp_df in children.groupby( - [grouper_col, "assigned_parent_rank"] - ): - if parent_rank == "no_parent": # orphan group - base_rank = "0" - else: - base_rank = str(parent_rank) - - # Sort by date (null dates last) - grp_df = grp_df.assign( - sort_date=grp_df["fixed_effective_date"].fillna(pd.Timestamp.max) - ).sort_values("sort_date") - - # Sequential rank without deduplication - for i, idx in enumerate(grp_df.index, start=1): - rank_val = f"{base_rank}.{i}" - children.loc[idx, "child_rank"] = rank_val - - logging.info(children.columns) - # Combine back - result = pd.concat([parents, children]).sort_index() - # Make sure child_rank is string - result["child_rank"] = result["child_rank"].astype(str) - - # If parent rank is "no_parent", fall back to child rank - def pick_rank(row): - if row.get("assigned_parent_rank") == "no_parent": - return row["child_rank"] - return ( - str(row["parent_rank"]) - if pd.notna(row["parent_rank"]) - else row["child_rank"] - ) - - result["combined_rank"] = result.apply(pick_rank, axis=1) - - return result - - -def assign_orphans_based_on_unique_parent_and_payer( - df, - grouper_col="TIN", - payer_col="payer_name_cleaned", - assigned_parent_rank_col="assigned_parent_rank", -): - """ - Assigns orphans to their unique parent when: - - There's only one unique payer in the TIN group. - - There's exactly one unique parent in the TIN group (based on 'parent_rank'). - - The child has 'no_parent' in 'assigned_parent_rank'. - - There is at least one parent in the TIN group. - - The parent’s effective date is older than or equal to the child’s effective date, or the child’s effective date is missing. - - If there are multiple unique parents and exactly one unique payer, assign to the latest eligible parent (i.e., the lowest `parent_rank`). - - Parameters - ---------- - df : pd.DataFrame - The input DataFrame. - grouper_col : str - Column name for the grouping (e.g., 'TIN'). - payer_col : str - Column name for the PAYER_NAME (e.g., 'payer_name_cleaned'). - assigned_parent_rank_col : str - Column for the assigned parent rank (e.g., 'assigned_parent_rank'). - - Returns - ------- - pd.DataFrame - Updated DataFrame with some orphans assigned to their unique parent. - """ - # Ensure 'assigned_parent_rank' is initialized - if assigned_parent_rank_col not in df.columns: - raise KeyError(f"'{assigned_parent_rank_col}' column is missing.") - - # Step 1: Filter TIN groups where there are orphans ('no_parent') and at least one parent exists - df_work = df.copy() - - # Filter TIN groups where: - # - There are orphans (i.e., 'no_parent' in 'assigned_parent_rank') - # - At least one parent exists in the group - tin_present = df_work.groupby(grouper_col).filter( - lambda group: (group[assigned_parent_rank_col] == "no_parent").any() - and (group["parent"] == True).any() - ) - - if tin_present.empty: - logging.info("No TIN groups meet the filtering criteria.") - return df_work - - # Step 2: Identify orphans (children with 'no_parent') - orphan_groups = tin_present[tin_present[assigned_parent_rank_col] == "no_parent"] - if orphan_groups.empty: - logging.info("No orphans to assign.") - return df_work - - # Step 3: Iterate over TIN groups to assign orphans - assignments_made = 0 - for tin_group, group_df in orphan_groups.groupby(grouper_col): - # Find the parents in this TIN group - possible_parents = tin_present[ - (tin_present[grouper_col] == tin_group) & (tin_present["parent"] == True) - ] - - # Step 4: Check for unique parent ranks (i.e., exactly one unique parent in the TIN group) - unique_parent_ranks = possible_parents["parent_rank"].nunique() - - # Step 5: Check for unique PAYER_NAMEs (including NaN values) - unique_payer_names = possible_parents[payer_col].nunique(dropna=False) - - # Debug: Check unique parent ranks and PAYER_NAMEs in the group - # logging.info(f"Processing TIN group {tin_group}:") - # logging.info(f"Unique parent ranks: {unique_parent_ranks}") - # logging.info(f"Unique PAYER_NAMEs (including NaN): {unique_payer_names}") - - # Step 6: If there is more than one unique payer, do not assign any parent - if unique_payer_names > 1: - # logging.info(f"More than one unique payer in TIN group {tin_group}. Skipping assignment.") - continue - - # Step 7: If there's exactly one unique parent and one unique payer, assign to that parent - if unique_parent_ranks == 1: - parent = possible_parents.iloc[0] - parent_rank = parent["parent_rank"] - parent_effective_date = pd.to_datetime( - parent["fixed_effective_date"], errors="coerce" - ) - - # Step 8: Assign the single parent to the orphans in this TIN group - for _, orphan in group_df.iterrows(): - child_effective_date = pd.to_datetime( - orphan["fixed_effective_date"], errors="coerce" - ) - - # Check if the child's effective date is older than the parent's - if ( - pd.isna(child_effective_date) - or parent_effective_date <= child_effective_date - ): - df_work.at[orphan.name, assigned_parent_rank_col] = parent_rank - assignments_made += 1 - # logging.info(f"Assigned parent rank {parent_rank} to orphan {orphan.name}") - - # Step 9: If there's exactly one unique payer and more than one unique parent, assign to the latest parent (lowest rank) - elif unique_parent_ranks > 1: - # logging.info(f"Multiple unique parents in TIN group {tin_group}. Assigning to the latest parent (lowest rank).") - - # Sort parents by parent_rank (which is based on effective date) - sorted_parents = possible_parents.sort_values( - "parent_rank", ascending=False - ) # Lower rank is the latest parent - - # Assign the orphan to the latest eligible parent - for _, orphan in group_df.iterrows(): - child_effective_date = pd.to_datetime( - orphan["fixed_effective_date"], errors="coerce" - ) - - # Find the first parent who meets the effective date condition - for _, parent in sorted_parents.iterrows(): - parent_effective_date = pd.to_datetime( - parent["fixed_effective_date"], errors="coerce" - ) - - if ( - pd.isna(child_effective_date) - or parent_effective_date <= child_effective_date - ): - df_work.at[orphan.name, assigned_parent_rank_col] = parent[ - "parent_rank" - ] - assignments_made += 1 - # logging.info(f"Assigned parent rank {parent['parent_rank']} to orphan {orphan.name}") - break - - return df_work - - -def assign_parent_names(df, rank_col="combined_rank", name_col="CONTRACT_NAME"): - """ - Adds a 'parent_name' column mapping each child to its parent's name. - - Parents are detected as rows whose rank has only one number part (e.g. '1', '1.0', '001') - - Children starting with '0.' get no parent (None) - - Works with different numeric formats - - Parameters - ---------- - df : pd.DataFrame - Input dataframe - rank_col : str - Column containing rank (e.g. 'combined_rank') - name_col : str - Column containing name (e.g. 'CONTRACT_NAME') - - Returns - ------- - pd.DataFrame - Dataframe with 'parent_name' column added - """ - df = df.copy() - df[rank_col] = df[rank_col].astype(str).str.strip() - - # Detect parents: only one part when split by dot, or dot followed by zeros - parents_df = df[df[rank_col].str.match(r"^\d+(\.0+)?$", na=False)].copy() - - # Normalize parent rank to plain integer string (e.g. '1.0' -> '1') - parents_df["parent_key"] = ( - parents_df[rank_col].str.split(".").str[0].astype(int).astype(str) - ) - - # Create mapping parent_key -> parent name - parent_map = parents_df.set_index("parent_key")[name_col].to_dict() - - # Function to get parent name - def get_parent_name(rank): - rank = str(rank).strip() - if rank.startswith("0.") or rank in parent_map: - return None - parent_key = rank.split(".")[0] - return parent_map.get(parent_key) - - df["parent_name"] = df[rank_col].apply(get_parent_name) - return df - - -def extract_ordinal(row): - num_dict = io_utils.convert_json_to_dict("constants/mappings/numeric_mappigs.json") - word_to_num = num_dict.get("ordinal_word_to_number", {}) - cardinal_to_num = num_dict.get("cardinal_word_to_number", {}) - ordinal_pattern = num_dict.get("ordinal_regex_pattern", "") - - # Combine all text-based mappings - all_words_to_num = {**word_to_num, **cardinal_to_num} - text = str(row["contract_title_cleaned"]).lower() - - # Match ordinal or cardinal words - for word, num in all_words_to_num.items(): - if word in text: - return num - - # Match numeric ordinals (e.g., 1st, 2nd, 3rd, 25th) - num_match = re.search(ordinal_pattern, text) - if num_match: - value = re.sub(r"(st|nd|rd|th)$", "", num_match.group(0)) - if value.isdigit(): - return int(value) - - return None - - -def assign_parent_name_by_rank(df): - """ - Assigns the parent's 'FILE_NAME' to child rows based on TIN and a - modified combined_rank. - - Args: - df (pd.DataFrame): DataFrame with 'TIN', 'parent_child_flag', - 'combined_rank', and 'FILE_NAME' columns. - 'parent_child_flag' should be 'parent' or 'child'. - - Returns: - pd.DataFrame: DataFrame with an added 'parent_name' column. - """ - - df = df.copy() - df["parent_name"] = None # Initialize parent_name column - - # Separate parents and children based on parent_child_flag - parents_df = df[df["parent_child_flag"] == "Parent"].copy() - children_df = df[df["parent_child_flag"] == "Child"].copy() - - # Create a mapping for parents: (TIN, combined_rank) -> FILE_NAME - # This will be used to look up the parent's name efficiently - parent_lookup = parents_df.set_index(["TIN", "combined_rank"])[ - "FILE_NAME" - ].to_dict() - - # Iterate through each child to find its parent - for index, child_row in children_df.iterrows(): - child_tin = child_row["TIN"] - child_combined_rank = str( - child_row["combined_rank"] - ) # Ensure string for splitting - - # 2. Remove the last decimal to get the conceptual parent rank - if "." in child_combined_rank: - parent_conceptual_rank = ".".join(child_combined_rank.split(".")[:-1]) - else: - # If a child rank doesn't have a decimal (e.g., '1'), it might not have a parent of this type - parent_conceptual_rank = ( - None # Or handle as appropriate, maybe it's a top-level child - ) - - if parent_conceptual_rank: - # 3. Try to find a parent with the matching TIN and conceptual rank - parent_contract_name = parent_lookup.get( - (child_tin, parent_conceptual_rank) - ) - - # 3. Add parent name to the row where the corresponding child is in - if parent_contract_name: - df.at[index, "parent_name"] = parent_contract_name - - return df - - -# # molina -def parent_child_mapping(cleaned_df): - grouper_column = "TIN" - logging.info( - f"Unique Contract Count: {cleaned_df['contract_name_cleaned'].nunique()}" - ) - tin_present = create_df_per_tin_presence(cleaned_df, True) - logging.info( - f"Contract Count after dropping contracts w/o TINs: {tin_present.shape}" - ) - tin_present = filter_on_exclude_keywords(tin_present, True) - tin_present = filter_on_missing_title(tin_present) - logging.info( - f"Contract Count after filtering on exclude keywords and missing title: {tin_present.shape}" - ) - # tin_present = flag_duplicates_within_group(tin_present, grouper_col=grouper_column) - tin_present = rank_parents(tin_present, grouper_col=grouper_column) - tin_present = flag_multi_parent_with_early_dates( - tin_present, grouper_col=grouper_column - ) - tin_present = assign_parents_with_state_and_payer( - tin_present, grouper_col=grouper_column - ) - tin_present = assign_no_parent_by_title(tin_present, grouper_col=grouper_column) - tin_present = assign_orphans_based_on_unique_parent_and_payer( - tin_present, grouper_col="TIN", payer_col="payer_name_cleaned" - ) - tin_present = assign_child_ranks(tin_present, grouper_col=grouper_column) - tin_present["numbering"] = tin_present.apply(extract_ordinal, axis=1) - - tin_present = tin_present[ - [ - "FILE_NAME", - "CONTRACT_TITLE", - "PROV_GROUP_NAME_FULL", - "PAYER_STATE", - "contract_name_cleaned", - "contract_title_cleaned", - "TIN", - "fixed_effective_date", - "AARETE_DERIVED_TERMINATION_DT", - "payer_name_cleaned", - "PROV_GROUP_NAME_FULL_cleaned", - "consolidated_lob", - "consolidated_service_term", - "parent", - "combined_rank", - "numbering", - ] - ] - - tin_present["parent_child_flag"] = tin_present["parent"].map( - {True: "Parent", False: "Child"} - ) - tin_present.loc[ - (tin_present["parent"] == False) - & (tin_present["combined_rank"].astype(str).str.count(r"\.") == 1), - "parent_child_flag", - ] = "Orphan" - - tin_present = clean_termination_date(tin_present, "AARETE_DERIVED_TERMINATION_DT") - - cols_order = [ - "TIN", - "PROV_GROUP_NAME_FULL_cleaned", - "consolidated_lob", - "fixed_effective_date", - "parent_child_flag", - "combined_rank", - ] - - tin_present["naming_col"] = ( - tin_present[cols_order] - .replace("", "xx") - .fillna("xx") - .astype(str) - .agg("___".join, axis=1) - ) - - tin_present["FILE_NAME"] = tin_present["FILE_NAME"].str.replace("Filename: ", "") - - logging.info(f"tin present: {tin_present.columns}") - - tin_present = assign_parent_name_by_rank(tin_present) - - tin_present = tin_present[ - [ - "FILE_NAME", - "CONTRACT_TITLE", - "PROV_GROUP_NAME_FULL", - "PAYER_STATE", - "contract_name_cleaned", - "contract_title_cleaned", - "TIN", - "fixed_effective_date", - "fixed_termination_date", - "payer_name_cleaned", - "PROV_GROUP_NAME_FULL_cleaned", - "consolidated_lob", - "consolidated_service_term", - "parent", - "combined_rank", - "numbering", - "parent_child_flag", - "parent_name", - "naming_col", - ] - ] - - return tin_present diff --git a/src/parent_child/pipeline.py b/src/parent_child/pipeline.py new file mode 100644 index 0000000..cc1ce27 --- /dev/null +++ b/src/parent_child/pipeline.py @@ -0,0 +1,1127 @@ +from __future__ import annotations + +import pandas as pd +import re +from pathlib import Path +from src.parent_child.preprocessing import ( + clean_effective_date, + clean_termination_date, +) +import numpy as np +import logging +from src.utils import io_utils +from src.constants.parent_child import ( + GROUPING_KEY_SEPARATOR, + GROUPING_KEY_PREFIXES, + TIER_ALL_IDENTIFIERS, + TIER_TWO_IDENTIFIERS, + TIER_ONE_IDENTIFIER, + TIER_NO_IDENTIFIERS, + ASSIGNMENT_NO_PARENT, +) +from typing import Tuple, Optional, List, Dict, Any + +# Build paths relative to this module's location +_MODULE_DIR = Path(__file__).resolve().parent +_CONSTANTS_DIR = _MODULE_DIR.parent / "constants" / "mappings" + + +def create_grouping_key_and_tier(df): + """ + Creates grouping key based on available non-null values in PROV_GROUP_TIN, + PROV_GROUP_NPI, and PROV_GROUP_NAME_FULL. + + Returns df with added columns: + - grouping_key: String key for grouping (e.g., "TIN:123|NPI:456|NAME:ABC") + - grouping_tier: Integer (1=all 3, 2=any 2, 3=any 1, 0=none/immediate orphan) + - grouping_columns: List of column names used (for reasoning) + """ + df = df.copy() + + # Create helper columns for non-null and non-empty checks + has_tin = df["PROV_GROUP_TIN"].notna() & (df["PROV_GROUP_TIN"] != "") + has_npi = df["PROV_GROUP_NPI"].notna() & (df["PROV_GROUP_NPI"] != "") + has_name = df["PROV_GROUP_NAME_FULL_cleaned"].notna() & ( + df["PROV_GROUP_NAME_FULL_cleaned"] != "" + ) + + # Initialize columns + df["grouping_key"] = None + df["grouping_tier"] = 0 + df["grouping_columns"] = None + + # Function to build grouping key + def build_key(row, idx): + parts = [] + cols_used = [] + + if has_tin.iloc[idx]: + parts.append(f"{GROUPING_KEY_PREFIXES['TIN']}{row['PROV_GROUP_TIN']}") + cols_used.append("TIN") + + if has_npi.iloc[idx]: + parts.append(f"{GROUPING_KEY_PREFIXES['NPI']}{row['PROV_GROUP_NPI']}") + cols_used.append("NPI") + + if has_name.iloc[idx]: + parts.append( + f"{GROUPING_KEY_PREFIXES['NAME']}{row['PROV_GROUP_NAME_FULL_cleaned']}" + ) + cols_used.append("PROV_NAME") + + if not parts: + return None, TIER_NO_IDENTIFIERS, None + + key = GROUPING_KEY_SEPARATOR.join(parts) + num_cols = len(parts) + if num_cols == 3: + tier = TIER_ALL_IDENTIFIERS # Best tier + elif num_cols == 2: + tier = TIER_TWO_IDENTIFIERS # Middle tier + elif num_cols == 1: + tier = TIER_ONE_IDENTIFIER # Worst tier + else: + tier = TIER_NO_IDENTIFIERS + + return key, tier, cols_used + + # Apply to all rows with index + for idx, row in df.iterrows(): + key, tier, cols = build_key(row, df.index.get_loc(idx)) + df.at[idx, "grouping_key"] = key + df.at[idx, "grouping_tier"] = tier + df.at[idx, "grouping_columns"] = cols + + return df + + +def build_grouping_string(child_row, grouping_cols): + """ + Builds a formatted string showing which grouping columns and their values were used. + + Parameters: + ----------- + child_row : pd.Series + The child row containing grouping column values + grouping_cols : list + List of grouping column names (e.g., ['TIN', 'NPI', 'PROV_NAME']) + + Returns: + -------- + str + Formatted string like "TIN (123456) and NPI (789012)" + """ + if not grouping_cols: + return "grouping" + + parts = [] + for col in grouping_cols: + # Map column name to actual DataFrame column + if col == "TIN": + value = child_row.get("PROV_GROUP_TIN", "N/A") + elif col == "NPI": + value = child_row.get("PROV_GROUP_NPI", "N/A") + elif col == "PROV_NAME": + value = child_row.get("PROV_GROUP_NAME_FULL", "N/A") + else: + value = "N/A" + + # Format the value properly (handle None, NaN, floats, etc.) + if pd.isna(value) or value is None: + formatted_value = "N/A" + elif isinstance(value, float): + # If it's a whole number float, show as int + if value == int(value): + formatted_value = str(int(value)) + else: + formatted_value = str(value) + else: + formatted_value = str(value) + + parts.append(f"{col} ({formatted_value})") + + return " and ".join(parts) + + +def check_grouping_compatibility(parent_key, child_key): + """ + Checks if a parent's grouping key is compatible with a child's grouping key. + + Rules: + - Extract values from both keys + - Compatible if they share AT LEAST ONE matching value + - This allows cross-tier matching (e.g., Tier 1 parent can match Tier 3 child via TIN) + + Examples: + Parent: "TIN:123|NPI:456|NAME:ABC" + Child: "TIN:123|NPI:456" + Result: ✅ Compatible (share TIN and NPI) + + Parent: "TIN:123|NPI:456|NAME:ABC" + Child: "TIN:123|NPI:999" + Result: ✅ Compatible (share TIN:123, even though NPI differs) + + Parent: "TIN:123|NPI:456|NAME:ABC" + Child: "TIN:999" + Result: ❌ NOT Compatible (no shared values) + """ + if pd.isna(parent_key) or pd.isna(child_key): + return False + + # Parse keys into dicts + def parse_key(key): + parts = str(key).split(GROUPING_KEY_SEPARATOR) + result = {} + for part in parts: + if ":" in part: + k, v = part.split(":", 1) + result[k] = v + return result + + parent_dict = parse_key(parent_key) + child_dict = parse_key(child_key) + + # Check if they share AT LEAST ONE matching value + for key, child_value in child_dict.items(): + parent_value = parent_dict.get(key) + if parent_value == child_value: + return True # Found at least one match! + + return False # No matching values found + + +def create_df_per_tin_presence(df, is_tin_present): + """ + Legacy function - kept for compatibility but now handles grouping_tier instead. + Returns rows that are not immediate orphans (tier > 0). + """ + return df[df["grouping_tier"] > 0].copy() + + +def filter_on_exclude_keywords(df, is_tin_present): + """ + Flags rows as 'parent' based on exclusion keywords. + """ + df = df.copy() + df["parent"] = True + + exclude_keywords_dict = io_utils.convert_json_to_dict( + str(_CONSTANTS_DIR / "exclude_keywords_for_pc.json") + ) + exclude_keywords_list = exclude_keywords_dict[is_tin_present] + exclude_pattern = ( + r"\b(?:" + "|".join(re.escape(kw) for kw in exclude_keywords_list) + r")\b" + ) + + df["parent"] = ~( + df["contract_title_cleaned"].str.contains(exclude_pattern, case=False, na=False) + ) + return df + + +def rank_parents(df, grouper_col="grouping_key"): + """ + Assigns a rank to 'parent' contracts within each group defined by `grouper_col`. + + Modified to work with cascading tiers: + - Ranks parents within their grouping_key + - Each group gets independent ranking (rank 1, 2, 3...) + """ + df = df.copy() + df["parent_rank"] = np.nan + + df["_sort_date"] = pd.to_datetime(df["fixed_effective_date"], errors="coerce") + + for group_val, group_df in df.groupby(grouper_col): + parents = group_df[group_df["parent"] == True].copy() + if parents.empty: + continue + + parents["_rank_date"] = parents["_sort_date"].fillna(pd.Timestamp.max) + parents = parents.sort_values("_rank_date") + ranks = range(1, len(parents) + 1) + + df.loc[parents.index, "parent_rank"] = list(ranks) + + return df.drop(columns=["_sort_date", "_rank_date"], errors="ignore") + + +def flag_multi_parent_with_early_dates(df, grouper_col="grouping_key"): + """ + Flags groups with multiple parents and early dates. + """ + df = df.copy() + df["fixed_effective_date"] = pd.to_datetime( + df["fixed_effective_date"], errors="coerce" + ) + flag_dict = {} + + for group_val, group_df in df.groupby(grouper_col): + parents_df = group_df[group_df["parent"] == True] + children_df = group_df[group_df["parent"] == False] + + unique_parents = parents_df["parent_rank"].dropna().unique() + if len(unique_parents) <= 1 or children_df.empty: + continue + + child_dates = children_df["fixed_effective_date"].dropna() + if child_dates.empty: + continue + + count_early_parents = sum( + pd.notna(parent_row["fixed_effective_date"]) + and (child_dates > parent_row["fixed_effective_date"]).any() + for _, parent_row in parents_df.iterrows() + ) + + if count_early_parents >= 2: + flag_dict[group_val] = "multiple_parents_early_children" + + df["flag_multi_parent_with_early_dates"] = df[grouper_col].map(flag_dict) + return df + + +def build_matching_columns_string(parent_row, child_row): + """ + Builds a formatted string showing ONLY the columns that matched between parent and child. + + Parameters: + ----------- + parent_row : pd.Series + The parent row + child_row : pd.Series + The child row + + Returns: + -------- + str + Formatted string like "TIN (123456)" or "PROV_NAME (Hospital ABC)" showing only matched columns + """ + matched_parts = [] + + # Check TIN + parent_tin = parent_row.get("PROV_GROUP_TIN") + child_tin = child_row.get("PROV_GROUP_TIN") + if pd.notna(parent_tin) and pd.notna(child_tin) and parent_tin == child_tin: + if isinstance(child_tin, float) and child_tin == int(child_tin): + matched_parts.append(f"TIN ({int(child_tin)})") + else: + matched_parts.append(f"TIN ({child_tin})") + + # Check NPI + parent_npi = parent_row.get("PROV_GROUP_NPI") + child_npi = child_row.get("PROV_GROUP_NPI") + if pd.notna(parent_npi) and pd.notna(child_npi) and parent_npi == child_npi: + if isinstance(child_npi, float) and child_npi == int(child_npi): + matched_parts.append(f"NPI ({int(child_npi)})") + else: + matched_parts.append(f"NPI ({child_npi})") + + # Check PROV_NAME + parent_name = parent_row.get("PROV_GROUP_NAME_FULL") + child_name = child_row.get("PROV_GROUP_NAME_FULL") + if pd.notna(parent_name) and pd.notna(child_name) and parent_name == child_name: + matched_parts.append(f"PROV_NAME ({child_name})") + + if matched_parts: + return " and ".join(matched_parts) + else: + return "shared identifier" + + +def assign_parents_with_state_and_payer_cross_tier( + combined_df, tier, flag_col="flag_multi_parent_with_early_dates" +): + """ + Modified version that supports cross-tier matching. + + Key changes: + - Parents from any tier can match children + - Compatibility check ensures grouping key alignment + - Assignment reasoning includes grouping columns used + """ + df = combined_df.copy() + df["fixed_effective_date"] = pd.to_datetime( + df["fixed_effective_date"], errors="coerce" + ) + + parents = df[df["parent"]].copy() + children = df[~df["parent"]].copy() + + # Only process children that don't have a parent yet + children_to_process = children[ + children.get("assigned_parent_rank", ASSIGNMENT_NO_PARENT) + == ASSIGNMENT_NO_PARENT + ].copy() + children_already_matched = children[ + children.get("assigned_parent_rank", ASSIGNMENT_NO_PARENT) + != ASSIGNMENT_NO_PARENT + ].copy() + + assignments = [] + reasoning_map = {} + parent_name_map = {} # Track parent names during assignment + + for idx, child in children_to_process.iterrows(): + child_grouping_key = child["grouping_key"] + child_grouping_cols = child.get("grouping_columns", []) + + # Step 1: Filter compatible parents based on grouping key compatibility + compatible_parents = [] + for p_idx, parent in parents.iterrows(): + if check_grouping_compatibility(parent["grouping_key"], child_grouping_key): + compatible_parents.append(parent) + + if not compatible_parents: + assignments.append((ASSIGNMENT_NO_PARENT, idx)) + reasoning_map[idx] = "No compatible parent found in any tier" + continue + + compatible_parents_df = pd.DataFrame(compatible_parents) + + # Step 2: Filter by matching state + state_parents = compatible_parents_df[ + compatible_parents_df["PAYER_STATE"] == child["PAYER_STATE"] + ] + + if state_parents.empty: + assignments.append((ASSIGNMENT_NO_PARENT, idx)) + grouping_str = build_grouping_string(child, child_grouping_cols) + reasoning_map[idx] = ( + f"No parent found matching {grouping_str} and state ({child['PAYER_STATE']})" + ) + continue + + # Step 3: Apply effective date constraint + if pd.notna(child["fixed_effective_date"]): + valid_date_parents = state_parents[ + pd.to_datetime(state_parents["fixed_effective_date"], errors="coerce") + <= child["fixed_effective_date"] + ] + + if valid_date_parents.empty: + assignments.append((ASSIGNMENT_NO_PARENT, idx)) + grouping_str = build_grouping_string(child, child_grouping_cols) + reasoning_map[idx] = ( + f"No parent with effective date on or before child date ({child['fixed_effective_date'].date()})" + ) + continue + + state_parents = valid_date_parents + + # Step 4: Tie-breaker on payer_name + if len(state_parents) > 1: + if pd.notna(child.get("payer_name_cleaned")): + payer_match = state_parents[ + state_parents["payer_name_cleaned"] == child["payer_name_cleaned"] + ] + if len(payer_match) == 1: + parent = payer_match.iloc[0] + parent_rank = parent["parent_rank"] + parent_name = parent["FILE_NAME"] + assignments.append((parent_rank, idx)) + grouping_str = build_matching_columns_string(parent, child) + reasoning_map[idx] = ( + f"Matched on {grouping_str}; state, date, and payer" + ) + parent_name_map[idx] = parent_name + continue + elif len(payer_match) > 1: + # Multiple parents with matching payer - apply title matching + child_title = str(child.get("contract_title_cleaned", "")).lower() + + title_matches = [] + for p_idx, parent in payer_match.iterrows(): + parent_title = str( + parent.get("contract_title_cleaned", "") + ).lower() + is_match = parent_title and parent_title in child_title + + if is_match: + title_matches.append(parent) + + if len(title_matches) > 0: + # PRIORITIZE TITLE MATCH: Select latest parent among those with title match + title_matches_df = pd.DataFrame(title_matches) + latest_parent = title_matches_df.sort_values( + "parent_rank", ascending=False + ).iloc[0] + parent_rank = latest_parent["parent_rank"] + parent_name = latest_parent["FILE_NAME"] + assignments.append((parent_rank, idx)) + grouping_str = build_matching_columns_string( + latest_parent, child + ) + + if len(title_matches) > 1: + reasoning_map[idx] = ( + f"Matched on {grouping_str}; state, date, payer, and title contains parent title; selected latest among {len(title_matches)} parents with title match (rank {parent_rank})" + ) + else: + reasoning_map[idx] = ( + f"Matched on {grouping_str}; state, date, payer, and title contains parent title (rank {parent_rank})" + ) + + parent_name_map[idx] = parent_name + else: + # No title match - fall back to latest parent rank + latest_parent = payer_match.sort_values( + "parent_rank", ascending=False + ).iloc[0] + parent_rank = latest_parent["parent_rank"] + parent_name = latest_parent["FILE_NAME"] + assignments.append((parent_rank, idx)) + grouping_str = build_matching_columns_string( + latest_parent, child + ) + reasoning_map[idx] = ( + f"Matched on {grouping_str}; state, date, and payer; no title match, selected latest parent (rank {parent_rank})" + ) + + parent_name_map[idx] = parent_name + continue + else: + assignments.append((ASSIGNMENT_NO_PARENT, idx)) + grouping_str = build_grouping_string(child, child_grouping_cols) + reasoning_map[idx] = ( + f"Multiple parents found but none match child's payer ({child['payer_name_cleaned']})" + ) + continue + else: + assignments.append((ASSIGNMENT_NO_PARENT, idx)) + reasoning_map[idx] = ( + "Multiple parents found but child has no payer information" + ) + continue + else: + parent = state_parents.iloc[0] + parent_rank = parent["parent_rank"] + parent_name = parent["FILE_NAME"] + assignments.append((parent_rank, idx)) + grouping_str = build_matching_columns_string(parent, child) + reasoning_map[idx] = ( + f"Matched on {grouping_str}; state and date (single parent available)" + ) + parent_name_map[idx] = parent_name + + # Apply assignments + children_to_process["assigned_parent_rank"] = None + children_to_process["assignment_reasoning"] = None + if "parent_name" not in children_to_process.columns: + children_to_process["parent_name"] = None + + for rank, idx in assignments: + children_to_process.at[idx, "assigned_parent_rank"] = rank + children_to_process.at[idx, "assignment_reasoning"] = reasoning_map.get( + idx, "Unknown" + ) + if idx in parent_name_map: + children_to_process.at[idx, "parent_name"] = parent_name_map[idx] + + # Combine all children back + all_children = pd.concat([children_already_matched, children_to_process]) + + return pd.concat([parents, all_children]).sort_index() + + +def assign_no_parent_by_title_cross_tier( + combined_df, tier, contract_title_col="contract_title_cleaned" +): + """ + Modified second-pass assignment with cross-tier matching support. + """ + if combined_df.empty: + logging.warning("Input DataFrame is empty") + return combined_df.copy() + + df_work = combined_df.copy() + + if "assignment_reasoning" not in df_work.columns: + df_work["assignment_reasoning"] = None + + df_work["fixed_effective_date"] = pd.to_datetime( + df_work["fixed_effective_date"], errors="coerce" + ) + + parents = df_work[df_work["parent"] == True].copy() + children_no_parent = df_work[ + df_work.get("assigned_parent_rank", ASSIGNMENT_NO_PARENT) + == ASSIGNMENT_NO_PARENT + ].copy() + + logging.info( + f"Tier {tier} - Title matching: Found {len(parents)} parents and {len(children_no_parent)} children with no_parent" + ) + + if len(parents) == 0 or len(children_no_parent) == 0: + return df_work + + assignments_made = 0 + + for idx, child in children_no_parent.iterrows(): + child_grouping_key = child["grouping_key"] + child_grouping_cols = child.get("grouping_columns", []) + + # Find compatible parents + compatible_parents = [] + for p_idx, parent in parents.iterrows(): + if check_grouping_compatibility(parent["grouping_key"], child_grouping_key): + compatible_parents.append(parent) + + if not compatible_parents: + continue + + possible_parents = pd.DataFrame(compatible_parents) + + child_date = child["fixed_effective_date"] + if pd.isna(child_date): + continue + + child_title = child[contract_title_col] + if not isinstance(child_title, str): + continue + + child_title_lower = child_title.lower() + + def title_match(parent_title): + if not isinstance(parent_title, str): + return False + return parent_title.lower() in child_title_lower + + title_matches = possible_parents[ + possible_parents[contract_title_col].apply(title_match) + ] + + if len(title_matches) == 0: + continue + + eligible = title_matches[title_matches["fixed_effective_date"] <= child_date] + + if not eligible.empty: + chosen = eligible.sort_values("parent_rank", ascending=False).iloc[0] + chosen_parent_rank = chosen["parent_rank"] + parent_title = chosen[contract_title_col] + parent_filename = chosen["FILE_NAME"] + + grouping_str = build_matching_columns_string(chosen, child) + + df_work.at[idx, "assigned_parent_rank"] = chosen_parent_rank + df_work.at[idx, "assignment_reasoning"] = ( + f"Title-based match: parent title '{parent_title}' is substring of child title; matched on {grouping_str} and date" + ) + df_work.at[idx, "parent_name"] = parent_filename + + assignments_made += 1 + + logging.info(f"Tier {tier} - Made {assignments_made} new title-based assignments") + return df_work + + +def assign_orphans_based_on_unique_parent_and_payer_cross_tier( + combined_df, tier, payer_col="payer_name_cleaned" +): + """ + Modified third-pass assignment with cross-tier matching support. + """ + df_work = combined_df.copy() + + if "assignment_reasoning" not in df_work.columns: + df_work["assignment_reasoning"] = None + + # Get all parents and orphans + all_parents = df_work[df_work["parent"] == True].copy() + orphans = df_work[ + df_work.get("assigned_parent_rank", ASSIGNMENT_NO_PARENT) + == ASSIGNMENT_NO_PARENT + ].copy() + + if orphans.empty: + logging.info(f"Tier {tier} - No orphans to assign") + return df_work + + logging.info( + f"Tier {tier} - Unique parent/payer: Processing {len(orphans)} orphans" + ) + + assignments_made = 0 + + for idx, orphan in orphans.iterrows(): + child_grouping_key = orphan["grouping_key"] + child_grouping_cols = orphan.get("grouping_columns", []) + + # Find compatible parents + compatible_parents = [] + for p_idx, parent in all_parents.iterrows(): + if check_grouping_compatibility(parent["grouping_key"], child_grouping_key): + compatible_parents.append(parent) + + if not compatible_parents: + continue + + possible_parents = pd.DataFrame(compatible_parents) + + # Check for unique parent ranks + unique_parent_ranks = possible_parents["parent_rank"].nunique() + unique_payer_names = possible_parents[payer_col].nunique(dropna=False) + + payer_value = ( + possible_parents[payer_col].iloc[0] if len(possible_parents) > 0 else None + ) + payer_display = payer_value if pd.notna(payer_value) else "Unknown" + + if unique_payer_names > 1: + continue + + child_effective_date = pd.to_datetime( + orphan["fixed_effective_date"], errors="coerce" + ) + + if unique_parent_ranks == 1: + parent = possible_parents.iloc[0] + parent_rank = parent["parent_rank"] + parent_filename = parent["FILE_NAME"] + parent_effective_date = pd.to_datetime( + parent["fixed_effective_date"], errors="coerce" + ) + grouping_str = build_matching_columns_string(parent, orphan) + + if ( + pd.isna(child_effective_date) + or parent_effective_date <= child_effective_date + ): + df_work.at[idx, "assigned_parent_rank"] = parent_rank + df_work.at[idx, "assignment_reasoning"] = ( + f"Matched on {grouping_str} and payer ({payer_display}) with unique parent in group; matched on date" + ) + df_work.at[idx, "parent_name"] = parent_filename + assignments_made += 1 + + elif unique_parent_ranks > 1: + sorted_parents = possible_parents.sort_values( + "parent_rank", ascending=False + ) + + for _, parent in sorted_parents.iterrows(): + parent_effective_date = pd.to_datetime( + parent["fixed_effective_date"], errors="coerce" + ) + + if ( + pd.isna(child_effective_date) + or parent_effective_date <= child_effective_date + ): + grouping_str = build_matching_columns_string(parent, orphan) + df_work.at[idx, "assigned_parent_rank"] = parent["parent_rank"] + df_work.at[idx, "assignment_reasoning"] = ( + f"Matched on {grouping_str} and payer ({payer_display}) with date; selected most recent parent among {unique_parent_ranks} parents in group" + ) + df_work.at[idx, "parent_name"] = parent["FILE_NAME"] + assignments_made += 1 + break + + logging.info( + f"Tier {tier} - Assigned {assignments_made} orphans based on unique parent and payer logic" + ) + return df_work + + +def assign_child_ranks(df, grouper_col="grouping_key"): + """ + Assigns sequential child ranks to non-parent contracts. + Modified to work with cross-tier matching where children may have different + grouping_keys than their parent. + + Format: parent_rank.0.child_sequence (e.g., 2.0.1, 2.0.2, 2.0.3) + """ + df = df.copy() + + # Ensure fixed_effective_date is datetime type to avoid sorting errors + df["fixed_effective_date"] = pd.to_datetime( + df["fixed_effective_date"], errors="coerce" + ) + + parents = df[df["parent"]].copy() + children = df[~df["parent"]].copy() + + # For each child, find their actual parent's identity + # Then group children by the parent they're actually assigned to + children["_parent_identity"] = None + + for idx, child_row in children.iterrows(): + assigned_rank = child_row.get("assigned_parent_rank") + + if assigned_rank == ASSIGNMENT_NO_PARENT or pd.isna(assigned_rank): + children.at[idx, "_parent_identity"] = "orphan" + else: + # Find any parent with this rank that's compatible with the child + found = False + for p_idx, parent_row in parents.iterrows(): + if parent_row["parent_rank"] == assigned_rank: + if check_grouping_compatibility( + parent_row["grouping_key"], child_row["grouping_key"] + ): + # Use parent's grouping_key + rank as unique identifier + children.at[idx, "_parent_identity"] = ( + f"{parent_row['grouping_key']}_{parent_row['parent_rank']}" + ) + found = True + break + + if not found: + children.at[idx, "_parent_identity"] = f"unknown_{assigned_rank}" + + # Group by parent identity and assign sequential ranks + for parent_id, grp_df in children.groupby("_parent_identity"): + if parent_id == "orphan": + base_rank = "0.0" + else: + # Extract parent_rank from the child's assigned_parent_rank + assigned_rank = grp_df.iloc[0]["assigned_parent_rank"] + # Convert to int if it's a float (2.0 -> 2), then add .0 + if pd.notna(assigned_rank) and assigned_rank != ASSIGNMENT_NO_PARENT: + try: + if isinstance(assigned_rank, float): + base_rank = f"{int(assigned_rank)}.0" + else: + base_rank = f"{assigned_rank}.0" + except Exception as e: + logging.debug( + f"Could not convert rank '{assigned_rank}' to int: {e}" + ) + base_rank = f"{assigned_rank}.0" + else: + base_rank = "0.0" + + # Sort by effective date (already converted to datetime at function start) + grp_df = grp_df.copy() + grp_df["sort_date"] = grp_df["fixed_effective_date"].fillna(pd.Timestamp.max) + grp_df = grp_df.sort_values("sort_date") + + # Assign sequential ranks + for i, idx in enumerate(grp_df.index, start=1): + rank_val = f"{base_rank}.{i}" + children.loc[idx, "child_rank"] = rank_val + + # Clean up temp column + children = children.drop(columns=["_parent_identity"]) + + result = pd.concat([parents, children]).sort_index() + result["child_rank"] = result["child_rank"].astype(str) + + def pick_rank(row): + if row.get("assigned_parent_rank") == ASSIGNMENT_NO_PARENT: + return row["child_rank"] + if pd.notna(row.get("parent_rank")): + # Format parent rank as integer if it's a float + rank = row["parent_rank"] + if isinstance(rank, float) and rank == int(rank): + return str(int(rank)) + return str(rank) + return row["child_rank"] + + result["combined_rank"] = result.apply(pick_rank, axis=1) + + return result + + +def extract_ordinal(row): + """ + Extracts ordinal numbers from contract titles. + """ + num_dict = io_utils.convert_json_to_dict( + str(_CONSTANTS_DIR / "numeric_mappings.json") + ) + word_to_num = num_dict.get("ordinal_word_to_number", {}) + cardinal_to_num = num_dict.get("cardinal_word_to_number", {}) + ordinal_pattern = num_dict.get("ordinal_regex_pattern", "") + + all_words_to_num = {**word_to_num, **cardinal_to_num} + text = str(row["contract_title_cleaned"]).lower() + + for word, num in all_words_to_num.items(): + if word in text: + return num + + num_match = re.search(ordinal_pattern, text) + if num_match: + value = re.sub(r"(st|nd|rd|th)$", "", num_match.group(0)) + if value.isdigit(): + return int(value) + + return None + + +def parent_child_mapping( + cleaned_df: pd.DataFrame, original_df: pd.DataFrame +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """ + Main function implementing cascading cross-tier parent-child mapping. + + Process: + 1. Create grouping keys for all rows + 2. Identify immediate orphans (all 3 columns null) + 3. Identify and rank all parents + 4. Process children tier-by-tier (Tier 1 → Tier 2 → Tier 3) + 5. Each tier runs 3 passes: state/payer, title, unique parent + 6. Finalize rankings and flags + + Args: + cleaned_df: Preprocessed DataFrame from parent_child_preprocessing + original_df: Original DataFrame for merging PC results back + + Returns: + tuple: (original_df_with_pc, pc_df) where original_df_with_pc has PC flags + and pc_df contains detailed PC mapping information + """ + logging.info(f"Starting cascading parent-child mapping") + logging.info( + f"Unique Contract Count: {cleaned_df['contract_name_cleaned'].nunique()}" + ) + + # Phase 0: Create grouping keys for all rows + cleaned_df = create_grouping_key_and_tier(cleaned_df) + logging.info( + f"Grouping keys created. Tier distribution:\n{cleaned_df['grouping_tier'].value_counts().sort_index()}" + ) + + # Phase 1: Separate immediate orphans (tier 0) + immediate_orphans = cleaned_df[cleaned_df["grouping_tier"] == 0].copy() + immediate_orphans["parent"] = False + immediate_orphans["assigned_parent_rank"] = ASSIGNMENT_NO_PARENT + immediate_orphans["assignment_reasoning"] = ( + "Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null" + ) + + workable_df = cleaned_df[cleaned_df["grouping_tier"] > 0].copy() + logging.info( + f"Immediate orphans: {len(immediate_orphans)}, Workable rows: {len(workable_df)}" + ) + + # Phase 2: Identify parents and rank them + workable_df = filter_on_exclude_keywords(workable_df, True) + logging.info(f"After exclude keywords and title filtering: {workable_df.shape}") + + # Rank all parents once (within their groups) + workable_df = rank_parents(workable_df, grouper_col="grouping_key") + workable_df = flag_multi_parent_with_early_dates( + workable_df, grouper_col="grouping_key" + ) + + parent_count = workable_df["parent"].sum() + child_count = (~workable_df["parent"]).sum() + logging.info(f"Parents identified: {parent_count}, Children: {child_count}") + + # Initialize assignment columns for children + workable_df.loc[~workable_df["parent"], "assigned_parent_rank"] = ( + ASSIGNMENT_NO_PARENT + ) + workable_df.loc[~workable_df["parent"], "assignment_reasoning"] = None + + # Phase 3: Tier 1 Processing (all 3 columns non-null) + logging.info("=" * 80) + logging.info("TIER 1 PROCESSING: Children with TIN, NPI, and PROV_NAME") + logging.info("=" * 80) + + tier1_mask = (workable_df["grouping_tier"] == 1) & (~workable_df["parent"]) + tier1_count = tier1_mask.sum() + logging.info(f"Tier 1 children to process: {tier1_count}") + + if tier1_count > 0: + workable_df = assign_parents_with_state_and_payer_cross_tier( + workable_df, tier=1 + ) + workable_df = assign_no_parent_by_title_cross_tier(workable_df, tier=1) + workable_df = assign_orphans_based_on_unique_parent_and_payer_cross_tier( + workable_df, tier=1 + ) + + matched = ( + (workable_df["grouping_tier"] == 1) + & (~workable_df["parent"]) + & (workable_df["assigned_parent_rank"] != ASSIGNMENT_NO_PARENT) + ) + logging.info(f"Tier 1 - Matched: {matched.sum()} / {tier1_count}") + + # Phase 4: Tier 2 Processing (any 2 columns non-null) + logging.info("=" * 80) + logging.info("TIER 2 PROCESSING: Children with any 2 of (TIN, NPI, PROV_NAME)") + logging.info("=" * 80) + + tier2_mask = ( + (workable_df["grouping_tier"] == 2) + & (~workable_df["parent"]) + & (workable_df["assigned_parent_rank"] == ASSIGNMENT_NO_PARENT) + ) + tier2_count = tier2_mask.sum() + logging.info(f"Tier 2 children to process: {tier2_count}") + + if tier2_count > 0: + workable_df = assign_parents_with_state_and_payer_cross_tier( + workable_df, tier=2 + ) + workable_df = assign_no_parent_by_title_cross_tier(workable_df, tier=2) + workable_df = assign_orphans_based_on_unique_parent_and_payer_cross_tier( + workable_df, tier=2 + ) + + # Before tier processing + unmatched_before = tier2_mask.sum() + + # After tier processing + unmatched_after = ( + (workable_df["grouping_tier"] == 2) + & (~workable_df["parent"]) + & (workable_df["assigned_parent_rank"] == ASSIGNMENT_NO_PARENT) + ) + matched_in_tier2 = unmatched_before - unmatched_after.sum() + + logging.info(f"Tier 2 - Matched: {matched_in_tier2} / {unmatched_before}") + + # Phase 5: Tier 3 Processing (exactly 1 column non-null) + logging.info("=" * 80) + logging.info("TIER 3 PROCESSING: Children with only 1 of (TIN, NPI, PROV_NAME)") + logging.info("=" * 80) + + tier3_mask = ( + (workable_df["grouping_tier"] == 3) + & (~workable_df["parent"]) + & (workable_df["assigned_parent_rank"] == ASSIGNMENT_NO_PARENT) + ) + tier3_count = tier3_mask.sum() + logging.info(f"Tier 3 children to process: {tier3_count}") + + if tier3_count > 0: + workable_df = assign_parents_with_state_and_payer_cross_tier( + workable_df, tier=3 + ) + workable_df = assign_no_parent_by_title_cross_tier(workable_df, tier=3) + workable_df = assign_orphans_based_on_unique_parent_and_payer_cross_tier( + workable_df, tier=3 + ) + + matched = ( + (workable_df["grouping_tier"] == 3) + & (~workable_df["parent"]) + & (workable_df["assigned_parent_rank"] != ASSIGNMENT_NO_PARENT) + ) + logging.info(f"Tier 3 - Matched: {matched.sum()} / {tier3_count}") + + # Phase 6: Combine with immediate orphans and finalize + pc_df = pd.concat([workable_df, immediate_orphans]).sort_index() + + # Assign child ranks + pc_df = assign_child_ranks(pc_df, grouper_col="grouping_key") + pc_df["numbering"] = pc_df.apply(extract_ordinal, axis=1) + + # Set parent_child_flag + pc_df["parent_child_flag"] = pc_df["parent"].map({True: "Parent", False: "Child"}) + + # Mark as Orphan if: + # 1. Assigned parent rank is 'no_parent' (unmatched children or immediate orphans) + # 2. OR combined_rank starts with "0." (orphan child ranks) + pc_df.loc[ + (pc_df["parent"] == False) + & ( + (pc_df["assigned_parent_rank"] == ASSIGNMENT_NO_PARENT) + | (pc_df["combined_rank"].astype(str).str.startswith("0.")) + ), + "parent_child_flag", + ] = "Orphan" + + # Clear reasoning for parents only (keep reasoning for orphans and children) + pc_df.loc[pc_df["parent_child_flag"] == "Parent", "assignment_reasoning"] = None + + # Clean dates - use column that exists in the dataframe + termination_col = ( + "TERMINATION_DATE" + if "TERMINATION_DATE" in pc_df.columns + else "AARETE_DERIVED_TERMINATION_DT" + ) + if termination_col in pc_df.columns: + pc_df = clean_termination_date(pc_df, termination_col) + + # Create naming column + cols_order = [ + "grouping_key", + "PROV_GROUP_NAME_FULL_cleaned", + "consolidated_lob", + "fixed_effective_date", + "parent_child_flag", + "combined_rank", + ] + + # Filter cols_order to only include columns that exist + cols_order = [col for col in cols_order if col in pc_df.columns] + + pc_df["naming_col"] = ( + pc_df[cols_order] + .replace("", "xx") + .fillna("xx") + .astype(str) + .agg("___".join, axis=1) + ) + + pc_df["FILE_NAME"] = pc_df["FILE_NAME"].str.replace("Filename: ", "") + + # Final summary + logging.info("=" * 80) + logging.info("FINAL SUMMARY") + logging.info("=" * 80) + logging.info(f"Total rows: {len(pc_df)}") + logging.info( + f"Parent-Child Flag distribution:\n{pc_df['parent_child_flag'].value_counts()}" + ) + logging.info( + f"Tier distribution:\n{pc_df['grouping_tier'].value_counts().sort_index()}" + ) + + cols_to_keep = [ + "contract_name_cleaned", + "fixed_effective_date", + "parent_child_flag", + "parent_name", + ] + + # Ensure consistent types for merge - convert both to string to avoid datetime/object mismatch + pc_df["fixed_effective_date"] = pc_df["fixed_effective_date"].astype(str) + original_df["fixed_effective_date"] = original_df["fixed_effective_date"].astype( + str + ) + + # Perform the left merge to keep all rows from original_df + # and join on both contract_name_cleaned and fixed_effective_date + original_df_with_pc = original_df.merge( + pc_df[cols_to_keep], + on=["contract_name_cleaned", "fixed_effective_date"], + how="left", + ) + + original_df_with_pc = original_df_with_pc.drop( + columns=["contract_name_cleaned", "fixed_effective_date"] + ) + + # Build pc_df output columns dynamically based on available columns + cols_to_keep_pc_tab = [ + "FILE_NAME", + "fixed_effective_date", + "contract_title_cleaned", + "PROV_GROUP_NAME_FULL_cleaned", + "PROV_GROUP_TIN", + "PROV_GROUP_NPI", + "payer_name_cleaned", + "PAYER_STATE", + "grouping_key", + "parent", + "combined_rank", + "numbering", + "parent_child_flag", + "assignment_reasoning", + "parent_name", + ] + + # Filter to only columns that exist + cols_to_keep_pc_tab = [col for col in cols_to_keep_pc_tab if col in pc_df.columns] + + pc_df = pc_df[cols_to_keep_pc_tab] + + return original_df_with_pc, pc_df diff --git a/src/parent_child/parent_child_preprocessing.py b/src/parent_child/preprocessing.py similarity index 53% rename from src/parent_child/parent_child_preprocessing.py rename to src/parent_child/preprocessing.py index ff40bfa..9730df1 100644 --- a/src/parent_child/parent_child_preprocessing.py +++ b/src/parent_child/preprocessing.py @@ -1,23 +1,32 @@ -import warnings +from __future__ import annotations -warnings.filterwarnings("ignore") import pandas as pd import nltk import ssl import re import string import src.config as config +from pathlib import Path from datetime import datetime import logging from src.utils import io_utils +from collections import defaultdict +from typing import Optional, List, Tuple, TYPE_CHECKING -stop_words_dict = io_utils.convert_json_to_dict( - "src/constants/mappings/stop_words_for_pc_cleanup.json" +if TYPE_CHECKING: + from src.parent_child.column_mapper import ColumnMapper + +# Build path relative to this module's location +_MODULE_DIR = Path(__file__).resolve().parent +_STOP_WORDS_PATH = ( + _MODULE_DIR.parent / "constants" / "mappings" / "stop_words_for_pc_cleanup.json" ) +stop_words_dict = io_utils.convert_json_to_dict(str(_STOP_WORDS_PATH)) stop_words = stop_words_dict["stop_words"] -def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_NAME +def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): + """Fixing FILE_NAME by removing prefixes and extensions.""" all_fields_df["contract_name_cleaned"] = ( all_fields_df[contact_name_col] .str.replace(r"Filename: ?", "", regex=True) @@ -29,9 +38,6 @@ def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_ return all_fields_df -# ... rest of the file remains the same - - def create_one_to_one_df(df): """ Create a DataFrame with AC Output columns. @@ -41,7 +47,7 @@ def create_one_to_one_df(df): def clean_contract_title(one_to_one_df, contact_name_col="CONTRACT_TITLE"): - # Cleaning contract title + """Cleaning contract title by lowercasing, removing 'inc', and stop words.""" one_to_one_df["contract_title_cleaned"] = ( one_to_one_df[contact_name_col] .astype(str) @@ -81,7 +87,8 @@ def clean_termination_date( # Try to convert anything else try: return pd.to_datetime(x).strftime("%m/%d/%Y") - except: + except Exception as e: + logging.debug(f"Could not parse date value '{x}': {e}") return x one_to_one_df["fixed_termination_date"] = one_to_one_df[termination_date_col].apply( @@ -127,50 +134,287 @@ def clean_payer_name(text, hit_words): # Remove any extra spaces that might have been created text = re.sub(r"\s+", " ", text).strip() - # standadrizing for molina healthcare - # if text == 'molina': - # return 'molina healthcare' return text -def clean_PROV_GROUP_NAME_FULL( - name, cutoff_words=["inc", "ltd", "llc", "corp", "corporation", "incorporated"] +def create_group_column( + df, + provider_col="PROV_GROUP_NAME_FULL", + group_col="PROV_GROUP_NAME_FULL_cleaned", + dba_col="DBA_Name", ): """ - Cleans a string with the following steps: - 1. Lowercases everything - 2. Removes everything after the first occurrence of any word in cutoff_words - 3. Replaces all non-alphanumeric characters with a space - 4. Removes extra spaces + Creates a Group column from the provider column by: + 1. Removing non-alphanumeric characters (except spaces) + 2. Converting to lowercase + 3. Removing anything after 'llc', 'inc', 'dba', or 'd/b/a' (only as complete words) - Parameters - ---------- - name : str - Input string - cutoff_words : list[str], optional - List of keywords to cut off text after first occurrence (e.g. ['inc', 'llc', 'd/b/a']) + Also creates a DBA_Name column that extracts text after 'dba' or 'd/b/a' - Returns - ------- - str or None + Parameters: + ----------- + df : pandas.DataFrame + The dataframe containing the provider data + provider_col : str + Name of the provider column (default: 'provider') + group_col : str + Name of the new group column to create (default: 'provider Group') + dba_col : str + Name of the DBA name column to create (default: 'DBA_Name') + + Returns: + -------- + pandas.DataFrame + The dataframe with the new Group and DBA_Name columns added """ - if pd.isna(name): - return None - text = str(name).lower().strip() + def process_provider(provider): + if pd.isna(provider): + return "", "" - # Step 2: cutoff after first keyword match - if cutoff_words: - pattern = r"\b(" + "|".join(map(re.escape, cutoff_words)) + r")\b.*" - text = re.sub(pattern, "", text) + # Step 1: Remove non-alphanumeric characters except spaces + cleaned = re.sub(r"[^a-zA-Z0-9\s]", "", str(provider)) - # Step 3: replace non-alphanumeric with space - text = re.sub(r"[^a-z0-9]", " ", text) + # Step 2: Convert to lowercase + cleaned = cleaned.lower() - # Step 4: collapse spaces - text = re.sub(r"\s+", " ", text).strip() + # Step 3: Extract DBA name if it exists (as a complete word) + dba_name = "" + dba_match = re.search(r"\bdba\b", cleaned) - return text + if dba_match: + dba_pos = dba_match.start() + # Extract everything after 'dba' + after_dba = cleaned[dba_pos + 3 :].strip() + + # Remove anything after 'llc' or 'inc' in the DBA portion (as complete words) + for pattern in [ + "inc", + "ltd", + "corp", + "corporation", + "incorporated", + "lp", + "llc", + "inc", + "llp", + "pty", + ]: + match = re.search(r"\b" + pattern + r"\b", after_dba) + if match: + after_dba = after_dba[: match.start()] + break + + # Clean up extra whitespace + dba_name = " ".join(after_dba.split()).strip() + + # Step 4: Remove anything after llc, inc, or dba for the group name + # Find the earliest occurrence of these terms as complete words + patterns = [ + "dba", + "inc", + "ltd", + "corp", + "corporation", + "incorporated", + "lp", + "llc", + "inc", + "llp", + "pty", + ] + positions = [] + + for pattern in patterns: + # Use word boundaries to match complete words only + match = re.search(r"\b" + pattern + r"\b", cleaned) + if match: + positions.append(match.start()) + + # Handle multi-word patterns separately + multi_word_patterns = ["advisory services"] + for pattern in multi_word_patterns: + match = re.search(r"\b" + pattern + r"\b", cleaned) + if match: + positions.append(match.start()) + + if positions: + # Cut at the earliest position + min_pos = min(positions) + cleaned = cleaned[:min_pos] + + # Remove the word 'and' (as a complete word) + cleaned = re.sub(r"\band\b", "", cleaned) + cleaned = re.sub(r"\bassoc\b", "associates", cleaned) + cleaned = re.sub(r"\bcenters\b", "center", cleaned) + cleaned = re.sub(r"\bsolutions\b", "solution", cleaned) + cleaned = re.sub(r"\bsystems\b", "system", cleaned) + cleaned = re.sub(r"\bplans\b", "plan", cleaned) + cleaned = re.sub(r"\bservices\b", "service", cleaned) + cleaned = re.sub(r"\bcasemanagement\b", "case management", cleaned) + + # Remove extra whitespace + cleaned = " ".join(cleaned.split()) + + return cleaned.strip(), dba_name + + # Apply the function to create both columns + df[[group_col, dba_col]] = df[provider_col].apply( + lambda x: pd.Series(process_provider(x)) + ) + + return df + + +def update_group(df, group_col="PROV_GROUP_NAME_FULL_cleaned", dba_col="DBA_Name"): + """ + Updates provider Group values when a DBA_Name within the group matches any provider Group value. + This consolidates providers under their DBA name when the DBA operates as a separate entity. + + Logic: + - Group by provider Group + - Within each group, check if any DBA_Name matches ANY provider Group value in the entire dataframe + - If a match is found, update ALL rows in that group to use the matched DBA_Name + - Stop at the first match found within each group + + Parameters: + ----------- + df : pandas.DataFrame + The dataframe containing provider group and DBA name data + group_col : str + Name of the provider group column (default: 'provider Group') + dba_col : str + Name of the DBA name column (default: 'DBA_Name') + + Returns: + -------- + pandas.DataFrame + The dataframe with updated provider Group values + """ + + # Make a copy to avoid modifying the original + df = df.copy() + + # Get all unique provider Group values (excluding empty strings and NaN) + all_groups = set(df[group_col].dropna().unique()) + all_groups.discard("") # Remove empty string if present + + # Track updates for reporting + updates = [] + + # Group by provider Group + for group_name, group_df in df.groupby(group_col): + # Skip if group_name is NaN or empty + if pd.isna(group_name) or group_name == "": + continue + + # Check if any DBA_Name in this group matches a provider Group value + matched_dba = None + + for idx, row in group_df.iterrows(): + dba_name = row.get(dba_col, "") + + # Skip empty DBA names or if DBA equals current group + if not dba_name or dba_name == group_name: + continue + + # Check if this DBA name exists as a provider Group elsewhere + if dba_name in all_groups: + matched_dba = dba_name + break # Stop at first match + + # If a match was found, update ALL rows in this group + if matched_dba: + for idx in group_df.index: + df.at[idx, group_col] = matched_dba + updates.append( + { + "index": idx, + "original_group": group_name, + "updated_group": matched_dba, + "dba_name": matched_dba, + } + ) + + # Print summary of updates + if updates: + logging.info( + f"Updated {len(updates)} rows across {len(set(u['original_group'] for u in updates))} provider Groups:" + ) + # Group updates by original group for clearer reporting + updates_by_group = defaultdict(list) + for update in updates: + updates_by_group[update["original_group"]].append(update) + + for orig_group, group_updates in updates_by_group.items(): + logging.info( + f" Group '{orig_group}' → '{group_updates[0]['updated_group']}' ({len(group_updates)} rows updated)" + ) + else: + logging.info("No provider Group updates needed.") + + return df + + +def standardize_provider_groups(df, column_name="PROV_GROUP_NAME_FULL_cleaned"): + """ + Standardizes provider group names by finding shorter base names and + updating longer variations to match the shorter ones. + + Parameters: + ----------- + df : pandas.DataFrame + The dataframe containing provider group data + column_name : str + Name of the column containing provider groups (default: 'provider Group') + + Returns: + -------- + pandas.DataFrame + DataFrame with standardized provider group names + """ + # Create a copy to avoid modifying the original + df = df.copy() + + # Get unique provider groups and sort by length (shortest first) + unique_providers = df[column_name].dropna().unique() + sorted_providers = sorted(unique_providers, key=lambda x: len(x.split())) + + # Create a mapping dictionary + mapping = {} + + for provider in sorted_providers: + words = provider.lower().split() + + # Only proceed if there are 2 or more words + if len(words) >= 2: + # Get the first two words as potential base name + first_two_words = " ".join(words[:2]) + + # Check if this provider should be mapped to a shorter version + if provider not in mapping: + # Check all other providers to see if current one is a longer version + for base_provider in sorted_providers: + base_words = base_provider.lower().split() + + # Skip if same provider or base doesn't have at least 2 words + if base_provider == provider or len(base_words) < 2: + continue + + # Check if first two words of base match first two words of current + base_first_two = " ".join(base_words[:2]) + + if base_first_two == first_two_words: + # Current provider starts with same two words as base + # Map the longer one to the shorter one + if len(base_words) < len(words): + mapping[provider] = base_provider + break + + # Apply the mapping + df[column_name] = df[column_name].replace(mapping) + + return df def extract_tin_like(value): @@ -342,6 +586,8 @@ def drop_exact_matches( "- hdo -", "term - term", "- ros -", + "- cif -", + "- darf -", ] if exception_words is None: @@ -470,32 +716,74 @@ def drop_letter( return df -def parent_child_preprocessing(df_read): +def parent_child_preprocessing( + df_read: pd.DataFrame, col_mapper: Optional[ColumnMapper] = None +) -> Tuple[pd.DataFrame, pd.DataFrame]: + """ + Main preprocessing function that prepares data for parent-child mapping. + + Args: + df_read: Raw input DataFrame + col_mapper: ColumnMapper instance for automatic column detection + + Returns: + tuple: (cleaned_df, original_df) where cleaned_df is preprocessed + and original_df is the original with effective date cleaned + """ + # Use column mapper to rename columns to standardized names + if col_mapper: + df_read = col_mapper.rename_dataframe(df_read) + effective_date_col = "EFFECTIVE_DATE" + termination_date_col = "TERMINATION_DATE" + payer_name_col = "PAYER_NAME" + prov_group_col = "PROV_GROUP_NAME_FULL" + else: + effective_date_col = "AARETE_DERIVED_EFFECTIVE_DT" + termination_date_col = "AARETE_DERIVED_TERMINATION_DT" + payer_name_col = "PAYER_NAME" + prov_group_col = "PROV_GROUP_NAME_FULL" + all_fields_df = clean_filename(df_read) + one_to_one_df = clean_effective_date(all_fields_df, effective_date_col) + + # Keep original_df for later merge + original_df = one_to_one_df.copy() + one_to_one_df = create_one_to_one_df(all_fields_df) one_to_one_df = clean_contract_title(one_to_one_df) - one_to_one_df = clean_effective_date(one_to_one_df, "AARETE_DERIVED_EFFECTIVE_DT") - one_to_one_df["payer_name_cleaned"] = one_to_one_df["PAYER_NAME"].apply( + one_to_one_df["payer_name_cleaned"] = one_to_one_df[payer_name_col].apply( lambda x: clean_payer_name(str(x), ["inc", "llc", "dba"]) ) - one_to_one_df["PROV_GROUP_NAME_FULL_cleaned"] = one_to_one_df[ - "PROV_GROUP_NAME_FULL" - ].apply(clean_PROV_GROUP_NAME_FULL) - one_to_one_df["TIN"] = one_to_one_df["contract_name_cleaned"].apply( - extract_tin_like - ) - consolidated_one_to_n_fields = consolidate_one_to_n_fields( - "contract_name_cleaned", all_fields_df, ["LOB", "SERVICE_TERM"] - ) - df = pd.merge( - one_to_one_df, - consolidated_one_to_n_fields, - on="contract_name_cleaned", - how="left", - ) + + # Use create_group_column for provider name cleaning (from working code) + one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col) + one_to_one_df = update_group(one_to_one_df) + one_to_one_df = standardize_provider_groups(one_to_one_df) + + # Determine which fields to consolidate based on available columns + fields_to_consolidate = [] + if "LOB" in all_fields_df.columns: + fields_to_consolidate.append("LOB") + if "SERVICE_TERM" in all_fields_df.columns: + fields_to_consolidate.append("SERVICE_TERM") + + if fields_to_consolidate: + consolidated_one_to_n_fields = consolidate_one_to_n_fields( + "contract_name_cleaned", all_fields_df, fields_to_consolidate + ) + df = pd.merge( + one_to_one_df, + consolidated_one_to_n_fields, + on="contract_name_cleaned", + how="left", + ) + else: + df = one_to_one_df + df = drop_duplicates(df) df = drop_forms(df, partial_match=True) df = drop_exact_matches(df) df = drop_letter(df) logging.info(f"df after preprocessing: {df.columns}") - return df + + return df, original_df diff --git a/src/parent_child/qc.py b/src/parent_child/qc.py new file mode 100644 index 0000000..31388a9 --- /dev/null +++ b/src/parent_child/qc.py @@ -0,0 +1,2251 @@ +""" +Parent-Child Contract Mapping and Ranking System +""" + +import re +import sys +import time +import logging +import hashlib +import uuid +from collections import defaultdict +from typing import Dict, List, DefaultDict, Any, Optional, Tuple, Set, Callable +from datetime import datetime +from dataclasses import dataclass, field +from enum import Enum + +import numpy as np +import pandas as pd + +# ============================================================ +# Logging Configuration +# ============================================================ + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) + + +# ============================================================ +# Constants & Enums +# ============================================================ + + +class Strategy(Enum): + """Supported strategies for parent/child operations.""" + + KEYWORD = "keyword" + FIELD_VALUE = "field_value" + SOURCE_BOOLEAN = "source_boolean" + SOURCE_LABEL = "source_label" + SINGLE_FIELD = "single_field" + MULTI_IDENTIFIER = "multi_identifier" + + +# Legal Entity Suffixes +LEGAL_SUFFIXES = [ + "inc", + "inc.", + "llc", + "l.l.c.", + "corp", + "corp.", + "co", + "co.", + "ltd", + "ltd.", + "company", + "corporation", + "limited", + "dba", + "d/b/a", +] +SUFFIX_REGEX = re.compile( + r"\b(" + "|".join(re.escape(s) for s in LEGAL_SUFFIXES) + r")\b", + flags=re.IGNORECASE, +) + +# Keywords indicating child/amendment contracts +TITLE_KEYWORDS = [ + "amendment", + "addendum", + "exhibit", + "attachment", + "renewal", + "extension", + "modification", + "letter", + "rate letter", + "notice", + "notification", +] +TITLE_REGEX = re.compile(r"(?i)\b(" + "|".join(TITLE_KEYWORDS) + r")\b") + +# CareSource-specific keywords +CARESOURCE_NON_PARENT_KEYWORDS = [ + "addendum", + "amendment", + "tracking sheet", + "letter", + "notice", + "memo", + "form", + "term sheet", + "request", + "exhibit", + "attachment", + "renewal", + "extension", + "modification", +] + +# BCBS-specific keywords +BCBS_NON_PARENT_KEYWORDS = [ + "addendum", + "amendment", + "tracking sheet", + "letter", + "notice", + "memo", + "form", + "term sheet", + "request", + "exhibit", + "attachment", + "renewal", + "extension", + "modification", +] + +# Default stopwords for text cleaning +DEFAULT_STOPWORDS = [ + "the", + "a", + "an", + "and", + "or", + "for", + "of", + "to", + "in", + "on", + "at", + "by", + "with", + "agreement", + "contract", +] + +# Noise words for similarity calculations +NOISE_WORDS = { + "the", + "a", + "an", + "and", + "or", + "for", + "of", + "to", + "in", + "on", + "at", + "by", + "with", +} + + +# ============================================================ +# Utility Functions +# ============================================================ + + +def resolve_column( + df: pd.DataFrame, col_spec: Any, required: bool = False, label: str = "" +) -> Optional[str]: + """Resolve column name from various specifications.""" + if col_spec is None: + if required: + raise ValueError(f"Missing required column spec for {label}") + return None + + if isinstance(col_spec, (list, tuple)): + for c in col_spec: + if isinstance(c, str) and c in df.columns: + return c + if required: + raise ValueError( + f"None of candidate columns exist for {label}: {list(col_spec)}" + ) + return None + + if isinstance(col_spec, str): + if col_spec in df.columns: + return col_spec + if required: + raise ValueError(f"Required column missing for {label}: {col_spec}") + return None + + if required: + raise ValueError(f"Invalid column spec for {label}: {col_spec}") + return None + + +def to_string(series: pd.Series) -> pd.Series: + """Convert series to string, filling NaN with empty string.""" + return series.fillna("").astype("string") + + +def to_datetime(series: pd.Series) -> pd.Series: + """Convert series to datetime, coercing errors to NaT.""" + return pd.to_datetime(series, errors="coerce") + + +# ============================================================ +# Text Processing +# ============================================================ + + +class TextProcessor: + """Handles text cleaning and normalization.""" + + @staticmethod + def clean_text(series: pd.Series) -> pd.Series: + """Remove special chars, normalize whitespace.""" + text = to_string(series) + text = text.str.replace(r"[^0-9A-Za-z\s]+", " ", regex=True) + text = text.str.replace(r"\s+", " ", regex=True) + text = text.str.strip() + return text + + @staticmethod + def clean_text_lowercase(series: pd.Series) -> pd.Series: + """Clean text and convert to lowercase.""" + return TextProcessor.clean_text(series).str.lower() + + @staticmethod + def normalize_single_identifier(value: str) -> str: + """Normalize TIN/NPI identifier.""" + if not value: + return "" + normalized = re.sub(r"[^0-9A-Za-z]", "", value.strip()) + return normalized.upper() + + @staticmethod + def normalize_identifier(value: Any) -> str: + """Normalize TIN/NPI identifiers (handles pipe-separated values).""" + if value is None or pd.isna(value): + return "" + + val_str = str(value).strip() + if not val_str: + return "" + + if "|" in val_str: + parts = [ + TextProcessor.normalize_single_identifier(p) for p in val_str.split("|") + ] + seen = set() + unique_parts = [] + for p in parts: + if p and p not in seen: + seen.add(p) + unique_parts.append(p) + return "|".join(unique_parts) if unique_parts else "" + + return TextProcessor.normalize_single_identifier(val_str) + + @staticmethod + def normalize_identifier_series(series: pd.Series) -> pd.Series: + """Apply identifier normalization to series.""" + return series.apply(TextProcessor.normalize_identifier).astype("string") + + @staticmethod + def get_identifier_values(value: Any) -> Set[str]: + """Extract all individual identifier values (handles pipe-separated).""" + if value is None or pd.isna(value): + return set() + + val_str = str(value).strip() + if not val_str: + return set() + + if "|" in val_str: + parts = [ + TextProcessor.normalize_single_identifier(p) for p in val_str.split("|") + ] + return {p for p in parts if p} + + normalized = TextProcessor.normalize_single_identifier(val_str) + return {normalized} if normalized else set() + + @staticmethod + def normalize_label(series: pd.Series) -> pd.Series: + """Normalize parent/child/orphan labels.""" + text = series.astype("string").str.strip().str.lower() + variations = { + "parent": "Parent", + "child": "Child", + "orphan": "Orphan", + "standalone": "StandAlone", + "stand alone": "StandAlone", + "stand_alone": "StandAlone", + "stand-alone": "StandAlone", + } + text = text.replace(variations) + text = text.replace("", pd.NA) + return text + + @staticmethod + def get_word_tokens(text: Any) -> Set[str]: + """Extract and clean word tokens.""" + if text is None or pd.isna(text): + return set() + t = str(text).lower().strip() + if not t: + return set() + t = re.sub(r"[^0-9a-z\s]+", " ", t) + t = re.sub(r"\s+", " ", t).strip() + if not t: + return set() + tokens = set(t.split(" ")) + return {x for x in tokens if x and x not in NOISE_WORDS} + + @staticmethod + def calculate_text_similarity(text_a: Any, text_b: Any) -> float: + """Calculate text similarity as overlap ratio.""" + tokens_a = TextProcessor.get_word_tokens(text_a) + tokens_b = TextProcessor.get_word_tokens(text_b) + if not tokens_a or not tokens_b: + return 0.0 + overlap = len(tokens_a & tokens_b) + total = max(len(tokens_a), len(tokens_b)) + return float(overlap) / float(total) + + @staticmethod + def has_text_match(text_a: Any, text_b: Any) -> bool: + """Check if one text contains or equals the other (case-insensitive).""" + a = "" if (text_a is None or pd.isna(text_a)) else str(text_a) + b = "" if (text_b is None or pd.isna(text_b)) else str(text_b) + if not a or not b: + return False + a_lower = a.lower().strip() + b_lower = b.lower().strip() + return (a_lower in b_lower) or (b_lower in a_lower) + + +class TextCleaner: + """Field-specific text cleaning with configurable rules.""" + + def clean_field(self, field_name: str, series: pd.Series, rules: Dict) -> pd.Series: + """Apply field-specific cleaning rules.""" + cleaned = TextProcessor.clean_text_lowercase(series) + + fields = rules.get("fields", {}) if isinstance(rules, dict) else {} + field_rules = fields.get(field_name, {}) if isinstance(fields, dict) else {} + + # Remove stopwords + if field_rules.get("remove_stopwords"): + stopwords = set(field_rules.get("stopwords", [])) + if stopwords: + + def drop_stopwords(text: Any) -> str: + if text is None or pd.isna(text) or not str(text).strip(): + return "" + words = [w for w in str(text).split() if w not in stopwords] + return " ".join(words) + + cleaned = cleaned.apply(drop_stopwords) + + # Remove suffixes + if field_rules.get("remove_suffixes"): + suffixes = field_rules.get("suffixes", []) + if suffixes: + suffix_pattern = ( + r"\b(" + "|".join(re.escape(str(s)) for s in suffixes) + r")\b" + ) + cleaned = cleaned.str.replace(suffix_pattern, "", regex=True) + cleaned = cleaned.str.replace(r"\s+", " ", regex=True).str.strip() + + return cleaned + + +# ============================================================ +# Field Derivation +# ============================================================ + + +class FieldDeriver: + """Derives new fields from source data.""" + + def __init__(self): + self.pattern_cache: Dict[str, re.Pattern] = {} + + def get_cached_regex(self, pattern: str) -> re.Pattern: + """Get cached compiled regex.""" + if pattern not in self.pattern_cache: + self.pattern_cache[pattern] = re.compile(pattern) + return self.pattern_cache[pattern] + + def build_fields( + self, + df: pd.DataFrame, + derive_config: List[Dict], + text_cleaner: TextCleaner, + cleaning_rules: Dict, + ) -> pd.DataFrame: + """Apply field derivation rules to dataframe.""" + if not derive_config: + return df + + for rule in derive_config: + target_col = rule.get("target") + source_col = rule.get("source") + rule_type = rule.get("mode") + + if ( + not target_col + or not source_col + or source_col not in df.columns + or not rule_type + ): + continue + + source_data = df[source_col] + + if rule_type == "value_map": + self._apply_value_map(df, target_col, source_data, rule) + elif rule_type == "regex_map": + self._apply_regex_map(df, target_col, source_data, rule) + elif rule_type == "regex_contains_any": + self._apply_regex_contains(df, target_col, source_data, rule) + elif rule_type == "copy_clean": + self._apply_copy_clean( + df, target_col, source_data, rule, text_cleaner, cleaning_rules + ) + + return df + + def _apply_value_map( + self, df: pd.DataFrame, target_col: str, source_data: pd.Series, rule: Dict + ) -> None: + """Apply value mapping rule.""" + value_map = rule.get("map", {}) + default = rule.get("default", pd.NA) + normalized = to_string(source_data).str.lower().str.strip() + df[target_col] = ( + normalized.map(lambda x: value_map.get(str(x), default)) + .fillna(default) + .astype("string") + ) + + def _apply_regex_map( + self, df: pd.DataFrame, target_col: str, source_data: pd.Series, rule: Dict + ) -> None: + """Apply regex mapping rule.""" + patterns = rule.get("patterns", []) + default = rule.get("default", pd.NA) + text = to_string(source_data).str.lower() + + result = pd.Series([default] * len(df), index=df.index, dtype="string") + for pattern_rule in patterns: + pattern_str = pattern_rule.get("pattern") + value = pattern_rule.get("value") + if not pattern_str: + continue + compiled = self.get_cached_regex(pattern_str) + matches = text.str.contains(compiled, na=False, regex=True) + result.loc[matches & result.isna()] = value + df[target_col] = result + + def _apply_regex_contains( + self, df: pd.DataFrame, target_col: str, source_data: pd.Series, rule: Dict + ) -> None: + """Apply regex contains rule.""" + patterns = rule.get("patterns", []) + if patterns: + combined_pattern = "|".join(patterns) + text = to_string(source_data).str.lower() + df[target_col] = text.str.contains(combined_pattern, na=False, regex=True) + else: + df[target_col] = False + + def _apply_copy_clean( + self, + df: pd.DataFrame, + target_col: str, + source_data: pd.Series, + rule: Dict, + text_cleaner: TextCleaner, + cleaning_rules: Dict, + ) -> None: + """Apply copy and clean rule.""" + clean_type = rule.get("cleaner", "lower_clean") + if clean_type == "upper_clean": + df[target_col] = TextProcessor.clean_text( + to_string(source_data) + ).str.upper() + elif clean_type == "raw": + df[target_col] = to_string(source_data) + elif clean_type == "identifier": + df[target_col] = TextProcessor.normalize_identifier_series(source_data) + else: + df[target_col] = text_cleaner.clean_field( + target_col, to_string(source_data), cleaning_rules + ) + + +# ============================================================ +# Grouping +# ============================================================ + + +class GroupKeyBuilder: + """Builds grouping keys to identify contract families.""" + + def build_grouping_key( + self, df: pd.DataFrame, config: Dict + ) -> Tuple[pd.Series, pd.Series]: + """Build grouping key and tier level.""" + strategy_type = config.get("strategy_type", "single_field") + + if strategy_type == "single_field": + return self._build_single_field(df, config) + elif strategy_type == "multi_identifier": + return self._build_multi_identifier(df, config) + else: + raise ValueError(f"Unknown grouping strategy: {strategy_type}") + + def _build_single_field( + self, df: pd.DataFrame, config: Dict + ) -> Tuple[pd.Series, pd.Series]: + """Build grouping key from single field.""" + field = config.get("field") + if not field or field not in df.columns: + raise ValueError(f"Grouping field missing or not found: {field}") + grouping_key = df[field].astype("string") + tier = pd.Series([1] * len(df), index=df.index, dtype=np.int8) + return grouping_key, tier + + def _build_multi_identifier( + self, df: pd.DataFrame, config: Dict + ) -> Tuple[pd.Series, pd.Series]: + """Build grouping key from multiple identifiers.""" + identifiers = config.get("identifiers", []) + if not identifiers or not isinstance(identifiers, list): + raise ValueError("multi_identifier requires list 'identifiers'") + + # Ensure columns exist + for col in identifiers: + if col not in df.columns: + df[col] = pd.Series([pd.NA] * len(df), index=df.index, dtype="string") + + # Count identifiers present for each row + present = [ + df[col].fillna("").astype("string").ne("").to_numpy() for col in identifiers + ] + id_count = np.zeros(len(df), dtype=np.int8) + for has_value in present: + id_count += has_value.astype(np.int8) + + # Assign tier based on identifier count + tier = np.zeros(len(df), dtype=np.int8) + tier[id_count == len(identifiers)] = 1 + tier[(id_count >= 2) & (tier == 0)] = 2 + tier[(id_count == 1) & (tier == 0)] = 3 + + # Build keys for each tier + grouping_key = self._build_tier_keys(df, identifiers, tier) + + return grouping_key, pd.Series(tier, index=df.index, dtype=np.int8) + + def _build_tier_keys( + self, df: pd.DataFrame, identifiers: List[str], tier: np.ndarray + ) -> pd.Series: + """Build keys for each tier level.""" + grouping_key = pd.Series([pd.NA] * len(df), index=df.index, dtype="string") + + # Tier 1: all identifiers + mask_tier1 = tier == 1 + if mask_tier1.any(): + parts = [ + col + ":" + df.loc[mask_tier1, col].astype("string") + for col in identifiers + ] + grouping_key.loc[mask_tier1] = ( + parts[0] if len(parts) == 1 else parts[0].str.cat(parts[1:], sep="|") + ) + + # Tier 2: first 2 available identifiers + mask_tier2 = tier == 2 + if mask_tier2.any(): + keys = self._build_tier_keys_for_rows( + df[mask_tier2], identifiers, max_count=2 + ) + grouping_key.loc[mask_tier2] = keys + + # Tier 3: first available identifier + mask_tier3 = tier == 3 + if mask_tier3.any(): + keys = self._build_tier_keys_for_rows( + df[mask_tier3], identifiers, max_count=1 + ) + grouping_key.loc[mask_tier3] = keys + + return grouping_key + + @staticmethod + def _build_tier_keys_for_rows( + df_subset: pd.DataFrame, identifiers: List[str], max_count: int + ) -> List[str]: + """Build keys for a subset of rows with limited identifier count.""" + keys = [] + for idx in df_subset.index: + picked = [] + for col in identifiers: + val = df_subset.at[idx, col] + val_str = "" if (val is None or pd.isna(val)) else str(val) + if val_str: + picked.append(f"{col}:{val_str}") + if len(picked) == max_count: + break + keys.append("|".join(picked) if picked else pd.NA) + return keys + + +# ============================================================ +# Parent Identification +# ============================================================ + + +class ParentIdentifier: + """Identifies parent contracts using various strategies.""" + + def __init__(self): + self.pattern_cache: Dict[str, re.Pattern] = {} + + def find_parents(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """Identify parent contracts.""" + strategy_type = config.get("strategy_type", "keyword") + + if strategy_type == "keyword": + return self._identify_by_keyword(df, config) + elif strategy_type == "field_value": + return self._identify_by_field_value(df, config) + elif strategy_type == "source_boolean": + return self._identify_by_boolean(df, config) + elif strategy_type == "source_label": + return self._identify_by_label(df, config) + elif strategy_type == "bcbs_custom": + return self._identify_bcbs_custom(df, config) + else: + raise ValueError(f"Unknown parent identification strategy: {strategy_type}") + + def _identify_by_keyword(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """Identify parents by excluding keywords.""" + field = config.get("field") + exclude_keywords = config.get("exclude_keywords", []) + if not field or field not in df.columns: + raise ValueError(f"keyword strategy missing field: {field}") + if not exclude_keywords: + return pd.Series([True] * len(df), index=df.index) + + pattern_str = ( + r"\b(?:" + "|".join(re.escape(str(k)) for k in exclude_keywords) + r")\b" + ) + if pattern_str not in self.pattern_cache: + self.pattern_cache[pattern_str] = re.compile(pattern_str, re.IGNORECASE) + + compiled = self.pattern_cache[pattern_str] + has_keyword = ( + df[field].astype("string").str.contains(compiled, na=False, regex=True) + ) + return ~has_keyword + + def _identify_by_field_value(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """Identify parents by field value match.""" + field = config.get("field") + target_value = config.get("target_value") + if not field or field not in df.columns: + raise ValueError(f"field_value strategy missing field: {field}") + return df[field].astype("string") == str(target_value) + + def _identify_by_boolean(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """Identify parents from boolean field.""" + field = config.get("field") + if not field or field not in df.columns: + raise ValueError(f"source_boolean strategy missing field: {field}") + return df[field].fillna(False).astype(bool) + + def _identify_by_label(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """Identify parents from label field.""" + field = config.get("field") + if not field or field not in df.columns: + raise ValueError(f"source_label strategy missing field: {field}") + labels = TextProcessor.normalize_label(df[field]) + return (labels == "Parent").fillna(False) + + def _identify_bcbs_custom(self, df: pd.DataFrame, config: Dict) -> pd.Series: + """ + BCBS-specific parent identification matching original bcbs_parent_child_mapping.py logic. + + Rules: + 1. effective_date_rank == 1 AND title is not null -> Parent + 2. "agreement" in title but NOT "amendment/addendum" -> Parent + 3. LOB-based: has LOB AND title not null AND no "amendment/addendum" -> Parent + 4. Exclude if title/filename contains "amendment" or "addendum" + """ + title_col = config.get("field", "title_clean") + filename_col = config.get("filename_field", "id_clean") + grouping_key_col = "grouping_key" + eff_date_col = "eff_date" + lob_col = config.get("lob_field", "_lob") + + # Initialize all as non-parent + is_parent = pd.Series([False] * len(df), index=df.index) + + # Get title and ensure it's string + title = ( + df[title_col].fillna("").astype(str).str.lower() + if title_col in df.columns + else pd.Series([""] * len(df), index=df.index) + ) + filename = ( + df[filename_col].fillna("").astype(str).str.lower() + if filename_col in df.columns + else pd.Series([""] * len(df), index=df.index) + ) + + # Check for amendment/addendum in title or filename + amend_pattern = r"\b(amendment|addendum)\b" + has_amendment_title = title.str.contains(amend_pattern, na=False, regex=True) + has_amendment_filename = filename.str.contains( + amend_pattern, na=False, regex=True + ) + has_amendment = has_amendment_title | has_amendment_filename + + # Check for "agreement" in title + has_agreement = title.str.contains(r"\bagreement\b", na=False, regex=True) + + # Check for non-empty title + has_title = title.str.strip() != "" + + # Compute effective_date_rank within each group + if grouping_key_col in df.columns and eff_date_col in df.columns: + eff_date_rank = df.groupby(grouping_key_col)[eff_date_col].rank( + method="dense", ascending=True + ) + else: + eff_date_rank = pd.Series([1] * len(df), index=df.index) + + # Rule 1: effective_date_rank == 1 AND title not null -> Parent + case1 = (eff_date_rank == 1) & has_title + is_parent = is_parent | case1 + + # Rule 2: "agreement" in title but NOT "amendment/addendum" AND title not null -> Parent + case2 = has_agreement & ~has_amendment & has_title + is_parent = is_parent | case2 + + # Rule 3: LOB-based - has LOB AND title not null AND no "amendment/addendum" -> Parent + if lob_col in df.columns: + has_lob = df[lob_col].fillna("").astype(str).str.strip() != "" + case3 = has_lob & has_title & ~has_amendment + is_parent = is_parent | case3 + + # Final exclusion: if amendment/addendum is in title OR filename -> NOT Parent + is_parent = is_parent & ~has_amendment + + return is_parent + + +# ============================================================ +# Child Assignment (continues in next message due to length) +# ============================================================ + + +class ChildAssigner: + """Multi-pass child-to-parent assignment.""" + + def build_parent_index( + self, df: pd.DataFrame, index_fields: List[str] + ) -> Dict[str, DefaultDict[str, List[int]]]: + """Build parent lookup index by identifier fields.""" + maps: Dict[str, DefaultDict[str, List[int]]] = { + f: defaultdict(list) for f in index_fields + } + parents = df[df["is_parent"]] + + for f in index_fields: + if f not in df.columns: + continue + + for idx in parents.index: + val = parents.at[idx, f] + if val is None or pd.isna(val): + continue + + individual_values = TextProcessor.get_identifier_values(val) + for individual_val in individual_values: + if individual_val: + maps[f][individual_val].append(idx) + + return maps + + def find_candidates( + self, + parent_index: Dict[str, DefaultDict[str, List[int]]], + child_row: pd.Series, + index_fields: List[str], + ) -> List[int]: + """Find candidate parents for a child.""" + cand: Set[int] = set() + for f in index_fields: + if f not in child_row.index: + continue + v = child_row[f] + if v is None or pd.isna(v): + continue + + child_values = TextProcessor.get_identifier_values(v) + for individual_val in child_values: + if individual_val: + cand.update(parent_index[f].get(individual_val, [])) + + return list(cand) + + @staticmethod + def get_field_value(row: pd.Series, col: str) -> Any: + """Safely get field value from row.""" + if not col: + return None + return row.get(col, None) + + @staticmethod + def is_date_valid(parent_date: Any, child_date: Any) -> bool: + """Check if parent effective date is before or equal to child date.""" + parent_valid = pd.notna(parent_date) + child_valid = pd.notna(child_date) + + if parent_valid and child_valid: + return parent_date <= child_date + elif not parent_valid and child_valid: + return False + else: + return True + + def check_constraints( + self, + row: pd.Series, + parent: Dict[str, Any], + pass_config: Dict, + constraints: Dict, + ) -> bool: + """Verify assignment constraints between child and parent.""" + col_state = constraints.get("state") + col_payer = constraints.get("payer") + col_commodity = constraints.get("commodity") + col_eff = constraints.get("date") + col_lob = constraints.get("lob") + col_provider_type = constraints.get("provider_type") + + if pass_config.get("state") and col_state: + val_a = self.get_field_value(row, col_state) + val_b = parent.get(col_state) + if ( + pd.notna(val_a) + and pd.notna(val_b) + and str(val_a) + and str(val_b) + and str(val_a) != str(val_b) + ): + return False + + if pass_config.get("payer") and col_payer: + val_a = self.get_field_value(row, col_payer) + val_b = parent.get(col_payer) + if ( + pd.notna(val_a) + and pd.notna(val_b) + and str(val_a) + and str(val_b) + and str(val_a) != str(val_b) + ): + return False + + if pass_config.get("commodity") and col_commodity: + val_a = self.get_field_value(row, col_commodity) + val_b = parent.get(col_commodity) + if ( + pd.notna(val_a) + and pd.notna(val_b) + and str(val_a) + and str(val_b) + and str(val_a) != str(val_b) + ): + return False + + # LOB matching (for BCBS: extracted_lob must match) + if pass_config.get("lob") and col_lob: + val_a = self.get_field_value(row, col_lob) + val_b = parent.get(col_lob) + if ( + pd.notna(val_a) + and pd.notna(val_b) + and str(val_a) + and str(val_b) + and str(val_a) != str(val_b) + ): + return False + + # Provider Type matching (for BCBS: consolidated_provider_type must match) + if pass_config.get("provider_type") and col_provider_type: + val_a = self.get_field_value(row, col_provider_type) + val_b = parent.get(col_provider_type) + if ( + pd.notna(val_a) + and pd.notna(val_b) + and str(val_a) + and str(val_b) + and str(val_a) != str(val_b) + ): + return False + + if pass_config.get("date") and col_eff: + if not self.is_date_valid( + parent.get(col_eff), self.get_field_value(row, col_eff) + ): + return False + + return True + + def assign_children( + self, df: pd.DataFrame, config: Dict, parent_cache: Dict[int, Dict[str, Any]] + ) -> pd.DataFrame: + """Execute multi-pass child-to-parent assignment.""" + # Ensure columns exist + if "assigned_parent_idx" not in df.columns: + df["assigned_parent_idx"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int64" + ) + if "assignment_pass" not in df.columns: + df["assignment_pass"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int8" + ) + if "assignment_reasoning" not in df.columns: + df["assignment_reasoning"] = pd.Series( + [""] * len(df), index=df.index, dtype="string" + ) + + rules = config.get("assignment_rules", {}) or {} + pass1 = rules.get("pass1", {}) or {} + pass2 = rules.get("pass2", {}) or {} + pass3 = rules.get("pass3", {}) or {} + + index_fields = [ + f + for f in (config.get("candidate_index_fields", []) or []) + if isinstance(f, str) and f + ] + constraints = config.get("assignment_constraints", {}) or {} + + col_title = constraints.get("title") + col_eff = constraints.get("date") + + parent_index = self.build_parent_index(df, index_fields) + children_idx = df.index[(~df["is_parent"]) & (df["processable"])].tolist() + + if not children_idx: + return df + + # Pass 1: Strict constraints + title matching + self._assign_pass1( + df, + children_idx, + parent_index, + parent_cache, + col_title, + col_eff, + pass1, + constraints, + ) + + # Pass 2: Title-based matching + remaining = df.index[ + (~df["is_parent"]) & (df["processable"]) & (df["assignment_pass"].isna()) + ].tolist() + if remaining and pass2: + self._assign_pass2( + df, + remaining, + parent_index, + parent_cache, + col_title, + col_eff, + pass2, + constraints, + ) + + # Pass 3: Fallback (unique parent, unique payer) + remaining = df.index[ + (~df["is_parent"]) & (df["processable"]) & (df["assignment_pass"].isna()) + ].tolist() + if remaining and pass3: + self._assign_pass3( + df, remaining, parent_index, parent_cache, col_eff, pass3, constraints + ) + + return df + + def _assign_pass1( + self, + df: pd.DataFrame, + children_idx: List[int], + parent_index: Dict, + parent_cache: Dict, + col_title: str, + col_eff: str, + pass1: Dict, + constraints: Dict, + ) -> None: + """Pass 1: Strict constraints + title matching.""" + for cidx in children_idx: + row = df.loc[cidx] + cand = self.find_candidates( + parent_index, row, [f for f in parent_index.keys() if f] + ) + if not cand: + continue + + # Filter by strict constraints + filtered = [ + pidx + for pidx in cand + if self.check_constraints( + row, parent_cache.get(pidx, {}), pass1, constraints + ) + ] + + if not filtered: + continue + + # Title matching if enabled + if pass1.get("title") and col_title: + self._apply_title_priority( + df, cidx, filtered, parent_cache, col_title, pass_num=1 + ) + else: + # No title matching, use highest rank + best = max( + filtered, + key=lambda x: int(parent_cache.get(x, {}).get("parent_rank", 0)), + ) + best_rank = int(parent_cache.get(best, {}).get("parent_rank", 0)) + df.at[cidx, "assigned_parent_idx"] = best + df.at[cidx, "assignment_pass"] = 1 + df.at[cidx, "assignment_reasoning"] = ( + f"Pass 1: strict constraints (rank={best_rank})" + ) + + def _assign_pass2( + self, + df: pd.DataFrame, + remaining: List[int], + parent_index: Dict, + parent_cache: Dict, + col_title: str, + col_eff: str, + pass2: Dict, + constraints: Dict, + ) -> None: + """Pass 2: Title-based matching.""" + col_state = constraints.get("state") + col_payer = constraints.get("payer") + + for cidx in remaining: + row = df.loc[cidx] + cand = self.find_candidates( + parent_index, row, [f for f in parent_index.keys() if f] + ) + if not cand: + continue + + child_eff = row.get(col_eff, pd.NaT) if col_eff else pd.NaT + child_title = row.get(col_title, "") if col_title else "" + + substring_matches = [] + similarity_matches = [] + + for pidx in cand: + p = parent_cache.get(pidx) + if not p: + continue + + # Date constraint + if pass2.get("date") and col_eff and pd.notna(child_eff): + parent_eff = p.get(col_eff, pd.NaT) + if pd.notna(parent_eff) and parent_eff > child_eff: + continue + + # Title matching + if pass2.get("title") and col_title: + parent_title = p.get(col_title, "") + is_substring = TextProcessor.has_text_match( + parent_title, child_title + ) + similarity = TextProcessor.calculate_text_similarity( + parent_title, child_title + ) + parent_rank = int(p.get("parent_rank", 0)) + + if is_substring: + substring_matches.append((parent_rank, similarity, pidx)) + else: + threshold = float(pass2.get("title_similarity_min", 0.50)) + if similarity >= threshold: + similarity_matches.append((similarity, parent_rank, pidx)) + + if substring_matches: + substring_matches.sort(reverse=True) + best_rank, best_sim, best_idx = substring_matches[0] + df.at[cidx, "assigned_parent_idx"] = best_idx + df.at[cidx, "assignment_pass"] = 2 + df.at[cidx, "assignment_reasoning"] = ( + f"Pass 2: title substring match (rank={best_rank})" + ) + elif similarity_matches: + similarity_matches.sort(reverse=True) + best_sim, best_rank, best_idx = similarity_matches[0] + df.at[cidx, "assigned_parent_idx"] = best_idx + df.at[cidx, "assignment_pass"] = 2 + df.at[cidx, "assignment_reasoning"] = ( + f"Pass 2: title similarity (sim={best_sim:.2f},rank={best_rank})" + ) + + def _assign_pass3( + self, + df: pd.DataFrame, + remaining: List[int], + parent_index: Dict, + parent_cache: Dict, + col_eff: str, + pass3: Dict, + constraints: Dict, + ) -> None: + """Pass 3: Fallback (unique parent, unique payer).""" + col_payer = constraints.get("payer") + + for cidx in remaining: + row = df.loc[cidx] + cand = self.find_candidates( + parent_index, row, [f for f in parent_index.keys() if f] + ) + if not cand: + continue + + child_eff = row.get(col_eff, pd.NaT) if col_eff else pd.NaT + + # Filter by date + date_valid_cand = [ + pidx + for pidx in cand + if self.is_date_valid( + parent_cache.get(pidx, {}).get(col_eff, pd.NaT), child_eff + ) + ] + + if not date_valid_cand: + continue + + # Unique parent + if pass3.get("unique_parent") and len(date_valid_cand) == 1: + df.at[cidx, "assigned_parent_idx"] = date_valid_cand[0] + df.at[cidx, "assignment_pass"] = 3 + df.at[cidx, "assignment_reasoning"] = ( + "Pass 3: unique parent (date valid)" + ) + continue + + # Unique payer + if pass3.get("unique_payer") and col_payer: + child_payer = row.get(col_payer, "") + child_payer = ( + "" + if (child_payer is None or pd.isna(child_payer)) + else str(child_payer) + ) + if child_payer: + payer_matches = [ + pidx + for pidx in date_valid_cand + if str(parent_cache.get(pidx, {}).get(col_payer, "")) + == child_payer + ] + if len(payer_matches) == 1: + df.at[cidx, "assigned_parent_idx"] = payer_matches[0] + df.at[cidx, "assignment_pass"] = 3 + df.at[cidx, "assignment_reasoning"] = ( + "Pass 3: unique payer (date valid)" + ) + + @staticmethod + def _apply_title_priority( + df: pd.DataFrame, + cidx: int, + filtered: List[int], + parent_cache: Dict, + col_title: str, + pass_num: int, + ) -> None: + """Apply priority logic: substring matches before similarity.""" + child_title = df.loc[cidx, col_title] if col_title in df.columns else "" + + substring_matches = [] + non_substring = [] + + for pidx in filtered: + parent_title = parent_cache.get(pidx, {}).get(col_title, "") + parent_rank = int(parent_cache.get(pidx, {}).get("parent_rank", 0)) + is_substring = TextProcessor.has_text_match(parent_title, child_title) + similarity = TextProcessor.calculate_text_similarity( + parent_title, child_title + ) + + if is_substring: + substring_matches.append((parent_rank, similarity, pidx)) + else: + non_substring.append((parent_rank, similarity, pidx)) + + if substring_matches: + substring_matches.sort(reverse=True) + best_rank, best_sim, best_idx = substring_matches[0] + df.at[cidx, "assigned_parent_idx"] = best_idx + df.at[cidx, "assignment_pass"] = pass_num + df.at[cidx, "assignment_reasoning"] = ( + f"Pass {pass_num}: title substring (rank={best_rank})" + ) + elif non_substring: + non_substring.sort(reverse=True) + best_rank, best_sim, best_idx = non_substring[0] + df.at[cidx, "assigned_parent_idx"] = best_idx + df.at[cidx, "assignment_pass"] = pass_num + df.at[cidx, "assignment_reasoning"] = ( + f"Pass {pass_num}: rank only (rank={best_rank})" + ) + + +# ============================================================ +# BCBS-Specific Child Assignment (5 Rules) +# ============================================================ + + +class BCBSChildAssigner: + """BCBS-specific child-to-parent assignment implementing 5 rules from original logic.""" + + @staticmethod + def is_missing_lob(lob_value: Any) -> bool: + """Check if LOB value is missing or empty.""" + if lob_value is None or pd.isna(lob_value): + return True + return str(lob_value).strip() == "" + + @staticmethod + def get_str(value: Any) -> str: + """Safely convert value to lowercase string.""" + if value is None or pd.isna(value): + return "" + return str(value).lower().strip() + + def assign_children_bcbs(self, df: pd.DataFrame, config: Dict) -> pd.DataFrame: + """ + BCBS-specific child assignment implementing 5 rules: + Rule 0: Professional logic (title substring + keyword) + Rule 1: LOB + Provider Type exact match + Rule 2: Title substring match (for missing LOB) + Rule 3: Provider Type Count fallback + Rule 4: Commercial LOB fallback + Rule 5: Latest parent fallback + """ + # Ensure columns exist + if "assigned_parent_idx" not in df.columns: + df["assigned_parent_idx"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int64" + ) + if "assignment_pass" not in df.columns: + df["assignment_pass"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int8" + ) + if "assignment_reasoning" not in df.columns: + df["assignment_reasoning"] = pd.Series( + [""] * len(df), index=df.index, dtype="string" + ) + + constraints = config.get("assignment_constraints", {}) or {} + col_title = constraints.get("title", "title_clean") + col_eff = constraints.get("date", "eff_date") + col_lob = constraints.get("lob", "_lob") + col_provider_type = constraints.get("provider_type", "_provider_type") + + # Group by Folder + IRS_Group (grouping_key) + for gk, group in df.groupby("grouping_key", dropna=False): + if pd.isna(gk): + continue + + parents = group[group["is_parent"]].copy() + children = group[~group["is_parent"]].copy() + + if parents.empty or children.empty: + continue + + # Sort parents by effective date + parents = parents.sort_values(col_eff, na_position="last") + + # Build parent cache for this group + parent_cache = {} + for pidx in parents.index: + parent_cache[pidx] = { + "title": ( + self.get_str(parents.at[pidx, col_title]) + if col_title in parents.columns + else "" + ), + "eff_date": ( + parents.at[pidx, col_eff] + if col_eff in parents.columns + else pd.NaT + ), + "lob": ( + self.get_str(parents.at[pidx, col_lob]) + if col_lob in parents.columns + else "" + ), + "provider_type": ( + self.get_str(parents.at[pidx, col_provider_type]) + if col_provider_type in parents.columns + else "" + ), + "parent_rank": ( + int(parents.at[pidx, "parent_rank"]) + if "parent_rank" in parents.columns + else 0 + ), + } + + # Process each child + for cidx in children.index: + child_title = ( + self.get_str(df.at[cidx, col_title]) + if col_title in df.columns + else "" + ) + child_eff = df.at[cidx, col_eff] if col_eff in df.columns else pd.NaT + child_lob = ( + self.get_str(df.at[cidx, col_lob]) if col_lob in df.columns else "" + ) + child_ptype = ( + self.get_str(df.at[cidx, col_provider_type]) + if col_provider_type in df.columns + else "" + ) + + candidate_idx = None + rule_used = "" + + # Get date-eligible parents (parent_eff <= child_eff) + def is_date_eligible(pidx: int) -> bool: + p_eff = parent_cache[pidx]["eff_date"] + if pd.isna(child_eff): + return True # No child date, all parents eligible + if pd.isna(p_eff): + return False # Parent has no date, not eligible + return p_eff <= child_eff + + eligible_parents = [ + pidx for pidx in parent_cache.keys() if is_date_eligible(pidx) + ] + + # Rule 0: Professional logic + if "professional" in child_title and candidate_idx is None: + # 0a: Parent title fully contained in child title + for pidx in sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min, + reverse=True, + ): + p_title = parent_cache[pidx]["title"] + if p_title and len(p_title) > 0 and p_title in child_title: + candidate_idx = pidx + rule_used = "Rule 0a: Professional subset match" + break + + # 0b: Parent title contains "professional" + if candidate_idx is None: + for pidx in sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] + or pd.Timestamp.min, + reverse=True, + ): + if "professional" in parent_cache[pidx]["title"]: + candidate_idx = pidx + rule_used = "Rule 0b: Professional keyword" + break + + # Rule 1: LOB + Provider Type match + if ( + candidate_idx is None + and not self.is_missing_lob(child_lob) + and child_ptype + ): + for pidx in sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min, + reverse=True, + ): + p = parent_cache[pidx] + if ( + p["lob"] == child_lob + and p["provider_type"] + and p["provider_type"] == child_ptype + ): + candidate_idx = pidx + rule_used = "Rule 1: LOB + Provider Type match" + break + + # Rule 2: Title substring match (for missing LOB) + if candidate_idx is None and self.is_missing_lob(child_lob): + for pidx in sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min, + reverse=True, + ): + p_title = parent_cache[pidx]["title"] + if p_title and len(p_title) > 0 and p_title in child_title: + candidate_idx = pidx + rule_used = "Rule 2: Title substring (missing LOB)" + break + + # Rule 3: Provider Type logic + if candidate_idx is None: + if child_ptype: + # 3a: Match by LOB + Provider Type + for pidx in sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] + or pd.Timestamp.min, + reverse=True, + ): + p = parent_cache[pidx] + if ( + p["lob"] == child_lob + and p["provider_type"] == child_ptype + ): + candidate_idx = pidx + rule_used = "Rule 3a: Provider Type match" + break + else: + # 3b: Fallback - find parent with same LOB, prefer 'facility' provider type + lob_matches = [ + pidx + for pidx in eligible_parents + if parent_cache[pidx]["lob"] == child_lob + ] + if lob_matches: + # Prefer facility + facility_matches = [ + pidx + for pidx in lob_matches + if "facility" in parent_cache[pidx]["provider_type"] + ] + if facility_matches: + candidate_idx = sorted( + facility_matches, + key=lambda x: parent_cache[x]["eff_date"] + or pd.Timestamp.min, + reverse=True, + )[0] + rule_used = "Rule 3b: Provider Type Count (facility)" + else: + candidate_idx = sorted( + lob_matches, + key=lambda x: parent_cache[x]["eff_date"] + or pd.Timestamp.min, + reverse=True, + )[0] + rule_used = "Rule 3b: Provider Type Count fallback" + + # Rule 4: Commercial LOB fallback + if candidate_idx is None: + comm_matches = [ + pidx + for pidx in eligible_parents + if "comm" in parent_cache[pidx]["lob"] + ] + if comm_matches: + candidate_idx = sorted( + comm_matches, + key=lambda x: parent_cache[x]["eff_date"] + or pd.Timestamp.min, + reverse=True, + )[0] + rule_used = "Rule 4: Commercial fallback" + + # Rule 5: Latest parent fallback + if candidate_idx is None and eligible_parents: + candidate_idx = sorted( + eligible_parents, + key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min, + reverse=True, + )[0] + rule_used = "Rule 5: Latest parent fallback" + + # Assign the candidate + if candidate_idx is not None: + p_rank = parent_cache[candidate_idx]["parent_rank"] + df.at[cidx, "assigned_parent_idx"] = candidate_idx + df.at[cidx, "assignment_pass"] = 1 # All BCBS rules are pass 1 + df.at[cidx, "assignment_reasoning"] = f"{rule_used} (rank={p_rank})" + + return df + + +# ============================================================ +# Configuration Management +# ============================================================ + + +@dataclass +class ClientConfig: + """Client configuration factory.""" + + name: str + engine_type: str + columns: Dict[str, Any] + prep_fields: Dict[str, Dict] = field(default_factory=dict) + cleaning_rules: Dict = field(default_factory=dict) + derive_fields: List[Dict] = field(default_factory=list) + grouping: Dict = field(default_factory=dict) + parent_identification: Dict = field(default_factory=dict) + candidate_index_fields: List[str] = field(default_factory=list) + assignment_constraints: Dict = field(default_factory=dict) + assignment_rules: Dict = field(default_factory=dict) + validation: Dict = field(default_factory=dict) + rank_output: Dict = field(default_factory=dict) + qa_flags: Dict = field(default_factory=dict) + keep_work_cols: bool = False + + def to_dict(self) -> Dict: + """Convert to dictionary for engine compatibility.""" + return { + "engine_type": self.engine_type, + "columns": self.columns, + "prep_fields": self.prep_fields, + "cleaning_rules": self.cleaning_rules, + "derive_fields": self.derive_fields, + "grouping": self.grouping, + "parent_identification": self.parent_identification, + "candidate_index_fields": self.candidate_index_fields, + "assignment_constraints": self.assignment_constraints, + "assignment_rules": self.assignment_rules, + "validation": self.validation, + "rank_output": self.rank_output, + "qa_flags": self.qa_flags, + "keep_work_cols": self.keep_work_cols, + } + + +class ConfigFactory: + """Factory for creating client configurations.""" + + @staticmethod + def create_molina() -> ClientConfig: + """Create Molina configuration.""" + return ClientConfig( + name="molina", + engine_type="parent_child", + columns={ + "id": "FILE_NAME", + "title": "contract_title_cleaned", + "effective_date": "fixed_effective_date", + }, + prep_fields={ + "_tin": { + "source": "PROV_GROUP_TIN", + "mode": "copy_clean", + "cleaner": "identifier", + }, + "_npi": { + "source": "PROV_GROUP_NPI", + "mode": "copy_clean", + "cleaner": "identifier", + }, + "_prov_name": { + "source": "PROV_GROUP_NAME_FULL_cleaned", + "mode": "copy_clean", + "cleaner": "upper_clean", + }, + "_payer_clean": { + "source": "payer_name_cleaned", + "mode": "copy_clean", + "cleaner": "lower_clean", + }, + "_state_clean": { + "source": "PAYER_STATE", + "mode": "copy_clean", + "cleaner": "upper_clean", + }, + }, + cleaning_rules={ + "fields": { + "title": { + "remove_stopwords": True, + "stopwords": DEFAULT_STOPWORDS, + }, + } + }, + derive_fields=[ + { + "target": "_prov_name_clean", + "source": "_prov_name", + "mode": "copy_clean", + "cleaner": "upper_clean", + }, + ], + grouping={ + "strategy_type": "multi_identifier", + "identifiers": ["_tin", "_npi", "_prov_name_clean"], + }, + parent_identification={ + "strategy_type": "keyword", + "field": "title_clean", + "exclude_keywords": [ + "exhibit", + "amendment", + "amend", + "addendum", + "adden", + "renewal", + "extension", + "modification", + "attachment", + "letter", + "rate letter", + "notice", + "notification", + "settlement", + "release", + ], + }, + candidate_index_fields=["_tin", "_npi", "_prov_name_clean"], + assignment_constraints={ + "state": "_state_clean", + "payer": "_payer_clean", + "date": "eff_date", + "title": "title_clean", + }, + assignment_rules={ + "pass1": {"state": True, "payer": True, "date": True, "title": True}, + "pass2": {"title": True, "date": True, "title_similarity_min": 0.50}, + "pass3": {"unique_parent": True, "unique_payer": True}, + }, + validation={"source_label_col": "parent_child_flag"}, + rank_output={"preserve_orphan_combined_rank": True}, + qa_flags={ + "flag_mapping_differences": True, + "flag_rank_mismatch": True, + "flag_parent_title_keywords": True, + "flag_payer_suffix": True, + "flag_similar_group_names": False, + "flag_low_confidence": False, + "flag_low_similarity": False, + "similarity_threshold": 0.60, + }, + keep_work_cols=False, + ) + + @staticmethod + def create_caresource() -> ClientConfig: + """Create CareSource configuration.""" + return ClientConfig( + name="caresource", + engine_type="caresource", + columns={ + "file_name": ["File Name", "Filename", "FILE_NAME"], + "supplier_group": ["Supplier Group", "SupplierGroup", "SUPPLIER_GROUP"], + "hierarchical_type": [ + "Hierarchical Type", + "hierarchical_type", + "HIERARCHICAL_TYPE", + ], + "effective_date": [ + "Fixed_Effective_Date", + "EffectiveDate", + "Effective Date", + ], + "commodity": ["Commodity", "COMMODITY"], + "rank_source": ["Rank", "RANK"], + }, + keep_work_cols=False, + ) + + @staticmethod + def create_bcbs() -> ClientConfig: + """Create BCBS configuration matching original bcbs_parent_child_mapping.py logic.""" + return ClientConfig( + name="bcbs", + engine_type="bcbs_custom", + columns={ + "id": ["File Name", "FILE_NAME", "Contract Name"], + "title": [ + "contract_title_clean", + "fixed_contract_name", + "Contract Name", + ], + "effective_date": ["final_effective_date", "Contract Effective Date"], + }, + prep_fields={ + # Grouping fields - match original: Folder + IRS_Group + "_folder": { + "source": "Folder", + "mode": "copy_clean", + "cleaner": "upper_clean", + }, + "_irs_group": { + "source": "IRS_Group", + "mode": "copy_clean", + "cleaner": "lower_clean", + }, + # Additional fields for matching + "_provider_type": { + "source": "consolidated_provider_type", + "mode": "copy_clean", + "cleaner": "lower_clean", + }, + "_lob": { + "source": "extracted_lob", + "mode": "copy_clean", + "cleaner": "lower_clean", + }, + }, + # Match original grouping: Folder + IRS_Group + grouping={ + "strategy_type": "multi_identifier", + "identifiers": ["_folder", "_irs_group"], + }, + parent_identification={ + # Use BCBS-specific parent identification logic + "strategy_type": "bcbs_custom", + "field": "title_clean", + "filename_field": "id_clean", + "lob_field": "_lob", + }, + # Match original candidate fields + candidate_index_fields=["_folder", "_irs_group"], + assignment_constraints={ + "date": "eff_date", + "title": "title_clean", + "lob": "_lob", + "provider_type": "_provider_type", + }, + assignment_rules={ + # Pass 1: LOB + Provider Type match (Rule 1 from original) + "pass1": { + "date": True, + "title": True, + "lob": True, + "provider_type": True, + }, + # Pass 2: Title substring match (Rules 0, 2 from original) + "pass2": {"title": True, "date": True, "title_similarity_min": 0.50}, + # Pass 3: Fallback to latest commercial/parent (Rules 4, 5 from original) + "pass3": {"unique_parent": True, "commercial_fallback": True}, + }, + validation={"source_label_col": "parent_child_flag"}, + # Parent ranking: Commercial LOB parents ranked first (like original) + rank_output={ + "preserve_orphan_combined_rank": True, + "commercial_lob_priority": True, # Commercial LOB parents get priority + "lob_field": "_lob", + }, + qa_flags={ + "flag_mapping_differences": True, + "flag_rank_mismatch": True, + "flag_parent_title_keywords": True, + }, + keep_work_cols=False, + ) + + @staticmethod + def create_clover() -> ClientConfig: + """Create Clover Health configuration (same as Molina with source labels).""" + molina = ConfigFactory.create_molina() + molina.name = "clover_health" + molina.validation = {"source_label_col": "parent_child_flag"} + molina.rank_output = {"preserve_orphan_combined_rank": False} + return molina + + @staticmethod + def get_config(client: str) -> ClientConfig: + """Get configuration for specified client.""" + client = (client or "").strip().lower() + + configs = { + "molina": ConfigFactory.create_molina, + "cnc": ConfigFactory.create_molina, # CNC uses same config as Molina + "caresource": ConfigFactory.create_caresource, + "bcbs": ConfigFactory.create_bcbs, + "clover_health": ConfigFactory.create_clover, + } + + if client not in configs: + logger.warning( + f"Unknown client '{client}'. Available: {list(configs.keys())}. Defaulting to molina." + ) + return ConfigFactory.create_molina() + + return configs[client]() + + +# ============================================================ +# Main Engine (continued in next section) +# ============================================================ + + +class ParentChildEngine: + """Main orchestration pipeline for parent-child mapping.""" + + def __init__(self, config: Dict): + self.cfg = config + self.cols = config.get("columns", {}) or {} + self.keep_work_cols = bool(config.get("keep_work_cols", False)) + self.engine_type = config.get("engine_type", "parent_child") + + self.text_cleaner = TextCleaner() + self.field_deriver = FieldDeriver() + self.group_builder = GroupKeyBuilder() + self.parent_identifier = ParentIdentifier() + self.child_assigner = ChildAssigner() + self.bcbs_child_assigner = BCBSChildAssigner() # BCBS-specific assigner + + def run(self, df_in: pd.DataFrame) -> pd.DataFrame: + """Execute full pipeline.""" + t0 = time.time() + df = df_in.copy() + df["input_row_order"] = np.arange(len(df), dtype=np.int32) + + # Initialize assignment columns + df["assigned_parent_idx"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int64" + ) + df["assignment_pass"] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="Int8" + ) + df["assignment_reasoning"] = pd.Series( + [""] * len(df), index=df.index, dtype="string" + ) + + df = self.prepare_columns(df) + df = self.field_deriver.build_fields( + df, + self.cfg.get("derive_fields", []) or [], + self.text_cleaner, + self.cfg.get("cleaning_rules", {}) or {}, + ) + df = self.build_grouping(df) + df = self.identify_parents(df) + df = self.assign_children(df) + df = self.prepare_output(df) + df = self.compute_ranks(df) + self.apply_qa_flags(df) + self.show_metrics(df) + + elapsed = time.time() - t0 + logger.info("DONE in %.2fs | rows=%s", elapsed, len(df)) + + if not self.keep_work_cols: + df = self._remove_work_columns(df) + + return df + + def prepare_columns(self, df: pd.DataFrame) -> pd.DataFrame: + """Prepare and clean source columns.""" + rules = self.cfg.get("cleaning_rules", {}) or {} + + id_col = resolve_column(df, self.cols.get("id"), required=True, label="id") + title_col = resolve_column( + df, self.cols.get("title"), required=False, label="title" + ) + eff_col = resolve_column( + df, self.cols.get("effective_date"), required=False, label="effective_date" + ) + + df["id_raw"] = df[id_col].astype("string") + df["id_clean"] = TextProcessor.clean_text_lowercase(df["id_raw"]) + + if title_col: + df["title_raw"] = df[title_col].astype("string") + df["title_clean"] = self.text_cleaner.clean_field( + "title", df["title_raw"], rules + ) + else: + df["title_clean"] = pd.Series([""] * len(df), dtype="string") + + if eff_col: + df["eff_date"] = to_datetime(df[eff_col]) + else: + df["eff_date"] = pd.Series([pd.NaT] * len(df)) + + # Build prep fields + prep = self.cfg.get("prep_fields", {}) or {} + for internal_col, spec in prep.items(): + src = spec.get("source") + mode = spec.get("mode", "copy_clean") + cleaner = spec.get("cleaner", "raw") + + if not src or src not in df.columns: + df[internal_col] = pd.Series( + [pd.NA] * len(df), index=df.index, dtype="string" + ) + continue + + if mode == "copy_clean": + if cleaner == "upper_clean": + df[internal_col] = TextProcessor.clean_text( + to_string(df[src]) + ).str.upper() + elif cleaner == "lower_clean": + df[internal_col] = TextProcessor.clean_text_lowercase( + to_string(df[src]) + ) + elif cleaner == "identifier": + df[internal_col] = TextProcessor.normalize_identifier_series( + df[src] + ) + else: + df[internal_col] = to_string(df[src]).str.strip() + + df["processable"] = True + return df + + def build_grouping(self, df: pd.DataFrame) -> pd.DataFrame: + """Build grouping keys.""" + gk, tier = self.group_builder.build_grouping_key( + df, self.cfg.get("grouping", {}) or {} + ) + df["grouping_key"] = gk + df["tier_level"] = tier.astype(np.int8) + df["processable"] = ( + df["tier_level"].ne(0) & df["grouping_key"].notna() + ).astype(bool) + return df + + def identify_parents(self, df: pd.DataFrame) -> pd.DataFrame: + """Identify parents and compute rank.""" + parent_cfg = self.cfg.get("parent_identification", {}) or {} + parent_mask = self.parent_identifier.find_parents(df, parent_cfg) + df["is_parent"] = (parent_mask & df["processable"]).astype(bool) + + df["parent_rank"] = pd.Series(0, index=df.index, dtype=np.int32) + + pm = df["is_parent"] + if pm.any(): + rank_output = self.cfg.get("rank_output", {}) or {} + commercial_priority = rank_output.get("commercial_lob_priority", False) + lob_field = rank_output.get("lob_field", "_lob") + + cols_needed = ["grouping_key", "eff_date", "id_clean", "input_row_order"] + if commercial_priority and lob_field in df.columns: + cols_needed.append(lob_field) + + tmp = df.loc[pm, cols_needed].copy() + tmp["eff_sort"] = tmp["eff_date"].fillna(pd.Timestamp.max) + + # Commercial LOB priority: Commercial parents ranked first within each group + if commercial_priority and lob_field in tmp.columns: + # has_commercial = True if 'comm' is in LOB value + tmp["_has_commercial"] = ( + tmp[lob_field] + .fillna("") + .astype(str) + .str.contains("comm", case=False, na=False) + ) + # Sort: group, commercial first (True=0, False=1), then by date + tmp["_comm_sort"] = (~tmp["_has_commercial"]).astype(int) + tmp = tmp.sort_values( + [ + "grouping_key", + "_comm_sort", + "eff_sort", + "input_row_order", + "id_clean", + ], + kind="mergesort", + ) + tmp = tmp.drop(columns=["_has_commercial", "_comm_sort"]) + else: + tmp = tmp.sort_values( + ["grouping_key", "eff_sort", "input_row_order", "id_clean"], + kind="mergesort", + ) + + tmp["parent_rank"] = tmp.groupby("grouping_key", sort=False).cumcount() + 1 + df.loc[tmp.index, "parent_rank"] = ( + tmp["parent_rank"].astype(np.int32).values + ) + + return df + + def assign_children(self, df: pd.DataFrame) -> pd.DataFrame: + """Execute child assignment.""" + # Use BCBS-specific 5-rule assigner for bcbs_custom engine type + if self.engine_type == "bcbs_custom": + return self.bcbs_child_assigner.assign_children_bcbs(df, self.cfg) + + # Default: use generic multi-pass assigner + constraints = self.cfg.get("assignment_constraints", {}) or {} + cols_needed = ["parent_rank"] + # Include all constraint fields: state, payer, commodity, date, title, lob, provider_type + for k in [ + "state", + "payer", + "commodity", + "date", + "title", + "lob", + "provider_type", + ]: + col = constraints.get(k) + if col and col in df.columns: + cols_needed.append(col) + cols_needed = list(dict.fromkeys(cols_needed)) + parent_cache = df.loc[df["is_parent"], cols_needed].to_dict("index") + return self.child_assigner.assign_children(df, self.cfg, parent_cache) + + def prepare_output(self, df: pd.DataFrame) -> pd.DataFrame: + """Initialize output columns.""" + df["parent_child_flag_dest"] = pd.Series([pd.NA] * len(df), dtype="string") + df["parent_rank_dest"] = pd.Series([pd.NA] * len(df), dtype="string") + df["child_rank_dest"] = pd.Series([pd.NA] * len(df), dtype="string") + df["orphan_rank_dest"] = pd.Series([pd.NA] * len(df), dtype="string") + df["combined_rank_dest"] = pd.Series([pd.NA] * len(df), dtype="string") + + pm = df["is_parent"] + cm = df["assigned_parent_idx"].notna() + om = (~pm) & (~cm) + + df.loc[pm, "parent_child_flag_dest"] = "Parent" + df.loc[cm, "parent_child_flag_dest"] = "Child" + df.loc[om, "parent_child_flag_dest"] = "Orphan" + + df.loc[pm, "parent_rank_dest"] = df.loc[pm, "parent_rank"].astype("string") + df.loc[pm, "combined_rank_dest"] = df.loc[pm, "parent_rank_dest"] + + return df + + def compute_ranks(self, df: pd.DataFrame) -> pd.DataFrame: + """Compute child and orphan ranks.""" + # Child ranks + child_mask = df["assigned_parent_idx"].notna() + if child_mask.any(): + tmp = df.loc[ + child_mask, ["assigned_parent_idx", "eff_date", "input_row_order"] + ].copy() + tmp["eff_sort"] = tmp["eff_date"].fillna(pd.Timestamp.max) + tmp = tmp.sort_values( + ["assigned_parent_idx", "eff_sort", "input_row_order"], kind="mergesort" + ) + + seq = tmp.groupby("assigned_parent_idx", sort=False).cumcount() + 1 + seq_str = seq.astype("string").reset_index(drop=True).values + + parent_idx = tmp["assigned_parent_idx"].astype("int64").values + parent_rank_str = ( + df.loc[parent_idx, "parent_rank"] + .astype("string") + .reset_index(drop=True) + .values + ) + + df.loc[tmp.index, "child_rank_dest"] = parent_rank_str + ".0." + seq_str + df.loc[tmp.index, "combined_rank_dest"] = df.loc[ + tmp.index, "child_rank_dest" + ] + + # Orphan ranks + pm = df["is_parent"] + cm = df["assigned_parent_idx"].notna() + orphan_mask = (~pm) & (~cm) + + if orphan_mask.any(): + tmp = df.loc[orphan_mask, ["input_row_order"]].copy() + tmp = tmp.sort_values(["input_row_order"], kind="mergesort") + seq = pd.Series(np.arange(1, len(tmp) + 1, dtype=np.int32), index=tmp.index) + df.loc[tmp.index, "orphan_rank_dest"] = ( + "0.0." + seq.astype("string") + ).values + df.loc[tmp.index, "combined_rank_dest"] = df.loc[ + tmp.index, "orphan_rank_dest" + ] + + return df + + def apply_qa_flags(self, df: pd.DataFrame) -> None: + """Apply QA flags for manual review.""" + qa_config = self.cfg.get("qa_flags", {}) or {} + + # Flag mapping differences + if qa_config.get("flag_mapping_differences", True): + if ( + "parent_child_flag" in df.columns + and "parent_child_flag_dest" in df.columns + ): + src = TextProcessor.normalize_label(df["parent_child_flag"]) + dest = TextProcessor.normalize_label(df["parent_child_flag_dest"]) + mask = ( + src.notna() + & dest.notna() + & (src != "") + & (dest != "") + & (src != dest) + ) + self._add_flag(df, mask, "Incorrect Mapping") + + # Flag rank mismatches + if qa_config.get("flag_rank_mismatch", True): + if "combined_rank_dest" in df.columns and "combined_rank" in df.columns: + mask = self._find_significant_rank_differences(df) + self._add_flag(df, mask, "Rank Mismatch") + + # Flag parent titles with keywords + if qa_config.get("flag_parent_title_keywords", True): + title_col = self._find_title_column(df) + if title_col: + has_excluded = ( + df[title_col].astype("string").str.contains(TITLE_REGEX, na=False) + ) + is_parent = df["parent_child_flag_dest"] == "Parent" + self._add_flag(df, has_excluded & is_parent, "Title Contains Keywords") + + # Flag payer suffixes + if qa_config.get("flag_payer_suffix", True): + if "payer_name_cleaned" in df.columns: + has_suffix = ( + df["payer_name_cleaned"] + .astype("string") + .str.contains(SUFFIX_REGEX, na=False) + ) + self._add_flag(df, has_suffix, "Payer Suffix Present") + + def _find_significant_rank_differences(self, df: pd.DataFrame) -> pd.Series: + """Identify significant rank differences.""" + src = df["combined_rank"].astype("string").fillna("") + dest = df["combined_rank_dest"].astype("string").fillna("") + + def is_significant(src_rank, dest_rank): + if src_rank == "" or dest_rank == "" or src_rank == dest_rank: + return False + + src_parts = str(src_rank).split(".") + dest_parts = str(dest_rank).split(".") + + if len(src_parts) == 2 and src_parts[1] == "0": + src_parts = [src_parts[0]] + if len(dest_parts) == 2 and dest_parts[1] == "0": + dest_parts = [dest_parts[0]] + + if len(src_parts) != len(dest_parts): + return True + + if len(src_parts) >= 3: + if src_parts[0] != dest_parts[0]: + return True + + return False + + return pd.Series( + [is_significant(src.iloc[i], dest.iloc[i]) for i in range(len(df))], + index=df.index, + ) + + @staticmethod + def _find_title_column(df: pd.DataFrame) -> Optional[str]: + """Find title column in dataframe.""" + candidates = ["contract_title_cleaned", "contract_title_clean", "_title_clean"] + for col in candidates: + if col in df.columns: + return col + return None + + @staticmethod + def _add_flag(df: pd.DataFrame, row_mask: pd.Series, flag: str) -> None: + """Add manual review flag.""" + if "Manual_Review_Flag" not in df.columns: + df["Manual_Review_Flag"] = pd.Series([pd.NA] * len(df), dtype="string") + + existing = df.loc[row_mask, "Manual_Review_Flag"] + df.loc[row_mask, "Manual_Review_Flag"] = existing.mask( + existing.notna(), existing + " | " + flag + ).fillna(flag) + + def show_metrics(self, df: pd.DataFrame) -> None: + """Log accuracy metrics.""" + vcfg = self.cfg.get("validation", {}) or {} + src_col = vcfg.get("source_label_col") + if not src_col or src_col not in df.columns: + return + src = TextProcessor.normalize_label(df[src_col]) + dest = TextProcessor.normalize_label(df["parent_child_flag_dest"]) + valid = src.notna() & dest.notna() + match = (src == dest) & valid + total_valid = int(valid.sum()) + match_cnt = int(match.sum()) + acc = (match_cnt / total_valid * 100.0) if total_valid else 0.0 + logger.info("Classification accuracy: %.2f%% (valid=%s)", acc, total_valid) + + @staticmethod + def _remove_work_columns(df: pd.DataFrame) -> pd.DataFrame: + """Remove internal working columns.""" + work_cols = [ + c for c in df.columns if c.startswith("_") or c.startswith("temp_") + ] + df.drop(columns=work_cols, inplace=True, errors="ignore") + return df + + +# ============================================================ +# Main Entry Point +# ============================================================ + + +def qc_main(df_in, client): + """Main entry point.""" + # output_file = f"output_{client}.csv" + + logger.info("Client: %s", client) + + # Get configuration + config = ConfigFactory.get_config(client) + cfg_dict = config.to_dict() + + logger.info("Loaded: %s rows, %s cols", len(df_in), len(df_in.columns)) + + # Process + engine = ParentChildEngine(cfg_dict) + df_out = engine.run(df_in) + + # Export + input_cols = df_in.columns.tolist() + out_cols = df_out.columns.tolist() + extra_cols = [ + "parent_child_flag_dest", + "combined_rank_dest", + "Manual_Review_Flag", + "assignment_reasoning", + "parent_name_dest", + ] + keep_cols = [c for c in input_cols if c in out_cols] + keep_cols.extend([c for c in extra_cols if c in out_cols and c not in keep_cols]) + + df_final = df_out.loc[:, keep_cols].copy() + # df_final.to_csv(output_file, index=False, encoding="utf-8", quoting=1) + # logger.info("Exported: %s", output_file) + return df_final diff --git a/src/pipelines/runner.py b/src/pipelines/runner.py index 4594f51..3cf1770 100644 --- a/src/pipelines/runner.py +++ b/src/pipelines/runner.py @@ -247,7 +247,7 @@ def main(client: str = "saas", testing=False, test_params={}): io_utils.write_local(ERROR_RESULT_DF, "", run_timestamp, "error") if config.PERFORM_PARENT_CHILD_MAPPING: - parent_child_main.main(doczy_output_for_pc) + parent_child_main.main(doczy_output_for_pc, client=client) if not per_file_usage_df.empty and not batch_summary_df.empty: logging.info("Exporting usage and cost tracking data locally...") diff --git a/src/pipelines/saas/main.py b/src/pipelines/saas/main.py index 0344109..679cfdc 100644 --- a/src/pipelines/saas/main.py +++ b/src/pipelines/saas/main.py @@ -276,7 +276,14 @@ def main(testing=False, test_params={}): f"Starting Parent-Child mapping with {len(FINAL_RESULT_DF)} records from: {doczy_output_for_pc}" ) with timing_utils.timed_block("parent_child_mapping", log_level="INFO"): - parent_child_main.main(doczy_output_for_pc) + client = ( + config.CLIENT[0] + if config.CLIENT + and len(config.CLIENT) == 1 + and config.CLIENT[0] != "None" + else None + ) + parent_child_main.main(doczy_output_for_pc, client=client) # Export usage and cost tracking data locally only if both dataframes have data if not per_file_usage_df.empty and not batch_summary_df.empty: diff --git a/src/tests/test_parent_child.py b/src/tests/test_parent_child.py new file mode 100644 index 0000000..0f89a80 --- /dev/null +++ b/src/tests/test_parent_child.py @@ -0,0 +1,641 @@ +""" +Tests for the parent_child module. + +Tests cover: +- Grouping key creation and tier assignment +- Grouping compatibility checks +- Parent identification (exclude keywords) +- Parent ranking +- Child assignment logic +- Utility functions (main.py) +- QC engine utilities +""" + +import unittest +import pandas as pd +import numpy as np +from datetime import datetime + +from src.parent_child.pipeline import ( + create_grouping_key_and_tier, + check_grouping_compatibility, + filter_on_exclude_keywords, + rank_parents, + build_grouping_string, + build_matching_columns_string, + flag_multi_parent_with_early_dates, + assign_child_ranks, + extract_ordinal, +) +from src.parent_child.__main__ import extract_client_name +from src.parent_child.qc import ( + to_string, + to_datetime, + resolve_column, + TextProcessor, + ConfigFactory, +) + + +class TestGroupingKeyCreation(unittest.TestCase): + """Tests for create_grouping_key_and_tier function.""" + + def test_tier_1_all_columns_present(self): + """Tier 1: All 3 columns (TIN, NPI, PROV_NAME) are non-null.""" + df = pd.DataFrame( + { + "PROV_GROUP_TIN": ["123456789"], + "PROV_GROUP_NPI": ["1234567890"], + "PROV_GROUP_NAME_FULL_cleaned": ["Test Hospital"], + } + ) + result = create_grouping_key_and_tier(df) + + self.assertEqual(result["grouping_tier"].iloc[0], 1) + self.assertIn("TIN:123456789", result["grouping_key"].iloc[0]) + self.assertIn("NPI:1234567890", result["grouping_key"].iloc[0]) + self.assertIn("NAME:Test Hospital", result["grouping_key"].iloc[0]) + + def test_tier_2_two_columns_present(self): + """Tier 2: Any 2 columns are non-null.""" + df = pd.DataFrame( + { + "PROV_GROUP_TIN": ["123456789"], + "PROV_GROUP_NPI": ["1234567890"], + "PROV_GROUP_NAME_FULL_cleaned": [""], # Empty + } + ) + result = create_grouping_key_and_tier(df) + + self.assertEqual(result["grouping_tier"].iloc[0], 2) + self.assertIn("TIN:123456789", result["grouping_key"].iloc[0]) + self.assertIn("NPI:1234567890", result["grouping_key"].iloc[0]) + + def test_tier_3_one_column_present(self): + """Tier 3: Only 1 column is non-null.""" + df = pd.DataFrame( + { + "PROV_GROUP_TIN": ["123456789"], + "PROV_GROUP_NPI": [""], + "PROV_GROUP_NAME_FULL_cleaned": [None], + } + ) + result = create_grouping_key_and_tier(df) + + self.assertEqual(result["grouping_tier"].iloc[0], 3) + self.assertEqual(result["grouping_key"].iloc[0], "TIN:123456789") + + def test_tier_0_no_columns_present(self): + """Tier 0: No columns are non-null (immediate orphan).""" + df = pd.DataFrame( + { + "PROV_GROUP_TIN": [None], + "PROV_GROUP_NPI": [""], + "PROV_GROUP_NAME_FULL_cleaned": [""], + } + ) + result = create_grouping_key_and_tier(df) + + self.assertEqual(result["grouping_tier"].iloc[0], 0) + self.assertIsNone(result["grouping_key"].iloc[0]) + + +class TestGroupingCompatibility(unittest.TestCase): + """Tests for check_grouping_compatibility function.""" + + def test_compatible_same_tin(self): + """Compatible: Parent and child share same TIN.""" + parent_key = "TIN:123|NPI:456|NAME:Hospital" + child_key = "TIN:123|NPI:789" + + self.assertTrue(check_grouping_compatibility(parent_key, child_key)) + + def test_compatible_same_npi(self): + """Compatible: Parent and child share same NPI.""" + parent_key = "TIN:123|NPI:456" + child_key = "NPI:456" + + self.assertTrue(check_grouping_compatibility(parent_key, child_key)) + + def test_not_compatible_different_values(self): + """Not compatible: No shared values.""" + parent_key = "TIN:123|NPI:456" + child_key = "TIN:999|NPI:888" + + self.assertFalse(check_grouping_compatibility(parent_key, child_key)) + + def test_not_compatible_null_keys(self): + """Not compatible: Null keys.""" + self.assertFalse(check_grouping_compatibility(None, "TIN:123")) + self.assertFalse(check_grouping_compatibility("TIN:123", None)) + self.assertFalse(check_grouping_compatibility(None, None)) + + +class TestFilterOnExcludeKeywords(unittest.TestCase): + """Tests for filter_on_exclude_keywords function.""" + + def test_parent_no_exclude_keywords(self): + """Row without exclude keywords should be marked as parent.""" + df = pd.DataFrame( + { + "contract_title_cleaned": ["Provider Agreement"], + "grouping_tier": [1], + } + ) + result = filter_on_exclude_keywords(df, True) + + self.assertTrue(result["parent"].iloc[0]) + + def test_not_parent_with_amendment(self): + """Row with 'amendment' should NOT be marked as parent.""" + df = pd.DataFrame( + { + "contract_title_cleaned": ["First Amendment to Agreement"], + "grouping_tier": [1], + } + ) + result = filter_on_exclude_keywords(df, True) + + self.assertFalse(result["parent"].iloc[0]) + + def test_not_parent_with_addendum(self): + """Row with 'addendum' should NOT be marked as parent.""" + df = pd.DataFrame( + { + "contract_title_cleaned": ["Addendum A - Rate Schedule"], + "grouping_tier": [1], + } + ) + result = filter_on_exclude_keywords(df, True) + + self.assertFalse(result["parent"].iloc[0]) + + def test_not_parent_with_exhibit(self): + """Row with 'exhibit' should NOT be marked as parent.""" + df = pd.DataFrame( + { + "contract_title_cleaned": ["Exhibit B - Service Areas"], + "grouping_tier": [1], + } + ) + result = filter_on_exclude_keywords(df, True) + + self.assertFalse(result["parent"].iloc[0]) + + +class TestRankParents(unittest.TestCase): + """Tests for rank_parents function.""" + + def test_single_parent_gets_rank_1(self): + """Single parent in group gets rank 1.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123"], + "parent": [True], + "fixed_effective_date": ["2024-01-01"], + } + ) + result = rank_parents(df) + + self.assertEqual(result["parent_rank"].iloc[0], 1) + + def test_multiple_parents_ranked_by_date(self): + """Multiple parents ranked by effective date (earliest = rank 1).""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123", "TIN:123"], + "parent": [True, True, False], + "fixed_effective_date": ["2024-03-01", "2024-01-01", "2024-02-01"], + } + ) + result = rank_parents(df) + + # Index 1 has earliest date, should be rank 1 + self.assertEqual(result["parent_rank"].iloc[1], 1) + # Index 0 has latest date, should be rank 2 + self.assertEqual(result["parent_rank"].iloc[0], 2) + # Index 2 is not a parent, should have NaN rank + self.assertTrue(pd.isna(result["parent_rank"].iloc[2])) + + def test_non_parents_get_no_rank(self): + """Non-parents should not receive a parent rank.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123"], + "parent": [False], + "fixed_effective_date": ["2024-01-01"], + } + ) + result = rank_parents(df) + + self.assertTrue(pd.isna(result["parent_rank"].iloc[0])) + + +class TestExtractClientName(unittest.TestCase): + """Tests for extract_client_name function from main.py.""" + + def test_simple_filename(self): + """Extract client name from simple filename.""" + result = extract_client_name("Molina_123.xlsx") + self.assertEqual(result, "molina") + + def test_filename_with_spaces(self): + """Extract client name with spaces converted to underscores.""" + result = extract_client_name("Clover Health_132.xlsx") + self.assertEqual(result, "clover_health") + + def test_full_path(self): + """Extract client name from full file path.""" + result = extract_client_name("/data/uploads/BCBS_NC_456.xlsx") + self.assertEqual(result, "bcbs_nc") + + def test_s3_path(self): + """Extract client name from S3 path.""" + result = extract_client_name("s3://bucket/folder/CareSource_789.xlsx") + self.assertEqual(result, "caresource") + + def test_empty_returns_output(self): + """Empty filename after cleaning returns 'output'.""" + result = extract_client_name("123.xlsx") + self.assertEqual(result, "output") + + +class TestQCUtilities(unittest.TestCase): + """Tests for utility functions from generic_hierarchy_lineage_qc.py.""" + + def test_to_string_with_values(self): + """to_string converts series to string type.""" + series = pd.Series(["abc", "def", None, 123]) + result = to_string(series) + + self.assertEqual(result.dtype, "string") + self.assertEqual(result.iloc[0], "abc") + self.assertEqual(result.iloc[2], "") # None becomes empty string + + def test_to_string_with_nan(self): + """to_string fills NaN with empty string.""" + series = pd.Series([np.nan, "value", pd.NA]) + result = to_string(series) + + self.assertEqual(result.iloc[0], "") + self.assertEqual(result.iloc[1], "value") + self.assertEqual(result.iloc[2], "") + + def test_to_datetime_valid_dates(self): + """to_datetime parses valid date strings.""" + series = pd.Series(["2024-01-15", "2024-06-30"]) + result = to_datetime(series) + + self.assertEqual(result.iloc[0], pd.Timestamp("2024-01-15")) + self.assertEqual(result.iloc[1], pd.Timestamp("2024-06-30")) + + def test_to_datetime_invalid_dates(self): + """to_datetime coerces invalid dates to NaT.""" + series = pd.Series(["2024-01-15", "not a date", None]) + result = to_datetime(series) + + self.assertEqual(result.iloc[0], pd.Timestamp("2024-01-15")) + self.assertTrue(pd.isna(result.iloc[1])) + self.assertTrue(pd.isna(result.iloc[2])) + + def test_resolve_column_string_exists(self): + """resolve_column finds column by string name.""" + df = pd.DataFrame({"col_a": [1], "col_b": [2]}) + result = resolve_column(df, "col_a") + + self.assertEqual(result, "col_a") + + def test_resolve_column_string_not_exists(self): + """resolve_column returns None for non-existent column.""" + df = pd.DataFrame({"col_a": [1]}) + result = resolve_column(df, "col_x") + + self.assertIsNone(result) + + def test_resolve_column_list_first_match(self): + """resolve_column returns first matching column from list.""" + df = pd.DataFrame({"col_b": [1], "col_c": [2]}) + result = resolve_column(df, ["col_a", "col_b", "col_c"]) + + self.assertEqual(result, "col_b") + + def test_resolve_column_required_raises(self): + """resolve_column raises ValueError when required column missing.""" + df = pd.DataFrame({"col_a": [1]}) + + with self.assertRaises(ValueError): + resolve_column(df, "col_x", required=True, label="test_col") + + +class TestTextProcessor(unittest.TestCase): + """Tests for TextProcessor class methods.""" + + def test_clean_text_removes_special_chars(self): + """clean_text removes special characters.""" + series = pd.Series(["Hello, World! @#$%"]) + result = TextProcessor.clean_text(series) + + self.assertEqual(result.iloc[0], "Hello World") + + def test_clean_text_normalizes_whitespace(self): + """clean_text normalizes multiple spaces.""" + series = pd.Series(["Hello World"]) + result = TextProcessor.clean_text(series) + + self.assertEqual(result.iloc[0], "Hello World") + + def test_clean_text_lowercase(self): + """clean_text_lowercase converts to lowercase.""" + series = pd.Series(["HELLO World"]) + result = TextProcessor.clean_text_lowercase(series) + + self.assertEqual(result.iloc[0], "hello world") + + def test_normalize_identifier_simple(self): + """normalize_identifier handles simple values.""" + result = TextProcessor.normalize_identifier("123-45-6789") + self.assertEqual(result, "123456789") + + def test_normalize_identifier_pipe_separated(self): + """normalize_identifier handles pipe-separated values.""" + result = TextProcessor.normalize_identifier("123|456|123") + self.assertEqual(result, "123|456") # Deduped + + def test_normalize_identifier_none(self): + """normalize_identifier returns empty string for None.""" + result = TextProcessor.normalize_identifier(None) + self.assertEqual(result, "") + + def test_has_text_match_substring(self): + """has_text_match detects substring matches.""" + self.assertTrue(TextProcessor.has_text_match("Agreement", "Provider Agreement")) + self.assertTrue(TextProcessor.has_text_match("Provider Agreement", "Agreement")) + + def test_has_text_match_no_match(self): + """has_text_match returns False for no match.""" + self.assertFalse( + TextProcessor.has_text_match("Amendment", "Provider Agreement") + ) + + def test_has_text_match_case_insensitive(self): + """has_text_match is case insensitive.""" + self.assertTrue(TextProcessor.has_text_match("AGREEMENT", "provider agreement")) + + def test_calculate_text_similarity(self): + """calculate_text_similarity computes overlap ratio.""" + # Identical texts should have similarity 1.0 + sim = TextProcessor.calculate_text_similarity("hello world", "hello world") + self.assertEqual(sim, 1.0) + + # Completely different texts should have similarity 0.0 + sim = TextProcessor.calculate_text_similarity("abc def", "xyz uvw") + self.assertEqual(sim, 0.0) + + # Partial overlap + sim = TextProcessor.calculate_text_similarity("hello world", "hello there") + self.assertGreater(sim, 0.0) + self.assertLess(sim, 1.0) + + +class TestBuildStrings(unittest.TestCase): + """Tests for string building helper functions.""" + + def test_build_grouping_string_single_col(self): + """build_grouping_string with single column.""" + child_row = pd.Series( + { + "PROV_GROUP_TIN": "123456789", + "PROV_GROUP_NPI": None, + "PROV_GROUP_NAME_FULL": None, + } + ) + result = build_grouping_string(child_row, ["TIN"]) + + self.assertIn("TIN", result) + self.assertIn("123456789", result) + + def test_build_grouping_string_multiple_cols(self): + """build_grouping_string with multiple columns.""" + child_row = pd.Series( + { + "PROV_GROUP_TIN": "123456789", + "PROV_GROUP_NPI": "9876543210", + "PROV_GROUP_NAME_FULL": "Test Hospital", + } + ) + result = build_grouping_string(child_row, ["TIN", "NPI"]) + + self.assertIn("TIN", result) + self.assertIn("NPI", result) + self.assertIn("and", result) + + def test_build_matching_columns_string_tin_match(self): + """build_matching_columns_string shows matched TIN.""" + parent_row = pd.Series( + { + "PROV_GROUP_TIN": "123456789", + "PROV_GROUP_NPI": "111", + "PROV_GROUP_NAME_FULL": "Parent Hospital", + } + ) + child_row = pd.Series( + { + "PROV_GROUP_TIN": "123456789", + "PROV_GROUP_NPI": "222", + "PROV_GROUP_NAME_FULL": "Child Clinic", + } + ) + result = build_matching_columns_string(parent_row, child_row) + + self.assertIn("TIN", result) + self.assertNotIn("NPI", result) # NPI didn't match + + +class TestFlagMultiParentWithEarlyDates(unittest.TestCase): + """Tests for flag_multi_parent_with_early_dates function.""" + + def test_flags_group_with_multiple_early_parents(self): + """Flags groups where multiple parents have dates before children.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123", "TIN:123", "TIN:123"], + "parent": [True, True, False, False], + "parent_rank": [1.0, 2.0, np.nan, np.nan], + "fixed_effective_date": [ + "2024-01-01", + "2024-02-01", + "2024-06-01", + "2024-07-01", + ], + } + ) + result = flag_multi_parent_with_early_dates(df) + + # Group should be flagged because both parents have dates before children + self.assertEqual( + result["flag_multi_parent_with_early_dates"].iloc[0], + "multiple_parents_early_children", + ) + + def test_no_flag_single_parent(self): + """No flag when only one parent in group.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123"], + "parent": [True, False], + "parent_rank": [1.0, np.nan], + "fixed_effective_date": ["2024-01-01", "2024-06-01"], + } + ) + result = flag_multi_parent_with_early_dates(df) + + self.assertTrue(pd.isna(result["flag_multi_parent_with_early_dates"].iloc[0])) + + def test_no_flag_no_children(self): + """No flag when group has no children.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123"], + "parent": [True, True], + "parent_rank": [1.0, 2.0], + "fixed_effective_date": ["2024-01-01", "2024-02-01"], + } + ) + result = flag_multi_parent_with_early_dates(df) + + self.assertTrue(pd.isna(result["flag_multi_parent_with_early_dates"].iloc[0])) + + +class TestAssignChildRanks(unittest.TestCase): + """Tests for assign_child_ranks function.""" + + def test_orphan_children_get_0_prefix(self): + """Orphan children (no parent) get ranks starting with 0.0.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123"], + "parent": [False, False], + "assigned_parent_rank": ["no_parent", "no_parent"], + "fixed_effective_date": ["2024-01-01", "2024-02-01"], + } + ) + result = assign_child_ranks(df) + + # Both should have ranks starting with "0.0" + self.assertTrue(result["child_rank"].iloc[0].startswith("0.0")) + self.assertTrue(result["child_rank"].iloc[1].startswith("0.0")) + + def test_assigned_children_get_parent_prefix(self): + """Children assigned to parent get ranks with parent's rank prefix.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123", "TIN:123"], + "parent": [True, False, False], + "parent_rank": [1.0, np.nan, np.nan], + "assigned_parent_rank": [np.nan, 1.0, 1.0], + "fixed_effective_date": ["2024-01-01", "2024-03-01", "2024-02-01"], + } + ) + result = assign_child_ranks(df) + + # Parent should have combined_rank "1" + self.assertEqual(result["combined_rank"].iloc[0], "1") + # Children should have ranks like "1.0.1", "1.0.2" + child_ranks = result[result["parent"] == False]["child_rank"].tolist() + self.assertTrue(all(r.startswith("1.0.") for r in child_ranks)) + + def test_children_ranked_by_effective_date(self): + """Children within same parent group ranked by effective date.""" + df = pd.DataFrame( + { + "grouping_key": ["TIN:123", "TIN:123", "TIN:123"], + "parent": [True, False, False], + "parent_rank": [1.0, np.nan, np.nan], + "assigned_parent_rank": [np.nan, 1.0, 1.0], + "fixed_effective_date": ["2024-01-01", "2024-06-01", "2024-03-01"], + } + ) + result = assign_child_ranks(df) + + # Child with earlier date (index 2) should have lower rank number + child_df = result[result["parent"] == False].sort_values("fixed_effective_date") + ranks = child_df["child_rank"].tolist() + # Earlier date should have .1, later date should have .2 + self.assertIn(".1", ranks[0]) + self.assertIn(".2", ranks[1]) + + +class TestExtractOrdinal(unittest.TestCase): + """Tests for extract_ordinal function.""" + + def test_extract_numeric_ordinal(self): + """Extracts numeric ordinals like '1st', '2nd', '3rd'.""" + row = pd.Series({"contract_title_cleaned": "1st Amendment to Agreement"}) + result = extract_ordinal(row) + self.assertEqual(result, 1) + + row = pd.Series({"contract_title_cleaned": "2nd Amendment"}) + result = extract_ordinal(row) + self.assertEqual(result, 2) + + row = pd.Series({"contract_title_cleaned": "3rd Addendum"}) + result = extract_ordinal(row) + self.assertEqual(result, 3) + + def test_extract_word_ordinal(self): + """Extracts word ordinals like 'first', 'second', 'third'.""" + row = pd.Series({"contract_title_cleaned": "First Amendment to Agreement"}) + result = extract_ordinal(row) + self.assertEqual(result, 1) + + row = pd.Series({"contract_title_cleaned": "Second Amendment"}) + result = extract_ordinal(row) + self.assertEqual(result, 2) + + def test_no_ordinal_returns_none(self): + """Returns None when no ordinal found.""" + row = pd.Series({"contract_title_cleaned": "Provider Agreement"}) + result = extract_ordinal(row) + self.assertIsNone(result) + + +class TestConfigFactory(unittest.TestCase): + """Tests for ConfigFactory class.""" + + def test_get_molina_config(self): + """ConfigFactory returns Molina configuration.""" + config = ConfigFactory.get_config("molina") + + self.assertEqual(config.name, "molina") + self.assertIsNotNone(config.parent_identification) + + def test_get_caresource_config(self): + """ConfigFactory returns CareSource configuration.""" + config = ConfigFactory.get_config("caresource") + + self.assertEqual(config.name, "caresource") + self.assertIsNotNone(config.parent_identification) + + def test_get_bcbs_config(self): + """ConfigFactory returns BCBS configuration.""" + config = ConfigFactory.get_config("bcbs") + + self.assertEqual(config.name, "bcbs") + self.assertIsNotNone(config.parent_identification) + + def test_get_clover_config(self): + """ConfigFactory returns Clover Health configuration.""" + config = ConfigFactory.get_config("clover_health") + + self.assertEqual(config.name, "clover_health") + + def test_unknown_client_returns_molina_default(self): + """Unknown client falls back to Molina config.""" + config = ConfigFactory.get_config("unknown_client") + + # Should return a valid config (Molina as default) + self.assertIsNotNone(config) + + +if __name__ == "__main__": + unittest.main()