Merged in feature/generic-parent-child (pull request #852)
Feature/generic parent child
* Refactor parent_child module with generic hierarchy lineage
- Add generic_hierarchy_lineage.py for core parent-child mapping
- Add generic_hierarchy_lineage_qc.py for QC validation engine
- Add bcbs_generic_hierarchy_lineage.py for BCBS-specific processing
- Add column_mapper.py for automatic column detection
- Rename parent_child_preprocessing.py to generic_hierarchy_lineage_preprocessing.py
- Remove old parent_child_mapping.py (replaced by generic_hierarchy_lineage.py)
- Fix import paths from src.generic_hierarchy_lineage to src.parent_child
- Replace hardcoded BCBS xwalk path with configurable parameter
- Remove global warning suppression
* Fix code review issues in parent_child module
- Fix import paths from src.generic_hierarchy_lineage to src.parent_child
- Replace hardcoded BCBS crosswalk path with configurable xwalk_path param
- Remove global warnings.filterwarnings('ignore') suppressions
- Change bare except clauses to except Exception with logging
- Add explicit client= CLI argument with backward compatible fallback
- Convert print statements to logging in bcbs_generic_hierarchy_lineage.py
* Add tests for parent_child module and fix JSON path resolution
- Add 55 unit tests covering grouping keys, parent identification,
child ranking, and QC utilities
- Use pathlib to resolve JSON config paths relative to module location,
allowing tests to run from any directory
* WIP: Refactor parent_child module structure
* Extract constants to centralized location for parent_child module
- Create src/constants/parent_child/ with generic.py and bcbs.py
- Update pipeline.py to use centralized constants (ASSIGNMENT_NO_PARENT, tier values, grouping key config)
- Update bcbsnc/mapping.py to use BCBS-specific constants
- Update imports in __main__.py and bcbsnc/__init__.py
- Pass client parameter from runner.py and saas/main.py
* Fix code formatting in parent_child module
* Merge remote-tracking branch 'origin/main' into feature/generic-parent-child
* Fix client detection and output path in parent_child module
* Fix typo in numeric_mappings.json filename and update logging
* Add type hints to parent_child module
* Merged main into feature/generic-parent-child
Approved-by: Katon Minhas
This commit is contained in:
Siddhant Medar
committed by
Katon Minhas
Katon Minhas
parent
9e2de58ee4
commit
f412754167
@@ -0,0 +1,25 @@
|
||||
"""Parent-child module constants."""
|
||||
|
||||
from src.constants.parent_child.generic import (
|
||||
GROUPING_KEY_SEPARATOR,
|
||||
GROUPING_KEY_PREFIXES,
|
||||
TIER_ALL_IDENTIFIERS,
|
||||
TIER_TWO_IDENTIFIERS,
|
||||
TIER_ONE_IDENTIFIER,
|
||||
TIER_NO_IDENTIFIERS,
|
||||
ASSIGNMENT_NO_PARENT,
|
||||
DATE_FORMAT,
|
||||
LEGAL_SUFFIXES,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GROUPING_KEY_SEPARATOR",
|
||||
"GROUPING_KEY_PREFIXES",
|
||||
"TIER_ALL_IDENTIFIERS",
|
||||
"TIER_TWO_IDENTIFIERS",
|
||||
"TIER_ONE_IDENTIFIER",
|
||||
"TIER_NO_IDENTIFIERS",
|
||||
"ASSIGNMENT_NO_PARENT",
|
||||
"DATE_FORMAT",
|
||||
"LEGAL_SUFFIXES",
|
||||
]
|
||||
@@ -0,0 +1,103 @@
|
||||
"""
|
||||
BCBS-specific constants for parent-child module.
|
||||
|
||||
These constants are used by the BCBSNC-specific mapping logic.
|
||||
"""
|
||||
|
||||
# Crosswalk configuration
|
||||
XWALK_SHEET_NAME = "File Name to System Crosswalk"
|
||||
|
||||
# Column Names - Original (from source data)
|
||||
COL_CONTRACT_NAME_ORIG = "File Name"
|
||||
COL_AGREEMENT_NAME_ORIG = "Contract Name"
|
||||
COL_MULTIPLE_IRS_ORIG = "Multiple IRS Names"
|
||||
COL_PAGE_COUNT_ORIG = "Total Page Count"
|
||||
COL_LOB_ORIG = "Line of Business"
|
||||
COL_PROVIDER_TYPE_ORIG = "Provider Type"
|
||||
COL_PROVIDER_TYPE_L2_ORIG = "Provider Type - Level 2"
|
||||
COL_CONTRACT_EFF_DATE = "Contract Effective Date"
|
||||
|
||||
# Column Names - Standardized (after processing)
|
||||
COL_CONTRACT_NAME = "Contract Name"
|
||||
COL_AGREEMENT_NAME = "Agreement_Name (Contract Title)"
|
||||
COL_MULTIPLE_IRS = "MULTIPLE_IRS_NAMES"
|
||||
COL_PAGES = "Pages"
|
||||
COL_LOB = "Line of Business"
|
||||
COL_CONSOLIDATED_LOB = "consolidated_line_of_business"
|
||||
COL_CONSOLIDATED_PROVIDER = "consolidated_provider_type"
|
||||
COL_FIXED_CONTRACT_NAME = "fixed_contract_name"
|
||||
COL_IRS_NAME = "irs_name"
|
||||
COL_FOLDER = "Folder"
|
||||
COL_IRS_GROUP = "IRS_Group"
|
||||
COL_CONTRACT_TITLE_CLEAN = "contract_title_clean"
|
||||
COL_FINAL_EFF_DATE = "final_effective_date"
|
||||
COL_EXTRACTED_LOB = "extracted_lob"
|
||||
COL_IS_PARENT = "is_parent"
|
||||
COL_PARENT_CHILD_FLAG = "parent_child_flag"
|
||||
COL_COMBINED_RANK = "combined_rank"
|
||||
COL_NAMING_COL = "naming_col"
|
||||
COL_ASSIGNED_PARENT = "assigned_parent"
|
||||
COL_PROVIDER_TYPE_COUNT = "provider_type_count"
|
||||
COL_EFF_DATE_RANK = "effective_date_rank"
|
||||
COL_FINAL_RANK = "final_rank"
|
||||
COL_CHILD_RANK = "child_rank"
|
||||
COL_CHILD_INDEX = "child_index"
|
||||
|
||||
# Crosswalk columns
|
||||
XWALK_COL_FILE_NAME = "File Name"
|
||||
XWALK_COL_FOLDER = "Folder"
|
||||
|
||||
# Fields to consolidate
|
||||
FIELDS_TO_CONSOLIDATE = ["Line of Business", "Provider Type"]
|
||||
|
||||
# File extensions regex pattern
|
||||
FILE_EXTENSIONS_PATTERN = r"\.(pdf|txt|msg|xlsx|docx|tif|mht|doc|htm|xps)$"
|
||||
|
||||
# LOB Search Terms
|
||||
LOB_SEARCH_TERMS = [
|
||||
"MCR",
|
||||
"Medicare",
|
||||
"MCD",
|
||||
"Medicaid",
|
||||
"COMM",
|
||||
"Commercial",
|
||||
"Commercial Exchange",
|
||||
"MA",
|
||||
"Medicare Advantage",
|
||||
]
|
||||
|
||||
# LOB Mapping Dictionary
|
||||
LOB_MAPPING = {
|
||||
"medicare": "mcr",
|
||||
"medicare advantage": "mcr",
|
||||
"ma": "mcr",
|
||||
"medicaid": "mcd",
|
||||
"commercial": "comm",
|
||||
}
|
||||
|
||||
# Non-parent Keywords (contracts with these are children)
|
||||
NON_PARENT_KEYWORDS = [
|
||||
"tracking sheet",
|
||||
"letter",
|
||||
"notice",
|
||||
"memo",
|
||||
"form",
|
||||
"term sheet",
|
||||
"request",
|
||||
]
|
||||
|
||||
# Date extraction regex patterns
|
||||
DATE_PATTERNS = [
|
||||
r"(\d{1,2}[/-]\d{1,2}[/-]\d{2,4})", # 3-27-2015, 10/15/14
|
||||
r"(\d{1,2}[.]\d{1,2}[.]\d{2,4})", # 10.01.24, 07.01.2018
|
||||
r"(\d{1,2}[_]\d{1,2}[_]\d{2,4})", # 7_1_17, 01_01_2022
|
||||
r"(\d{1,2}[~]\d{1,2}[~]\d{2,4})", # 08~01~18
|
||||
r"(\d{1,2}\s+\d{1,2}\s+\d{2,4})", # 3 27 2015
|
||||
r"(\d{4}\s+\d{4})", # 0415 0515 (month-year chunks)
|
||||
r"(\d{8})", # 01012025 (MMDDYYYY)
|
||||
r"(\d{6})", # 120618 (MMDDYY)
|
||||
r"([A-Za-z]+ ?\d{1,2},?\s*\d{2,4})", # June1, 2016
|
||||
]
|
||||
|
||||
# Special IRS Group Replacements
|
||||
IRS_GROUP_REPLACEMENTS = {"cape": "cumberland"}
|
||||
@@ -0,0 +1,72 @@
|
||||
"""
|
||||
Generic constants for parent-child module.
|
||||
|
||||
These constants are used across the generic pipeline (non-client-specific).
|
||||
"""
|
||||
|
||||
# Grouping key configuration
|
||||
GROUPING_KEY_SEPARATOR = "|"
|
||||
GROUPING_KEY_PREFIXES = {
|
||||
"TIN": "TIN:",
|
||||
"NPI": "NPI:",
|
||||
"NAME": "NAME:",
|
||||
}
|
||||
|
||||
# Tier levels (based on number of identifiers present)
|
||||
TIER_ALL_IDENTIFIERS = 1 # All 3: TIN + NPI + NAME
|
||||
TIER_TWO_IDENTIFIERS = 2 # Any 2 identifiers
|
||||
TIER_ONE_IDENTIFIER = 3 # Only 1 identifier
|
||||
TIER_NO_IDENTIFIERS = 0 # No identifiers (immediate orphan)
|
||||
|
||||
# Assignment status
|
||||
ASSIGNMENT_NO_PARENT = "no_parent"
|
||||
|
||||
# Date format
|
||||
DATE_FORMAT = "%m/%d/%Y"
|
||||
|
||||
# Legal entity suffixes to remove during cleaning
|
||||
LEGAL_SUFFIXES = [
|
||||
"inc",
|
||||
"inc.",
|
||||
"llc",
|
||||
"l.l.c.",
|
||||
"corp",
|
||||
"corp.",
|
||||
"co",
|
||||
"co.",
|
||||
"ltd",
|
||||
"ltd.",
|
||||
"lp",
|
||||
"llp",
|
||||
"pty",
|
||||
"company",
|
||||
"corporation",
|
||||
"incorporated",
|
||||
"limited",
|
||||
"dba",
|
||||
"d/b/a",
|
||||
]
|
||||
|
||||
# Pluralization rules for text normalization
|
||||
PLURAL_TO_SINGULAR = {
|
||||
"centers": "center",
|
||||
"plans": "plan",
|
||||
"services": "service",
|
||||
"systems": "system",
|
||||
"solutions": "solution",
|
||||
"associates": "associate",
|
||||
}
|
||||
|
||||
# File extensions to remove from contract names
|
||||
FILE_EXTENSIONS = [
|
||||
".txt",
|
||||
".pdf",
|
||||
".msg",
|
||||
".xlsx",
|
||||
".docx",
|
||||
".tif",
|
||||
".mht",
|
||||
".doc",
|
||||
".htm",
|
||||
".xps",
|
||||
]
|
||||
@@ -0,0 +1,18 @@
|
||||
"""
|
||||
Parent-Child Contract Hierarchy Module.
|
||||
|
||||
Public API:
|
||||
main - Main module (backward compatible)
|
||||
ColumnMapper - Auto-detection of column names
|
||||
"""
|
||||
|
||||
# Re-export main module for backward compatibility
|
||||
# This allows: from src.parent_child import main
|
||||
from src.parent_child import __main__ as main
|
||||
|
||||
from src.parent_child.column_mapper import ColumnMapper
|
||||
|
||||
__all__ = [
|
||||
"main",
|
||||
"ColumnMapper",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from src.parent_child.qc import qc_main
|
||||
import src.config as config
|
||||
from src.config import get_arg_value
|
||||
from src.parent_child.preprocessing import (
|
||||
parent_child_preprocessing,
|
||||
)
|
||||
from src.parent_child.pipeline import parent_child_mapping
|
||||
from src.parent_child.column_mapper import ColumnMapper
|
||||
from src.parent_child.bcbsnc import bcbs_main
|
||||
import os
|
||||
import re
|
||||
import pandas as pd
|
||||
from src.utils import io_utils
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
# Optional command-line arguments
|
||||
BCBSNC_XWALK_PATH = get_arg_value("bcbsnc_xwalk_path", None)
|
||||
# Explicit client override (takes precedence over string matching)
|
||||
# Supported values: bcbs, caresource, molina, cnc, clover_health
|
||||
CLIENT_ARG = get_arg_value("client", None)
|
||||
|
||||
|
||||
def extract_client_name(file_path: str) -> str:
|
||||
"""
|
||||
Extract client name from input file path for output naming.
|
||||
|
||||
Args:
|
||||
file_path: Input file path (e.g., 'Clover Health_132.xlsx' or full path)
|
||||
|
||||
Returns:
|
||||
Cleaned client name (e.g., 'clover_health')
|
||||
"""
|
||||
# Get just the filename without extension
|
||||
filename = os.path.basename(file_path)
|
||||
name_without_ext = os.path.splitext(filename)[0]
|
||||
|
||||
# Remove trailing numbers and underscores (e.g., '_132' from 'Clover Health_132')
|
||||
cleaned_name = re.sub(r"[_\s]*\d+$", "", name_without_ext)
|
||||
|
||||
# Convert to lowercase and replace spaces with underscores
|
||||
cleaned_name = cleaned_name.lower().strip().replace(" ", "_")
|
||||
|
||||
# Remove any double underscores
|
||||
cleaned_name = re.sub(r"_+", "_", cleaned_name)
|
||||
|
||||
return cleaned_name if cleaned_name else "output"
|
||||
|
||||
|
||||
def write_pc_excel(
|
||||
mapped_df: pd.DataFrame,
|
||||
pc_df: Optional[pd.DataFrame],
|
||||
output_dir: str,
|
||||
output_name: str,
|
||||
) -> str:
|
||||
"""
|
||||
Write parent-child mapping results to Excel with multiple sheets.
|
||||
|
||||
Args:
|
||||
mapped_df: Main dataframe with PC flags merged back
|
||||
pc_df: Detailed PC mapping dataframe
|
||||
output_dir: Directory to write to
|
||||
output_name: Client-based output name for filename
|
||||
"""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
output_path = os.path.join(
|
||||
output_dir, f"{output_name}_generic_hierarchy_lineage_output.xlsx"
|
||||
)
|
||||
|
||||
with pd.ExcelWriter(output_path, engine="openpyxl") as writer:
|
||||
mapped_df.to_excel(writer, sheet_name="Main_Output", index=False)
|
||||
if pc_df is not None:
|
||||
pc_df.to_excel(writer, sheet_name="PC_Details", index=False)
|
||||
|
||||
logging.info(f"Excel file written to: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
def main(doczy_output_for_pc: str, client: Optional[str] = None) -> int:
|
||||
"""
|
||||
Main entry point for parent-child mapping.
|
||||
|
||||
Args:
|
||||
doczy_output_for_pc: Path to input file (local or S3)
|
||||
client: Optional client identifier. If provided, takes precedence over
|
||||
CLI argument and path-based detection.
|
||||
Supported values: bcbs, caresource, molina, cnc, clover_health
|
||||
|
||||
Returns:
|
||||
Number of rows processed
|
||||
"""
|
||||
# Extract client name from input file for output naming
|
||||
client_name = extract_client_name(doczy_output_for_pc)
|
||||
|
||||
# Generate timestamp for this run
|
||||
run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{client_name}"
|
||||
|
||||
try:
|
||||
# Read input file
|
||||
df_read = io_utils.read_DataFrame(doczy_output_for_pc)
|
||||
logging.info(f"Original shape: {df_read.shape}")
|
||||
logging.info(f"Original columns: {df_read.columns.tolist()}")
|
||||
|
||||
# Determine client: function param > CLI arg > path-based detection
|
||||
if client:
|
||||
client = client.lower()
|
||||
logging.info(f"Client specified via parameter: {client}")
|
||||
elif CLIENT_ARG:
|
||||
client = CLIENT_ARG.lower()
|
||||
logging.info(f"Client specified via CLI argument: {client}")
|
||||
else:
|
||||
# Fallback: detect client from file path (backward compatibility)
|
||||
path_lower = doczy_output_for_pc.lower()
|
||||
if "bcbsnc" in path_lower:
|
||||
client = "bcbsnc"
|
||||
elif "caresource" in path_lower:
|
||||
client = "caresource"
|
||||
else:
|
||||
client = "molina"
|
||||
logging.info(f"Client detected from path: {client}")
|
||||
|
||||
# BCBS-specific processing
|
||||
if client == "bcbsnc":
|
||||
logging.info("Using BCBSNC specialized reader.")
|
||||
mapped_df = df_read
|
||||
pc_df = bcbs_main(df_read, xwalk_path=BCBSNC_XWALK_PATH)
|
||||
else:
|
||||
# Auto-detect column mappings
|
||||
logging.info("=" * 80)
|
||||
logging.info("Starting automatic column detection...")
|
||||
logging.info("=" * 80)
|
||||
col_mapper = ColumnMapper(df_read)
|
||||
col_mapper.log_mapping_summary()
|
||||
|
||||
# Preprocess with column mapper - returns cleaned_df and original_df
|
||||
cleaned_df, original_df = parent_child_preprocessing(df_read, col_mapper)
|
||||
logging.info(
|
||||
f"Unique contracts in the output: {cleaned_df['FILE_NAME'].nunique()}"
|
||||
)
|
||||
|
||||
# Perform parent-child mapping - returns original_df_with_pc and pc_df
|
||||
original_df_with_pc, pc_df = parent_child_mapping(cleaned_df, original_df)
|
||||
|
||||
# Save detailed results
|
||||
output_dir = os.path.join("outputs/parent_child", run_timestamp)
|
||||
pc_df1 = qc_main(pc_df, client)
|
||||
|
||||
output_filename = f"{client_name}_generic_hierarchy_lineage_output.xlsx"
|
||||
if config.WRITE_PC_TO_S3:
|
||||
# Write locally first, then upload to S3
|
||||
write_pc_excel(df_read, pc_df1, output_dir, client_name)
|
||||
# TODO: Add S3 upload logic if needed
|
||||
logging.info(f"PC results saved: {run_timestamp}/{output_filename}")
|
||||
else:
|
||||
write_pc_excel(df_read, pc_df1, output_dir, client_name)
|
||||
logging.info(f"PC results saved locally: {output_dir}/{output_filename}")
|
||||
|
||||
# TODO: check with dev
|
||||
return pc_df.shape[0] if pc_df is not None else mapped_df.shape[0]
|
||||
except Exception as e:
|
||||
logging.error(f"Parent child mapping failed due to {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
# Generate timestamp for this run
|
||||
run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}"
|
||||
|
||||
if config.DOCZY_OUTPUT_FOR_PC:
|
||||
results = main(config.DOCZY_OUTPUT_FOR_PC)
|
||||
logging.info(f"PC Mapping complete. Processed {results} files.")
|
||||
else:
|
||||
logging.warning("No input file specified in config.DOCZY_OUTPUT_FOR_PC")
|
||||
@@ -0,0 +1,5 @@
|
||||
"""BCBSNC-specific parent-child processing."""
|
||||
|
||||
from src.parent_child.bcbsnc.mapping import bcbs_main
|
||||
|
||||
__all__ = ["bcbs_main"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,266 @@
|
||||
"""
|
||||
Column Mapper Utility for Generic Parent-Child Processing
|
||||
|
||||
This module auto-detects column names from input DataFrames to make the
|
||||
parent-child processing pipeline work across different data sources without
|
||||
requiring JSON/YAML configuration files.
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import re
|
||||
import logging
|
||||
from typing import Dict, Optional, List
|
||||
|
||||
|
||||
class ColumnMapper:
|
||||
"""
|
||||
Automatically detects and maps column names from input DataFrame to
|
||||
standardized internal column names used in processing.
|
||||
"""
|
||||
|
||||
# Define patterns for each required field
|
||||
COLUMN_PATTERNS = {
|
||||
"FILE_NAME": [
|
||||
r"file_name",
|
||||
r"^file.*name$",
|
||||
r"^contract.*name$",
|
||||
r"^document.*name$",
|
||||
r"^filename$",
|
||||
r"^doc.*name$",
|
||||
],
|
||||
"CONTRACT_TITLE": [
|
||||
r"^agreement_name.*contract.*title.*$",
|
||||
r"^agreement.*name$",
|
||||
r"^contract.*title$",
|
||||
r"^title$",
|
||||
r"^contract.*name.*title$",
|
||||
r"^agreement.*title$",
|
||||
r"^doc.*title$",
|
||||
],
|
||||
"PAYER_NAME": [
|
||||
r"^payer.*name$",
|
||||
r"^payer$",
|
||||
r"^plan.*name$",
|
||||
r"^insurance.*name$",
|
||||
r"^carrier.*name$",
|
||||
],
|
||||
"PROV_GROUP_NAME_FULL": [
|
||||
r"^prov.*group.*name.*full$",
|
||||
r"^irs_name",
|
||||
r"^provider.*group.*name$",
|
||||
r"^provider.*name$",
|
||||
r"^prov.*name$",
|
||||
r"^group.*name$",
|
||||
],
|
||||
"PROV_GROUP_TIN": [
|
||||
r"^prov.*group.*tin$",
|
||||
r"^irs.*#$",
|
||||
r"^irs.*number$",
|
||||
r"^tin$",
|
||||
r"^tax.*id$",
|
||||
r"^tax.*identification$",
|
||||
r"^employer.*id$",
|
||||
r"^ein$",
|
||||
],
|
||||
"PROV_GROUP_NPI": [
|
||||
r"^prov.*group.*npi$",
|
||||
r"^npi.*10.*digit.*$",
|
||||
r"^npi$",
|
||||
r"^national.*provider.*id$",
|
||||
r"^provider.*npi$",
|
||||
],
|
||||
"EFFECTIVE_DATE": [
|
||||
r"aarete_derived_effective_dt",
|
||||
r"^.*effective.*date$",
|
||||
r"^eff.*date$",
|
||||
r"^start.*date$",
|
||||
r"^begin.*date$",
|
||||
r"^contract.*effective.*date$",
|
||||
],
|
||||
"TERMINATION_DATE": [
|
||||
r"aarete_derived_termination_dt",
|
||||
r"^.*termination.*date$",
|
||||
r"^term.*date$",
|
||||
r"^end.*date$",
|
||||
r"^expiration.*date$",
|
||||
r"^expire.*date$",
|
||||
],
|
||||
"LOB": [
|
||||
r"aarete_derived_lob",
|
||||
r"^lob$",
|
||||
r"^line.*of.*business$",
|
||||
r"^product.*line$",
|
||||
r"^business.*line$",
|
||||
],
|
||||
"SERVICE_TERM": [
|
||||
r"^service.*term$",
|
||||
r"^term$",
|
||||
r"^service.*type$",
|
||||
r"^service.*category$",
|
||||
],
|
||||
"PAYER_STATE": [
|
||||
r"^payer.*state$",
|
||||
r"^state$",
|
||||
r"^location.*state$",
|
||||
r"^plan.*state$",
|
||||
r"^health.*plan.*state$",
|
||||
],
|
||||
}
|
||||
|
||||
# Optional columns (won't fail if not found)
|
||||
OPTIONAL_COLUMNS = [
|
||||
"PAYER_STATE",
|
||||
"SERVICE_TERM",
|
||||
"PROV_GROUP_TIN",
|
||||
"PROV_GROUP_NPI",
|
||||
]
|
||||
|
||||
def __init__(self, df: pd.DataFrame):
|
||||
"""
|
||||
Initialize the ColumnMapper with a DataFrame.
|
||||
|
||||
Args:
|
||||
df: Input DataFrame to analyze
|
||||
"""
|
||||
self.df = df
|
||||
self.column_mapping = {}
|
||||
self._detect_columns()
|
||||
|
||||
def _normalize_column_name(self, col_name: str) -> str:
|
||||
"""Normalize column name for pattern matching."""
|
||||
return col_name.lower().strip().replace(" ", "_")
|
||||
|
||||
def _match_column(
|
||||
self, patterns: List[str], available_columns: List[str]
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
Match a column name against a list of regex patterns.
|
||||
|
||||
Args:
|
||||
patterns: List of regex patterns to match
|
||||
available_columns: List of available column names
|
||||
|
||||
Returns:
|
||||
Matched column name or None
|
||||
"""
|
||||
for col in available_columns:
|
||||
normalized_col = self._normalize_column_name(col)
|
||||
for pattern in patterns:
|
||||
if re.match(pattern, normalized_col, re.IGNORECASE):
|
||||
return col
|
||||
return None
|
||||
|
||||
def _detect_columns(self):
|
||||
"""Auto-detect column mappings from DataFrame."""
|
||||
available_columns = self.df.columns.tolist()
|
||||
logging.info(f"Available columns in input DataFrame: {available_columns}")
|
||||
|
||||
for standard_name, patterns in self.COLUMN_PATTERNS.items():
|
||||
matched_col = self._match_column(patterns, available_columns)
|
||||
|
||||
if matched_col:
|
||||
self.column_mapping[standard_name] = matched_col
|
||||
logging.info(f"Mapped '{standard_name}' -> '{matched_col}'")
|
||||
elif standard_name not in self.OPTIONAL_COLUMNS:
|
||||
logging.warning(
|
||||
f"Could not find column for '{standard_name}'. Patterns tried: {patterns}"
|
||||
)
|
||||
|
||||
# Validate required columns are present
|
||||
self._validate_required_columns()
|
||||
|
||||
def _validate_required_columns(self):
|
||||
"""Validate that all required columns were found."""
|
||||
required_fields = [
|
||||
"FILE_NAME",
|
||||
"CONTRACT_TITLE",
|
||||
"PAYER_NAME",
|
||||
"PROV_GROUP_NAME_FULL",
|
||||
"EFFECTIVE_DATE",
|
||||
"TERMINATION_DATE",
|
||||
"LOB",
|
||||
]
|
||||
|
||||
missing_fields = [
|
||||
field for field in required_fields if field not in self.column_mapping
|
||||
]
|
||||
|
||||
if missing_fields:
|
||||
error_msg = (
|
||||
f"Missing required columns: {missing_fields}\n"
|
||||
f"Available columns: {self.df.columns.tolist()}\n"
|
||||
f"Successfully mapped: {self.column_mapping}\n\n"
|
||||
f"Please ensure your input file contains columns that match these patterns:\n"
|
||||
)
|
||||
for field in missing_fields:
|
||||
patterns = self.COLUMN_PATTERNS.get(field, [])
|
||||
error_msg += f" {field}: {patterns}\n"
|
||||
|
||||
raise ValueError(error_msg)
|
||||
|
||||
def get(self, standard_name: str) -> Optional[str]:
|
||||
"""
|
||||
Get the actual column name for a standard field name.
|
||||
|
||||
Args:
|
||||
standard_name: Standard field name (e.g., 'FILE_NAME')
|
||||
|
||||
Returns:
|
||||
Actual column name in DataFrame or None if not found
|
||||
"""
|
||||
return self.column_mapping.get(standard_name)
|
||||
|
||||
def get_all_mappings(self) -> Dict[str, str]:
|
||||
"""Get all column mappings."""
|
||||
return self.column_mapping.copy()
|
||||
|
||||
def rename_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Rename DataFrame columns to standardized names.
|
||||
|
||||
Args:
|
||||
df: Input DataFrame
|
||||
|
||||
Returns:
|
||||
DataFrame with standardized column names
|
||||
"""
|
||||
reverse_mapping = {v: k for k, v in self.column_mapping.items()}
|
||||
return df.rename(columns=reverse_mapping)
|
||||
|
||||
def has_column(self, standard_name: str) -> bool:
|
||||
"""
|
||||
Check if a column mapping exists.
|
||||
|
||||
Args:
|
||||
standard_name: Standard field name
|
||||
|
||||
Returns:
|
||||
True if mapping exists, False otherwise
|
||||
"""
|
||||
return standard_name in self.column_mapping
|
||||
|
||||
def get_one_to_n_fields(self) -> List[str]:
|
||||
"""
|
||||
Get list of one-to-many fields that should be consolidated.
|
||||
|
||||
Returns:
|
||||
List of actual column names for one-to-many fields
|
||||
"""
|
||||
one_to_n_fields = []
|
||||
|
||||
if self.has_column("LOB"):
|
||||
one_to_n_fields.append(self.get("LOB"))
|
||||
|
||||
if self.has_column("SERVICE_TERM"):
|
||||
one_to_n_fields.append(self.get("SERVICE_TERM"))
|
||||
|
||||
return one_to_n_fields
|
||||
|
||||
def log_mapping_summary(self):
|
||||
"""Log a summary of detected column mappings."""
|
||||
logging.info("=" * 60)
|
||||
logging.info("Column Mapping Summary:")
|
||||
logging.info("=" * 60)
|
||||
for standard, actual in sorted(self.column_mapping.items()):
|
||||
logging.info(f" {standard:25} -> {actual}")
|
||||
logging.info("=" * 60)
|
||||
@@ -1,56 +0,0 @@
|
||||
import src.config as config
|
||||
|
||||
# from src.parent_child.file_reading_utils import read_file_from_path
|
||||
from src.parent_child.parent_child_preprocessing import parent_child_preprocessing
|
||||
from src.parent_child.parent_child_mapping import parent_child_mapping
|
||||
import os
|
||||
from src.utils import io_utils
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def main(doczy_output_for_pc):
|
||||
# Generate timestamp for this run
|
||||
run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}"
|
||||
|
||||
try:
|
||||
df_read = io_utils.read_DataFrame(doczy_output_for_pc)
|
||||
logging.info(f"original shape: {df_read.shape}")
|
||||
cleaned_df = parent_child_preprocessing(df_read)
|
||||
logging.info(
|
||||
f"Unique contracts in the output: {cleaned_df["FILE_NAME"].nunique()}"
|
||||
)
|
||||
mapped_df = parent_child_mapping(cleaned_df)
|
||||
|
||||
# Save detailed results using same pattern as investment pipeline
|
||||
if config.WRITE_PC_TO_S3:
|
||||
io_utils.write_s3(
|
||||
mapped_df, "Parent_Child_Mapping_", run_timestamp, "parent_child"
|
||||
)
|
||||
logging.info(
|
||||
f"PC results uploaded to S3: {config.BATCH_ID}/{run_timestamp}/{config.BATCH_ID}-PC.csv"
|
||||
)
|
||||
else:
|
||||
io_utils.write_local(
|
||||
mapped_df, "Parent_Child_Mapping_", run_timestamp, "parent_child"
|
||||
)
|
||||
logging.info(
|
||||
f"PC results saved locally: {config.CONSOLIDATED_OUTPUT_DIRECTORY}/{run_timestamp}/{config.BATCH_ID}-PC.csv"
|
||||
)
|
||||
return mapped_df.shape[0]
|
||||
except Exception as e:
|
||||
logging.error(f"Parent child mapping failed due to {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
)
|
||||
# Generate timestamp for this run
|
||||
run_timestamp = f"run_{datetime.now().strftime('%Y%m%d_%H-%M')}_{config.BATCH_ID}"
|
||||
|
||||
if config.DOCZY_OUTPUT_FOR_PC:
|
||||
results = main(config.DOCZY_OUTPUT_FOR_PC)
|
||||
|
||||
logging.info(f"PC Mapping complete. Processed {results} files.")
|
||||
@@ -1,890 +0,0 @@
|
||||
import warnings
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
import pandas as pd
|
||||
import re
|
||||
from src.parent_child.parent_child_preprocessing import (
|
||||
clean_effective_date,
|
||||
clean_termination_date,
|
||||
)
|
||||
import numpy as np
|
||||
import logging
|
||||
from src.utils import io_utils
|
||||
|
||||
|
||||
def create_df_per_tin_presence(df, is_tin_present):
|
||||
"""
|
||||
Filters the DataFrame based on the presence or absence of TIN.
|
||||
|
||||
Parameters:
|
||||
----------
|
||||
df : input DataFrame containing TIN and ICMP-related columns.
|
||||
is_tin_present : bool
|
||||
If True, returns rows where 'TIN' is present (not NaN).
|
||||
If False, returns rows where 'TIN' is missing and 'icmp_part' is present.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
pandas.DataFrame
|
||||
Filtered DataFrame according to the TIN presence logic.
|
||||
"""
|
||||
if is_tin_present:
|
||||
return df[~df["TIN"].isna()]
|
||||
else:
|
||||
return df[df["TIN"].isna() & df["icmp_part"].notna()]
|
||||
|
||||
|
||||
def filter_on_exclude_keywords(df, is_tin_present):
|
||||
"""
|
||||
Flags rows as 'parent' based on exclusion keywords.
|
||||
|
||||
Parameters:
|
||||
----------
|
||||
df : input DataFrame containing contract information.
|
||||
is_tin_present : bool
|
||||
Flag used to select the appropriate keyword list from `exclude_keywords_dict`.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
updated DataFrame with the 'parent' column set to False
|
||||
if either 'contract_title_cleaned' or 'fixed_contract_name'
|
||||
contains any of the exclusion keywords (case-insensitive).
|
||||
"""
|
||||
df["parent"] = True
|
||||
|
||||
exclude_keywords_dict = io_utils.convert_json_to_dict(
|
||||
"constants/mappings/exclude_keywords_for_pc.json"
|
||||
)
|
||||
exclude_keywords_list = exclude_keywords_dict[is_tin_present]
|
||||
exclude_pattern = (
|
||||
r"\b(?:" + "|".join(re.escape(kw) for kw in exclude_keywords_list) + r")\b"
|
||||
)
|
||||
|
||||
df["parent"] = ~(
|
||||
df["contract_title_cleaned"].str.contains(exclude_pattern, case=False, na=False)
|
||||
| df["contract_name_cleaned"].str.contains(
|
||||
exclude_pattern, case=False, na=False
|
||||
)
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def filter_on_missing_title(df):
|
||||
"""
|
||||
Updates 'parent' flag based on missing or empty contract titles and keywords in FILE_NAME.
|
||||
|
||||
Rules:
|
||||
1. If 'contract_title_cleaned' is NaN or empty, default 'parent' to False.
|
||||
2. Exception: if 'contract_title_cleaned' is null and 'agmt' in 'contract_name', set parent = True.
|
||||
3. Exception to exception: if 'contract_title_cleaned' is null and both 'agmt' and 'letter' in 'contract_name', set parent = False.
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
# Normalize text columns
|
||||
contract_name_lower = df["contract_name_cleaned"].astype(str).str.lower()
|
||||
title_missing = df["contract_title_cleaned"].isna() | df[
|
||||
"contract_title_cleaned"
|
||||
].astype(str).str.strip().isin(["", "nan"])
|
||||
|
||||
# Base: set parent False where title is missing
|
||||
df["parent"] = df["parent"].where(~title_missing, False)
|
||||
|
||||
# Compute final parent mask in one step
|
||||
# Parent = True if missing title & 'agmt' in name, except if also 'letter' in name
|
||||
df.loc[
|
||||
title_missing
|
||||
& contract_name_lower.str.contains("agmt")
|
||||
& ~contract_name_lower.str.contains("letter"),
|
||||
"parent",
|
||||
] = True
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def flag_duplicates_within_group(df, grouper_col):
|
||||
"""
|
||||
Flags duplicates within each group for rows where parent == True,
|
||||
based on an exact match of:
|
||||
- 'contract_title_cleaned'
|
||||
- 'fixed_effective_date'
|
||||
- 'payer_name_cleaned'
|
||||
|
||||
NaN values in 'payer_name_cleaned' are treated as equal when both are NaN,
|
||||
and as different when only one is NaN.
|
||||
|
||||
Non-parent rows are returned unchanged.
|
||||
"""
|
||||
df = df.copy()
|
||||
df["is_duplicate"] = False # default for all rows
|
||||
|
||||
# Work only on parent rows
|
||||
mask = df["parent"] == True
|
||||
df_parent = df.loc[mask].copy()
|
||||
|
||||
# Handle NaNs consistently
|
||||
df_parent["_payer"] = df_parent["payer_name_cleaned"].fillna("__NULL__")
|
||||
|
||||
# Build duplicate key
|
||||
df_parent["_dup_key"] = (
|
||||
df_parent["contract_title_cleaned"].astype(str)
|
||||
+ "||"
|
||||
+ df_parent["fixed_effective_date"].astype(str)
|
||||
)
|
||||
|
||||
# Detect duplicates
|
||||
df_parent["is_duplicate"] = df_parent.groupby(grouper_col)["_dup_key"].transform(
|
||||
lambda x: x.duplicated(keep=False)
|
||||
)
|
||||
|
||||
# Update only parent rows in original df
|
||||
df.loc[mask, "is_duplicate"] = df_parent["is_duplicate"]
|
||||
|
||||
# Clean temp cols (only in df_parent, since they weren’t in df originally)
|
||||
return df
|
||||
|
||||
|
||||
def rank_parents(df, grouper_col):
|
||||
"""
|
||||
Assigns a rank to 'parent' contracts within each group defined by `grouper_col`.
|
||||
|
||||
Ranking logic:
|
||||
1. Only rows where `parent == True` are ranked.
|
||||
2. Ranking is based on earliest 'fixed_effective_date' first (NaT placed last).
|
||||
3. Each parent gets its own rank (duplicates are not merged).
|
||||
4. Null dates are ranked last.
|
||||
"""
|
||||
df = df.copy()
|
||||
df["parent_rank"] = np.nan # Initialize
|
||||
|
||||
# Ensure effective_date is datetime
|
||||
df["_sort_date"] = pd.to_datetime(df["fixed_effective_date"], errors="coerce")
|
||||
|
||||
for group_val, group_df in df.groupby(grouper_col):
|
||||
parents = group_df[group_df["parent"] == True].copy()
|
||||
if parents.empty:
|
||||
continue
|
||||
|
||||
# Replace NaT with a large future date for sorting → ensures null dates go last
|
||||
parents["_rank_date"] = parents["_sort_date"].fillna(pd.Timestamp.max)
|
||||
|
||||
# Sort and assign unique ranks
|
||||
parents = parents.sort_values("_rank_date")
|
||||
ranks = range(1, len(parents) + 1)
|
||||
|
||||
df.loc[parents.index, "parent_rank"] = list(ranks)
|
||||
|
||||
return df.drop(columns=["_sort_date", "_rank_date"], errors="ignore")
|
||||
|
||||
|
||||
def flag_multi_parent_with_early_dates(
|
||||
df: pd.DataFrame, grouper_col: str
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Flags groups where:
|
||||
1. There are multiple unique parent ranks.
|
||||
2. The group has children.
|
||||
3. At least two parents have effective dates earlier than at least one child.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
The DataFrame containing at least:
|
||||
- 'parent' (bool)
|
||||
- 'parent_rank'
|
||||
- 'fixed_effective_date'
|
||||
- Grouping column specified by `grouper_col`
|
||||
grouper_col : str
|
||||
The column name to group by (e.g., 'TIN').
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
The input DataFrame with an added column:
|
||||
'flag_multi_parent_with_early_dates' containing either the flag value
|
||||
'multiple_parents_early_children' or NaN.
|
||||
|
||||
"""
|
||||
df = df.copy()
|
||||
df["fixed_effective_date"] = pd.to_datetime(
|
||||
df["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
flag_dict = {}
|
||||
|
||||
for group_val, group_df in df.groupby(grouper_col):
|
||||
parents_df = group_df[group_df["parent"] == True]
|
||||
children_df = group_df[group_df["parent"] == False]
|
||||
|
||||
# Skip if less than 2 unique parents or no children
|
||||
unique_parents = parents_df["parent_rank"].dropna().unique()
|
||||
if len(unique_parents) <= 1 or children_df.empty:
|
||||
continue
|
||||
|
||||
# Get all child dates
|
||||
child_dates = children_df["fixed_effective_date"].dropna()
|
||||
if child_dates.empty:
|
||||
continue
|
||||
|
||||
# Count parents with earlier dates than any child
|
||||
count_early_parents = sum(
|
||||
pd.notna(parent_row["fixed_effective_date"])
|
||||
and (child_dates > parent_row["fixed_effective_date"]).any()
|
||||
for _, parent_row in parents_df.iterrows()
|
||||
)
|
||||
|
||||
if count_early_parents >= 2:
|
||||
flag_dict[group_val] = "multiple_parents_early_children"
|
||||
|
||||
# Map flags back to DataFrame
|
||||
df["flag_multi_parent_with_early_dates"] = df[grouper_col].map(flag_dict)
|
||||
return df
|
||||
|
||||
|
||||
def assign_parents_with_state_and_payer(
|
||||
df, grouper_col, flag_col="flag_multi_parent_with_early_dates"
|
||||
):
|
||||
"""
|
||||
Assigns each child contract to a parent contract with strict TIN, state, payer, and date logic.
|
||||
"""
|
||||
|
||||
df = df.copy()
|
||||
df["fixed_effective_date"] = pd.to_datetime(
|
||||
df["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
|
||||
parents = df[df["parent"]].copy()
|
||||
children = df[~df["parent"]].copy()
|
||||
assignments = []
|
||||
|
||||
for idx, child in children.iterrows():
|
||||
# Step 1: Filter parents within TIN group
|
||||
group_parents = parents[parents[grouper_col] == child[grouper_col]].copy()
|
||||
|
||||
if group_parents.empty:
|
||||
assignments.append(("no_parent", idx))
|
||||
continue
|
||||
|
||||
# Step 2: Filter by matching state
|
||||
state_parents = group_parents[
|
||||
group_parents["PAYER_STATE"] == child["PAYER_STATE"]
|
||||
]
|
||||
|
||||
if state_parents.empty:
|
||||
assignments.append(("no_parent", idx))
|
||||
continue
|
||||
|
||||
# Step 3: Apply effective date constraint (parent date <= child date)
|
||||
if pd.notna(child["fixed_effective_date"]):
|
||||
state_parents = state_parents[
|
||||
pd.to_datetime(state_parents["fixed_effective_date"], errors="coerce")
|
||||
<= child["fixed_effective_date"]
|
||||
]
|
||||
|
||||
if state_parents.empty:
|
||||
assignments.append(("no_parent", idx))
|
||||
continue
|
||||
|
||||
# Step 4: Tie-breaker on payer_name
|
||||
if len(state_parents) > 1:
|
||||
if pd.notna(child.get("payer_name_cleaned")):
|
||||
payer_match = state_parents[
|
||||
state_parents["payer_name_cleaned"] == child["payer_name_cleaned"]
|
||||
]
|
||||
if len(payer_match) == 1:
|
||||
assignments.append((payer_match.iloc[0]["parent_rank"], idx))
|
||||
continue
|
||||
elif len(payer_match) > 1:
|
||||
# Still multiple → pick latest
|
||||
latest_parent = payer_match.sort_values(
|
||||
"parent_rank", ascending=False
|
||||
).iloc[0]
|
||||
assignments.append((latest_parent["parent_rank"], idx))
|
||||
continue
|
||||
else:
|
||||
# No matching payer → no parent
|
||||
assignments.append(("no_parent", idx))
|
||||
continue
|
||||
else:
|
||||
# Child payer_name null → no parent
|
||||
assignments.append(("no_parent", idx))
|
||||
continue
|
||||
else:
|
||||
# Only one parent left → assign
|
||||
assignments.append((state_parents.iloc[0]["parent_rank"], idx))
|
||||
|
||||
children["assigned_parent_rank"] = None
|
||||
for rank, idx in assignments:
|
||||
children.at[idx, "assigned_parent_rank"] = rank
|
||||
|
||||
return pd.concat([parents, children]).sort_index()
|
||||
|
||||
|
||||
def assign_no_parent_by_title(
|
||||
df, grouper_col, contract_title_col="contract_title_cleaned"
|
||||
):
|
||||
"""
|
||||
Second-pass assignment for children with 'no_parent' in assigned_parent_rank.
|
||||
Assigns based on contract title substring match, with same restrictions:
|
||||
- Must match on grouper_col (e.g., TIN).
|
||||
- Parent's contract title must be substring of child's title (case-insensitive).
|
||||
- Parent's effective date must be <= child's date.
|
||||
- Pick latest parent_rank if multiple.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
Must contain:
|
||||
- 'assigned_parent_rank'
|
||||
- 'parent' (bool)
|
||||
- 'parent_rank'
|
||||
- 'fixed_effective_date'
|
||||
- contract_title_col (str)
|
||||
- grouper_col (str)
|
||||
grouper_col : str
|
||||
Column name for grouping (e.g., 'TIN').
|
||||
contract_title_col : str
|
||||
Column name containing the contract title.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Updated DataFrame with some 'no_parent' children now assigned.
|
||||
"""
|
||||
# Validate input DataFrame is not empty
|
||||
if df.empty:
|
||||
logging.info("Warning: Input DataFrame is empty")
|
||||
return df.copy()
|
||||
|
||||
# Validate required columns exist
|
||||
required_cols = [
|
||||
"assigned_parent_rank",
|
||||
"parent",
|
||||
"parent_rank",
|
||||
"fixed_effective_date",
|
||||
contract_title_col,
|
||||
grouper_col,
|
||||
]
|
||||
|
||||
missing_cols = [col for col in required_cols if col not in df.columns]
|
||||
if missing_cols:
|
||||
logging.info(f"Available columns: {list(df.columns)}")
|
||||
raise KeyError(f"Missing required columns: {missing_cols}")
|
||||
|
||||
logging.info(f"Processing {len(df)} rows")
|
||||
logging.info(f"Using columns - grouper: {grouper_col}, title: {contract_title_col}")
|
||||
|
||||
# Create a copy and ensure we're working with the right data types
|
||||
df_work = df.copy()
|
||||
|
||||
# Convert date column with explicit error handling
|
||||
try:
|
||||
df_work["fixed_effective_date"] = pd.to_datetime(
|
||||
df_work["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
except Exception as e:
|
||||
logging.info(f"Error converting dates: {e}")
|
||||
return df.copy()
|
||||
|
||||
# Filter parents and children with no parent
|
||||
parents = df_work[df_work["parent"] == True].copy()
|
||||
children_no_parent = df_work[df_work["assigned_parent_rank"] == "no_parent"].copy()
|
||||
|
||||
logging.info(
|
||||
f"Found {len(parents)} parents and {len(children_no_parent)} children with no_parent"
|
||||
)
|
||||
|
||||
if len(parents) == 0 or len(children_no_parent) == 0:
|
||||
logging.info("No work to do - either no parents or no unassigned children")
|
||||
return df.copy()
|
||||
|
||||
assignments_made = 0
|
||||
|
||||
# Process each child that needs a parent
|
||||
for idx, child in children_no_parent.iterrows():
|
||||
# Find potential parents in same group
|
||||
try:
|
||||
possible_parents = parents[
|
||||
parents[grouper_col] == child[grouper_col]
|
||||
].copy()
|
||||
except KeyError as e:
|
||||
logging.info(f"Error accessing grouper column '{grouper_col}': {e}")
|
||||
continue
|
||||
|
||||
if len(possible_parents) == 0:
|
||||
continue
|
||||
|
||||
child_date = child["fixed_effective_date"]
|
||||
if pd.isna(child_date):
|
||||
continue # skip if child's date missing
|
||||
|
||||
child_title = child[contract_title_col]
|
||||
if not isinstance(child_title, str):
|
||||
continue # skip if child title is not a string
|
||||
|
||||
child_title_lower = child_title.lower()
|
||||
|
||||
# Find parents whose title is substring of child's title (case-insensitive)
|
||||
def title_match(parent_title):
|
||||
if not isinstance(parent_title, str):
|
||||
return False
|
||||
return parent_title.lower() in child_title_lower
|
||||
|
||||
title_matches = possible_parents[
|
||||
possible_parents[contract_title_col].apply(title_match)
|
||||
]
|
||||
|
||||
if len(title_matches) == 0:
|
||||
continue
|
||||
|
||||
# Filter by effective date: parent date <= child date
|
||||
eligible = title_matches[title_matches["fixed_effective_date"] <= child_date]
|
||||
|
||||
if not eligible.empty:
|
||||
# Choose parent with latest parent_rank
|
||||
chosen_parent_rank = eligible.sort_values(
|
||||
"parent_rank", ascending=False
|
||||
).iloc[0]["parent_rank"]
|
||||
df_work.at[idx, "assigned_parent_rank"] = chosen_parent_rank
|
||||
assignments_made += 1
|
||||
|
||||
logging.info(f"Made {assignments_made} new parent assignments")
|
||||
return df_work
|
||||
|
||||
|
||||
def assign_child_ranks(df, grouper_col):
|
||||
"""
|
||||
Assigns sequential child ranks to non-parent contracts.
|
||||
|
||||
Ranking logic:
|
||||
- Parent rank assigned earlier is used as the base.
|
||||
- Orphans (assigned_parent_rank == 'no_parent') start with base rank "0".
|
||||
- Children are ordered by 'fixed_effective_date' (null dates last).
|
||||
- No deduplication: every child gets a unique sequential rank.
|
||||
- The child rank is always a string (e.g., "1.1", "1.2", "0.1").
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
DataFrame with added 'child_rank' and 'combined_rank' columns (as string).
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
# Separate parents and children
|
||||
parents = df[df["parent"]].copy()
|
||||
children = df[~df["parent"]].copy()
|
||||
|
||||
# Assign children to parents (and orphans)
|
||||
for (grp_val, parent_rank), grp_df in children.groupby(
|
||||
[grouper_col, "assigned_parent_rank"]
|
||||
):
|
||||
if parent_rank == "no_parent": # orphan group
|
||||
base_rank = "0"
|
||||
else:
|
||||
base_rank = str(parent_rank)
|
||||
|
||||
# Sort by date (null dates last)
|
||||
grp_df = grp_df.assign(
|
||||
sort_date=grp_df["fixed_effective_date"].fillna(pd.Timestamp.max)
|
||||
).sort_values("sort_date")
|
||||
|
||||
# Sequential rank without deduplication
|
||||
for i, idx in enumerate(grp_df.index, start=1):
|
||||
rank_val = f"{base_rank}.{i}"
|
||||
children.loc[idx, "child_rank"] = rank_val
|
||||
|
||||
logging.info(children.columns)
|
||||
# Combine back
|
||||
result = pd.concat([parents, children]).sort_index()
|
||||
# Make sure child_rank is string
|
||||
result["child_rank"] = result["child_rank"].astype(str)
|
||||
|
||||
# If parent rank is "no_parent", fall back to child rank
|
||||
def pick_rank(row):
|
||||
if row.get("assigned_parent_rank") == "no_parent":
|
||||
return row["child_rank"]
|
||||
return (
|
||||
str(row["parent_rank"])
|
||||
if pd.notna(row["parent_rank"])
|
||||
else row["child_rank"]
|
||||
)
|
||||
|
||||
result["combined_rank"] = result.apply(pick_rank, axis=1)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def assign_orphans_based_on_unique_parent_and_payer(
|
||||
df,
|
||||
grouper_col="TIN",
|
||||
payer_col="payer_name_cleaned",
|
||||
assigned_parent_rank_col="assigned_parent_rank",
|
||||
):
|
||||
"""
|
||||
Assigns orphans to their unique parent when:
|
||||
- There's only one unique payer in the TIN group.
|
||||
- There's exactly one unique parent in the TIN group (based on 'parent_rank').
|
||||
- The child has 'no_parent' in 'assigned_parent_rank'.
|
||||
- There is at least one parent in the TIN group.
|
||||
- The parent’s effective date is older than or equal to the child’s effective date, or the child’s effective date is missing.
|
||||
- If there are multiple unique parents and exactly one unique payer, assign to the latest eligible parent (i.e., the lowest `parent_rank`).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
The input DataFrame.
|
||||
grouper_col : str
|
||||
Column name for the grouping (e.g., 'TIN').
|
||||
payer_col : str
|
||||
Column name for the PAYER_NAME (e.g., 'payer_name_cleaned').
|
||||
assigned_parent_rank_col : str
|
||||
Column for the assigned parent rank (e.g., 'assigned_parent_rank').
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Updated DataFrame with some orphans assigned to their unique parent.
|
||||
"""
|
||||
# Ensure 'assigned_parent_rank' is initialized
|
||||
if assigned_parent_rank_col not in df.columns:
|
||||
raise KeyError(f"'{assigned_parent_rank_col}' column is missing.")
|
||||
|
||||
# Step 1: Filter TIN groups where there are orphans ('no_parent') and at least one parent exists
|
||||
df_work = df.copy()
|
||||
|
||||
# Filter TIN groups where:
|
||||
# - There are orphans (i.e., 'no_parent' in 'assigned_parent_rank')
|
||||
# - At least one parent exists in the group
|
||||
tin_present = df_work.groupby(grouper_col).filter(
|
||||
lambda group: (group[assigned_parent_rank_col] == "no_parent").any()
|
||||
and (group["parent"] == True).any()
|
||||
)
|
||||
|
||||
if tin_present.empty:
|
||||
logging.info("No TIN groups meet the filtering criteria.")
|
||||
return df_work
|
||||
|
||||
# Step 2: Identify orphans (children with 'no_parent')
|
||||
orphan_groups = tin_present[tin_present[assigned_parent_rank_col] == "no_parent"]
|
||||
if orphan_groups.empty:
|
||||
logging.info("No orphans to assign.")
|
||||
return df_work
|
||||
|
||||
# Step 3: Iterate over TIN groups to assign orphans
|
||||
assignments_made = 0
|
||||
for tin_group, group_df in orphan_groups.groupby(grouper_col):
|
||||
# Find the parents in this TIN group
|
||||
possible_parents = tin_present[
|
||||
(tin_present[grouper_col] == tin_group) & (tin_present["parent"] == True)
|
||||
]
|
||||
|
||||
# Step 4: Check for unique parent ranks (i.e., exactly one unique parent in the TIN group)
|
||||
unique_parent_ranks = possible_parents["parent_rank"].nunique()
|
||||
|
||||
# Step 5: Check for unique PAYER_NAMEs (including NaN values)
|
||||
unique_payer_names = possible_parents[payer_col].nunique(dropna=False)
|
||||
|
||||
# Debug: Check unique parent ranks and PAYER_NAMEs in the group
|
||||
# logging.info(f"Processing TIN group {tin_group}:")
|
||||
# logging.info(f"Unique parent ranks: {unique_parent_ranks}")
|
||||
# logging.info(f"Unique PAYER_NAMEs (including NaN): {unique_payer_names}")
|
||||
|
||||
# Step 6: If there is more than one unique payer, do not assign any parent
|
||||
if unique_payer_names > 1:
|
||||
# logging.info(f"More than one unique payer in TIN group {tin_group}. Skipping assignment.")
|
||||
continue
|
||||
|
||||
# Step 7: If there's exactly one unique parent and one unique payer, assign to that parent
|
||||
if unique_parent_ranks == 1:
|
||||
parent = possible_parents.iloc[0]
|
||||
parent_rank = parent["parent_rank"]
|
||||
parent_effective_date = pd.to_datetime(
|
||||
parent["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
|
||||
# Step 8: Assign the single parent to the orphans in this TIN group
|
||||
for _, orphan in group_df.iterrows():
|
||||
child_effective_date = pd.to_datetime(
|
||||
orphan["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
|
||||
# Check if the child's effective date is older than the parent's
|
||||
if (
|
||||
pd.isna(child_effective_date)
|
||||
or parent_effective_date <= child_effective_date
|
||||
):
|
||||
df_work.at[orphan.name, assigned_parent_rank_col] = parent_rank
|
||||
assignments_made += 1
|
||||
# logging.info(f"Assigned parent rank {parent_rank} to orphan {orphan.name}")
|
||||
|
||||
# Step 9: If there's exactly one unique payer and more than one unique parent, assign to the latest parent (lowest rank)
|
||||
elif unique_parent_ranks > 1:
|
||||
# logging.info(f"Multiple unique parents in TIN group {tin_group}. Assigning to the latest parent (lowest rank).")
|
||||
|
||||
# Sort parents by parent_rank (which is based on effective date)
|
||||
sorted_parents = possible_parents.sort_values(
|
||||
"parent_rank", ascending=False
|
||||
) # Lower rank is the latest parent
|
||||
|
||||
# Assign the orphan to the latest eligible parent
|
||||
for _, orphan in group_df.iterrows():
|
||||
child_effective_date = pd.to_datetime(
|
||||
orphan["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
|
||||
# Find the first parent who meets the effective date condition
|
||||
for _, parent in sorted_parents.iterrows():
|
||||
parent_effective_date = pd.to_datetime(
|
||||
parent["fixed_effective_date"], errors="coerce"
|
||||
)
|
||||
|
||||
if (
|
||||
pd.isna(child_effective_date)
|
||||
or parent_effective_date <= child_effective_date
|
||||
):
|
||||
df_work.at[orphan.name, assigned_parent_rank_col] = parent[
|
||||
"parent_rank"
|
||||
]
|
||||
assignments_made += 1
|
||||
# logging.info(f"Assigned parent rank {parent['parent_rank']} to orphan {orphan.name}")
|
||||
break
|
||||
|
||||
return df_work
|
||||
|
||||
|
||||
def assign_parent_names(df, rank_col="combined_rank", name_col="CONTRACT_NAME"):
|
||||
"""
|
||||
Adds a 'parent_name' column mapping each child to its parent's name.
|
||||
- Parents are detected as rows whose rank has only one number part (e.g. '1', '1.0', '001')
|
||||
- Children starting with '0.' get no parent (None)
|
||||
- Works with different numeric formats
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df : pd.DataFrame
|
||||
Input dataframe
|
||||
rank_col : str
|
||||
Column containing rank (e.g. 'combined_rank')
|
||||
name_col : str
|
||||
Column containing name (e.g. 'CONTRACT_NAME')
|
||||
|
||||
Returns
|
||||
-------
|
||||
pd.DataFrame
|
||||
Dataframe with 'parent_name' column added
|
||||
"""
|
||||
df = df.copy()
|
||||
df[rank_col] = df[rank_col].astype(str).str.strip()
|
||||
|
||||
# Detect parents: only one part when split by dot, or dot followed by zeros
|
||||
parents_df = df[df[rank_col].str.match(r"^\d+(\.0+)?$", na=False)].copy()
|
||||
|
||||
# Normalize parent rank to plain integer string (e.g. '1.0' -> '1')
|
||||
parents_df["parent_key"] = (
|
||||
parents_df[rank_col].str.split(".").str[0].astype(int).astype(str)
|
||||
)
|
||||
|
||||
# Create mapping parent_key -> parent name
|
||||
parent_map = parents_df.set_index("parent_key")[name_col].to_dict()
|
||||
|
||||
# Function to get parent name
|
||||
def get_parent_name(rank):
|
||||
rank = str(rank).strip()
|
||||
if rank.startswith("0.") or rank in parent_map:
|
||||
return None
|
||||
parent_key = rank.split(".")[0]
|
||||
return parent_map.get(parent_key)
|
||||
|
||||
df["parent_name"] = df[rank_col].apply(get_parent_name)
|
||||
return df
|
||||
|
||||
|
||||
def extract_ordinal(row):
|
||||
num_dict = io_utils.convert_json_to_dict("constants/mappings/numeric_mappigs.json")
|
||||
word_to_num = num_dict.get("ordinal_word_to_number", {})
|
||||
cardinal_to_num = num_dict.get("cardinal_word_to_number", {})
|
||||
ordinal_pattern = num_dict.get("ordinal_regex_pattern", "")
|
||||
|
||||
# Combine all text-based mappings
|
||||
all_words_to_num = {**word_to_num, **cardinal_to_num}
|
||||
text = str(row["contract_title_cleaned"]).lower()
|
||||
|
||||
# Match ordinal or cardinal words
|
||||
for word, num in all_words_to_num.items():
|
||||
if word in text:
|
||||
return num
|
||||
|
||||
# Match numeric ordinals (e.g., 1st, 2nd, 3rd, 25th)
|
||||
num_match = re.search(ordinal_pattern, text)
|
||||
if num_match:
|
||||
value = re.sub(r"(st|nd|rd|th)$", "", num_match.group(0))
|
||||
if value.isdigit():
|
||||
return int(value)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def assign_parent_name_by_rank(df):
|
||||
"""
|
||||
Assigns the parent's 'FILE_NAME' to child rows based on TIN and a
|
||||
modified combined_rank.
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): DataFrame with 'TIN', 'parent_child_flag',
|
||||
'combined_rank', and 'FILE_NAME' columns.
|
||||
'parent_child_flag' should be 'parent' or 'child'.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: DataFrame with an added 'parent_name' column.
|
||||
"""
|
||||
|
||||
df = df.copy()
|
||||
df["parent_name"] = None # Initialize parent_name column
|
||||
|
||||
# Separate parents and children based on parent_child_flag
|
||||
parents_df = df[df["parent_child_flag"] == "Parent"].copy()
|
||||
children_df = df[df["parent_child_flag"] == "Child"].copy()
|
||||
|
||||
# Create a mapping for parents: (TIN, combined_rank) -> FILE_NAME
|
||||
# This will be used to look up the parent's name efficiently
|
||||
parent_lookup = parents_df.set_index(["TIN", "combined_rank"])[
|
||||
"FILE_NAME"
|
||||
].to_dict()
|
||||
|
||||
# Iterate through each child to find its parent
|
||||
for index, child_row in children_df.iterrows():
|
||||
child_tin = child_row["TIN"]
|
||||
child_combined_rank = str(
|
||||
child_row["combined_rank"]
|
||||
) # Ensure string for splitting
|
||||
|
||||
# 2. Remove the last decimal to get the conceptual parent rank
|
||||
if "." in child_combined_rank:
|
||||
parent_conceptual_rank = ".".join(child_combined_rank.split(".")[:-1])
|
||||
else:
|
||||
# If a child rank doesn't have a decimal (e.g., '1'), it might not have a parent of this type
|
||||
parent_conceptual_rank = (
|
||||
None # Or handle as appropriate, maybe it's a top-level child
|
||||
)
|
||||
|
||||
if parent_conceptual_rank:
|
||||
# 3. Try to find a parent with the matching TIN and conceptual rank
|
||||
parent_contract_name = parent_lookup.get(
|
||||
(child_tin, parent_conceptual_rank)
|
||||
)
|
||||
|
||||
# 3. Add parent name to the row where the corresponding child is in
|
||||
if parent_contract_name:
|
||||
df.at[index, "parent_name"] = parent_contract_name
|
||||
|
||||
return df
|
||||
|
||||
|
||||
# # molina
|
||||
def parent_child_mapping(cleaned_df):
|
||||
grouper_column = "TIN"
|
||||
logging.info(
|
||||
f"Unique Contract Count: {cleaned_df['contract_name_cleaned'].nunique()}"
|
||||
)
|
||||
tin_present = create_df_per_tin_presence(cleaned_df, True)
|
||||
logging.info(
|
||||
f"Contract Count after dropping contracts w/o TINs: {tin_present.shape}"
|
||||
)
|
||||
tin_present = filter_on_exclude_keywords(tin_present, True)
|
||||
tin_present = filter_on_missing_title(tin_present)
|
||||
logging.info(
|
||||
f"Contract Count after filtering on exclude keywords and missing title: {tin_present.shape}"
|
||||
)
|
||||
# tin_present = flag_duplicates_within_group(tin_present, grouper_col=grouper_column)
|
||||
tin_present = rank_parents(tin_present, grouper_col=grouper_column)
|
||||
tin_present = flag_multi_parent_with_early_dates(
|
||||
tin_present, grouper_col=grouper_column
|
||||
)
|
||||
tin_present = assign_parents_with_state_and_payer(
|
||||
tin_present, grouper_col=grouper_column
|
||||
)
|
||||
tin_present = assign_no_parent_by_title(tin_present, grouper_col=grouper_column)
|
||||
tin_present = assign_orphans_based_on_unique_parent_and_payer(
|
||||
tin_present, grouper_col="TIN", payer_col="payer_name_cleaned"
|
||||
)
|
||||
tin_present = assign_child_ranks(tin_present, grouper_col=grouper_column)
|
||||
tin_present["numbering"] = tin_present.apply(extract_ordinal, axis=1)
|
||||
|
||||
tin_present = tin_present[
|
||||
[
|
||||
"FILE_NAME",
|
||||
"CONTRACT_TITLE",
|
||||
"PROV_GROUP_NAME_FULL",
|
||||
"PAYER_STATE",
|
||||
"contract_name_cleaned",
|
||||
"contract_title_cleaned",
|
||||
"TIN",
|
||||
"fixed_effective_date",
|
||||
"AARETE_DERIVED_TERMINATION_DT",
|
||||
"payer_name_cleaned",
|
||||
"PROV_GROUP_NAME_FULL_cleaned",
|
||||
"consolidated_lob",
|
||||
"consolidated_service_term",
|
||||
"parent",
|
||||
"combined_rank",
|
||||
"numbering",
|
||||
]
|
||||
]
|
||||
|
||||
tin_present["parent_child_flag"] = tin_present["parent"].map(
|
||||
{True: "Parent", False: "Child"}
|
||||
)
|
||||
tin_present.loc[
|
||||
(tin_present["parent"] == False)
|
||||
& (tin_present["combined_rank"].astype(str).str.count(r"\.") == 1),
|
||||
"parent_child_flag",
|
||||
] = "Orphan"
|
||||
|
||||
tin_present = clean_termination_date(tin_present, "AARETE_DERIVED_TERMINATION_DT")
|
||||
|
||||
cols_order = [
|
||||
"TIN",
|
||||
"PROV_GROUP_NAME_FULL_cleaned",
|
||||
"consolidated_lob",
|
||||
"fixed_effective_date",
|
||||
"parent_child_flag",
|
||||
"combined_rank",
|
||||
]
|
||||
|
||||
tin_present["naming_col"] = (
|
||||
tin_present[cols_order]
|
||||
.replace("", "xx")
|
||||
.fillna("xx")
|
||||
.astype(str)
|
||||
.agg("___".join, axis=1)
|
||||
)
|
||||
|
||||
tin_present["FILE_NAME"] = tin_present["FILE_NAME"].str.replace("Filename: ", "")
|
||||
|
||||
logging.info(f"tin present: {tin_present.columns}")
|
||||
|
||||
tin_present = assign_parent_name_by_rank(tin_present)
|
||||
|
||||
tin_present = tin_present[
|
||||
[
|
||||
"FILE_NAME",
|
||||
"CONTRACT_TITLE",
|
||||
"PROV_GROUP_NAME_FULL",
|
||||
"PAYER_STATE",
|
||||
"contract_name_cleaned",
|
||||
"contract_title_cleaned",
|
||||
"TIN",
|
||||
"fixed_effective_date",
|
||||
"fixed_termination_date",
|
||||
"payer_name_cleaned",
|
||||
"PROV_GROUP_NAME_FULL_cleaned",
|
||||
"consolidated_lob",
|
||||
"consolidated_service_term",
|
||||
"parent",
|
||||
"combined_rank",
|
||||
"numbering",
|
||||
"parent_child_flag",
|
||||
"parent_name",
|
||||
"naming_col",
|
||||
]
|
||||
]
|
||||
|
||||
return tin_present
|
||||
File diff suppressed because it is too large
Load Diff
+348
-60
@@ -1,23 +1,32 @@
|
||||
import warnings
|
||||
from __future__ import annotations
|
||||
|
||||
warnings.filterwarnings("ignore")
|
||||
import pandas as pd
|
||||
import nltk
|
||||
import ssl
|
||||
import re
|
||||
import string
|
||||
import src.config as config
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
import logging
|
||||
from src.utils import io_utils
|
||||
from collections import defaultdict
|
||||
from typing import Optional, List, Tuple, TYPE_CHECKING
|
||||
|
||||
stop_words_dict = io_utils.convert_json_to_dict(
|
||||
"src/constants/mappings/stop_words_for_pc_cleanup.json"
|
||||
if TYPE_CHECKING:
|
||||
from src.parent_child.column_mapper import ColumnMapper
|
||||
|
||||
# Build path relative to this module's location
|
||||
_MODULE_DIR = Path(__file__).resolve().parent
|
||||
_STOP_WORDS_PATH = (
|
||||
_MODULE_DIR.parent / "constants" / "mappings" / "stop_words_for_pc_cleanup.json"
|
||||
)
|
||||
stop_words_dict = io_utils.convert_json_to_dict(str(_STOP_WORDS_PATH))
|
||||
stop_words = stop_words_dict["stop_words"]
|
||||
|
||||
|
||||
def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_NAME
|
||||
def clean_filename(all_fields_df, contact_name_col="FILE_NAME"):
|
||||
"""Fixing FILE_NAME by removing prefixes and extensions."""
|
||||
all_fields_df["contract_name_cleaned"] = (
|
||||
all_fields_df[contact_name_col]
|
||||
.str.replace(r"Filename: ?", "", regex=True)
|
||||
@@ -29,9 +38,6 @@ def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_
|
||||
return all_fields_df
|
||||
|
||||
|
||||
# ... rest of the file remains the same
|
||||
|
||||
|
||||
def create_one_to_one_df(df):
|
||||
"""
|
||||
Create a DataFrame with AC Output columns.
|
||||
@@ -41,7 +47,7 @@ def create_one_to_one_df(df):
|
||||
|
||||
|
||||
def clean_contract_title(one_to_one_df, contact_name_col="CONTRACT_TITLE"):
|
||||
# Cleaning contract title
|
||||
"""Cleaning contract title by lowercasing, removing 'inc', and stop words."""
|
||||
one_to_one_df["contract_title_cleaned"] = (
|
||||
one_to_one_df[contact_name_col]
|
||||
.astype(str)
|
||||
@@ -81,7 +87,8 @@ def clean_termination_date(
|
||||
# Try to convert anything else
|
||||
try:
|
||||
return pd.to_datetime(x).strftime("%m/%d/%Y")
|
||||
except:
|
||||
except Exception as e:
|
||||
logging.debug(f"Could not parse date value '{x}': {e}")
|
||||
return x
|
||||
|
||||
one_to_one_df["fixed_termination_date"] = one_to_one_df[termination_date_col].apply(
|
||||
@@ -127,50 +134,287 @@ def clean_payer_name(text, hit_words):
|
||||
# Remove any extra spaces that might have been created
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
# standadrizing for molina healthcare
|
||||
# if text == 'molina':
|
||||
# return 'molina healthcare'
|
||||
return text
|
||||
|
||||
|
||||
def clean_PROV_GROUP_NAME_FULL(
|
||||
name, cutoff_words=["inc", "ltd", "llc", "corp", "corporation", "incorporated"]
|
||||
def create_group_column(
|
||||
df,
|
||||
provider_col="PROV_GROUP_NAME_FULL",
|
||||
group_col="PROV_GROUP_NAME_FULL_cleaned",
|
||||
dba_col="DBA_Name",
|
||||
):
|
||||
"""
|
||||
Cleans a string with the following steps:
|
||||
1. Lowercases everything
|
||||
2. Removes everything after the first occurrence of any word in cutoff_words
|
||||
3. Replaces all non-alphanumeric characters with a space
|
||||
4. Removes extra spaces
|
||||
Creates a Group column from the provider column by:
|
||||
1. Removing non-alphanumeric characters (except spaces)
|
||||
2. Converting to lowercase
|
||||
3. Removing anything after 'llc', 'inc', 'dba', or 'd/b/a' (only as complete words)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name : str
|
||||
Input string
|
||||
cutoff_words : list[str], optional
|
||||
List of keywords to cut off text after first occurrence (e.g. ['inc', 'llc', 'd/b/a'])
|
||||
Also creates a DBA_Name column that extracts text after 'dba' or 'd/b/a'
|
||||
|
||||
Returns
|
||||
-------
|
||||
str or None
|
||||
Parameters:
|
||||
-----------
|
||||
df : pandas.DataFrame
|
||||
The dataframe containing the provider data
|
||||
provider_col : str
|
||||
Name of the provider column (default: 'provider')
|
||||
group_col : str
|
||||
Name of the new group column to create (default: 'provider Group')
|
||||
dba_col : str
|
||||
Name of the DBA name column to create (default: 'DBA_Name')
|
||||
|
||||
Returns:
|
||||
--------
|
||||
pandas.DataFrame
|
||||
The dataframe with the new Group and DBA_Name columns added
|
||||
"""
|
||||
if pd.isna(name):
|
||||
return None
|
||||
|
||||
text = str(name).lower().strip()
|
||||
def process_provider(provider):
|
||||
if pd.isna(provider):
|
||||
return "", ""
|
||||
|
||||
# Step 2: cutoff after first keyword match
|
||||
if cutoff_words:
|
||||
pattern = r"\b(" + "|".join(map(re.escape, cutoff_words)) + r")\b.*"
|
||||
text = re.sub(pattern, "", text)
|
||||
# Step 1: Remove non-alphanumeric characters except spaces
|
||||
cleaned = re.sub(r"[^a-zA-Z0-9\s]", "", str(provider))
|
||||
|
||||
# Step 3: replace non-alphanumeric with space
|
||||
text = re.sub(r"[^a-z0-9]", " ", text)
|
||||
# Step 2: Convert to lowercase
|
||||
cleaned = cleaned.lower()
|
||||
|
||||
# Step 4: collapse spaces
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
# Step 3: Extract DBA name if it exists (as a complete word)
|
||||
dba_name = ""
|
||||
dba_match = re.search(r"\bdba\b", cleaned)
|
||||
|
||||
return text
|
||||
if dba_match:
|
||||
dba_pos = dba_match.start()
|
||||
# Extract everything after 'dba'
|
||||
after_dba = cleaned[dba_pos + 3 :].strip()
|
||||
|
||||
# Remove anything after 'llc' or 'inc' in the DBA portion (as complete words)
|
||||
for pattern in [
|
||||
"inc",
|
||||
"ltd",
|
||||
"corp",
|
||||
"corporation",
|
||||
"incorporated",
|
||||
"lp",
|
||||
"llc",
|
||||
"inc",
|
||||
"llp",
|
||||
"pty",
|
||||
]:
|
||||
match = re.search(r"\b" + pattern + r"\b", after_dba)
|
||||
if match:
|
||||
after_dba = after_dba[: match.start()]
|
||||
break
|
||||
|
||||
# Clean up extra whitespace
|
||||
dba_name = " ".join(after_dba.split()).strip()
|
||||
|
||||
# Step 4: Remove anything after llc, inc, or dba for the group name
|
||||
# Find the earliest occurrence of these terms as complete words
|
||||
patterns = [
|
||||
"dba",
|
||||
"inc",
|
||||
"ltd",
|
||||
"corp",
|
||||
"corporation",
|
||||
"incorporated",
|
||||
"lp",
|
||||
"llc",
|
||||
"inc",
|
||||
"llp",
|
||||
"pty",
|
||||
]
|
||||
positions = []
|
||||
|
||||
for pattern in patterns:
|
||||
# Use word boundaries to match complete words only
|
||||
match = re.search(r"\b" + pattern + r"\b", cleaned)
|
||||
if match:
|
||||
positions.append(match.start())
|
||||
|
||||
# Handle multi-word patterns separately
|
||||
multi_word_patterns = ["advisory services"]
|
||||
for pattern in multi_word_patterns:
|
||||
match = re.search(r"\b" + pattern + r"\b", cleaned)
|
||||
if match:
|
||||
positions.append(match.start())
|
||||
|
||||
if positions:
|
||||
# Cut at the earliest position
|
||||
min_pos = min(positions)
|
||||
cleaned = cleaned[:min_pos]
|
||||
|
||||
# Remove the word 'and' (as a complete word)
|
||||
cleaned = re.sub(r"\band\b", "", cleaned)
|
||||
cleaned = re.sub(r"\bassoc\b", "associates", cleaned)
|
||||
cleaned = re.sub(r"\bcenters\b", "center", cleaned)
|
||||
cleaned = re.sub(r"\bsolutions\b", "solution", cleaned)
|
||||
cleaned = re.sub(r"\bsystems\b", "system", cleaned)
|
||||
cleaned = re.sub(r"\bplans\b", "plan", cleaned)
|
||||
cleaned = re.sub(r"\bservices\b", "service", cleaned)
|
||||
cleaned = re.sub(r"\bcasemanagement\b", "case management", cleaned)
|
||||
|
||||
# Remove extra whitespace
|
||||
cleaned = " ".join(cleaned.split())
|
||||
|
||||
return cleaned.strip(), dba_name
|
||||
|
||||
# Apply the function to create both columns
|
||||
df[[group_col, dba_col]] = df[provider_col].apply(
|
||||
lambda x: pd.Series(process_provider(x))
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def update_group(df, group_col="PROV_GROUP_NAME_FULL_cleaned", dba_col="DBA_Name"):
|
||||
"""
|
||||
Updates provider Group values when a DBA_Name within the group matches any provider Group value.
|
||||
This consolidates providers under their DBA name when the DBA operates as a separate entity.
|
||||
|
||||
Logic:
|
||||
- Group by provider Group
|
||||
- Within each group, check if any DBA_Name matches ANY provider Group value in the entire dataframe
|
||||
- If a match is found, update ALL rows in that group to use the matched DBA_Name
|
||||
- Stop at the first match found within each group
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df : pandas.DataFrame
|
||||
The dataframe containing provider group and DBA name data
|
||||
group_col : str
|
||||
Name of the provider group column (default: 'provider Group')
|
||||
dba_col : str
|
||||
Name of the DBA name column (default: 'DBA_Name')
|
||||
|
||||
Returns:
|
||||
--------
|
||||
pandas.DataFrame
|
||||
The dataframe with updated provider Group values
|
||||
"""
|
||||
|
||||
# Make a copy to avoid modifying the original
|
||||
df = df.copy()
|
||||
|
||||
# Get all unique provider Group values (excluding empty strings and NaN)
|
||||
all_groups = set(df[group_col].dropna().unique())
|
||||
all_groups.discard("") # Remove empty string if present
|
||||
|
||||
# Track updates for reporting
|
||||
updates = []
|
||||
|
||||
# Group by provider Group
|
||||
for group_name, group_df in df.groupby(group_col):
|
||||
# Skip if group_name is NaN or empty
|
||||
if pd.isna(group_name) or group_name == "":
|
||||
continue
|
||||
|
||||
# Check if any DBA_Name in this group matches a provider Group value
|
||||
matched_dba = None
|
||||
|
||||
for idx, row in group_df.iterrows():
|
||||
dba_name = row.get(dba_col, "")
|
||||
|
||||
# Skip empty DBA names or if DBA equals current group
|
||||
if not dba_name or dba_name == group_name:
|
||||
continue
|
||||
|
||||
# Check if this DBA name exists as a provider Group elsewhere
|
||||
if dba_name in all_groups:
|
||||
matched_dba = dba_name
|
||||
break # Stop at first match
|
||||
|
||||
# If a match was found, update ALL rows in this group
|
||||
if matched_dba:
|
||||
for idx in group_df.index:
|
||||
df.at[idx, group_col] = matched_dba
|
||||
updates.append(
|
||||
{
|
||||
"index": idx,
|
||||
"original_group": group_name,
|
||||
"updated_group": matched_dba,
|
||||
"dba_name": matched_dba,
|
||||
}
|
||||
)
|
||||
|
||||
# Print summary of updates
|
||||
if updates:
|
||||
logging.info(
|
||||
f"Updated {len(updates)} rows across {len(set(u['original_group'] for u in updates))} provider Groups:"
|
||||
)
|
||||
# Group updates by original group for clearer reporting
|
||||
updates_by_group = defaultdict(list)
|
||||
for update in updates:
|
||||
updates_by_group[update["original_group"]].append(update)
|
||||
|
||||
for orig_group, group_updates in updates_by_group.items():
|
||||
logging.info(
|
||||
f" Group '{orig_group}' → '{group_updates[0]['updated_group']}' ({len(group_updates)} rows updated)"
|
||||
)
|
||||
else:
|
||||
logging.info("No provider Group updates needed.")
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def standardize_provider_groups(df, column_name="PROV_GROUP_NAME_FULL_cleaned"):
|
||||
"""
|
||||
Standardizes provider group names by finding shorter base names and
|
||||
updating longer variations to match the shorter ones.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df : pandas.DataFrame
|
||||
The dataframe containing provider group data
|
||||
column_name : str
|
||||
Name of the column containing provider groups (default: 'provider Group')
|
||||
|
||||
Returns:
|
||||
--------
|
||||
pandas.DataFrame
|
||||
DataFrame with standardized provider group names
|
||||
"""
|
||||
# Create a copy to avoid modifying the original
|
||||
df = df.copy()
|
||||
|
||||
# Get unique provider groups and sort by length (shortest first)
|
||||
unique_providers = df[column_name].dropna().unique()
|
||||
sorted_providers = sorted(unique_providers, key=lambda x: len(x.split()))
|
||||
|
||||
# Create a mapping dictionary
|
||||
mapping = {}
|
||||
|
||||
for provider in sorted_providers:
|
||||
words = provider.lower().split()
|
||||
|
||||
# Only proceed if there are 2 or more words
|
||||
if len(words) >= 2:
|
||||
# Get the first two words as potential base name
|
||||
first_two_words = " ".join(words[:2])
|
||||
|
||||
# Check if this provider should be mapped to a shorter version
|
||||
if provider not in mapping:
|
||||
# Check all other providers to see if current one is a longer version
|
||||
for base_provider in sorted_providers:
|
||||
base_words = base_provider.lower().split()
|
||||
|
||||
# Skip if same provider or base doesn't have at least 2 words
|
||||
if base_provider == provider or len(base_words) < 2:
|
||||
continue
|
||||
|
||||
# Check if first two words of base match first two words of current
|
||||
base_first_two = " ".join(base_words[:2])
|
||||
|
||||
if base_first_two == first_two_words:
|
||||
# Current provider starts with same two words as base
|
||||
# Map the longer one to the shorter one
|
||||
if len(base_words) < len(words):
|
||||
mapping[provider] = base_provider
|
||||
break
|
||||
|
||||
# Apply the mapping
|
||||
df[column_name] = df[column_name].replace(mapping)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def extract_tin_like(value):
|
||||
@@ -342,6 +586,8 @@ def drop_exact_matches(
|
||||
"- hdo -",
|
||||
"term - term",
|
||||
"- ros -",
|
||||
"- cif -",
|
||||
"- darf -",
|
||||
]
|
||||
|
||||
if exception_words is None:
|
||||
@@ -470,32 +716,74 @@ def drop_letter(
|
||||
return df
|
||||
|
||||
|
||||
def parent_child_preprocessing(df_read):
|
||||
def parent_child_preprocessing(
|
||||
df_read: pd.DataFrame, col_mapper: Optional[ColumnMapper] = None
|
||||
) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
Main preprocessing function that prepares data for parent-child mapping.
|
||||
|
||||
Args:
|
||||
df_read: Raw input DataFrame
|
||||
col_mapper: ColumnMapper instance for automatic column detection
|
||||
|
||||
Returns:
|
||||
tuple: (cleaned_df, original_df) where cleaned_df is preprocessed
|
||||
and original_df is the original with effective date cleaned
|
||||
"""
|
||||
# Use column mapper to rename columns to standardized names
|
||||
if col_mapper:
|
||||
df_read = col_mapper.rename_dataframe(df_read)
|
||||
effective_date_col = "EFFECTIVE_DATE"
|
||||
termination_date_col = "TERMINATION_DATE"
|
||||
payer_name_col = "PAYER_NAME"
|
||||
prov_group_col = "PROV_GROUP_NAME_FULL"
|
||||
else:
|
||||
effective_date_col = "AARETE_DERIVED_EFFECTIVE_DT"
|
||||
termination_date_col = "AARETE_DERIVED_TERMINATION_DT"
|
||||
payer_name_col = "PAYER_NAME"
|
||||
prov_group_col = "PROV_GROUP_NAME_FULL"
|
||||
|
||||
all_fields_df = clean_filename(df_read)
|
||||
one_to_one_df = clean_effective_date(all_fields_df, effective_date_col)
|
||||
|
||||
# Keep original_df for later merge
|
||||
original_df = one_to_one_df.copy()
|
||||
|
||||
one_to_one_df = create_one_to_one_df(all_fields_df)
|
||||
one_to_one_df = clean_contract_title(one_to_one_df)
|
||||
one_to_one_df = clean_effective_date(one_to_one_df, "AARETE_DERIVED_EFFECTIVE_DT")
|
||||
one_to_one_df["payer_name_cleaned"] = one_to_one_df["PAYER_NAME"].apply(
|
||||
one_to_one_df["payer_name_cleaned"] = one_to_one_df[payer_name_col].apply(
|
||||
lambda x: clean_payer_name(str(x), ["inc", "llc", "dba"])
|
||||
)
|
||||
one_to_one_df["PROV_GROUP_NAME_FULL_cleaned"] = one_to_one_df[
|
||||
"PROV_GROUP_NAME_FULL"
|
||||
].apply(clean_PROV_GROUP_NAME_FULL)
|
||||
one_to_one_df["TIN"] = one_to_one_df["contract_name_cleaned"].apply(
|
||||
extract_tin_like
|
||||
)
|
||||
consolidated_one_to_n_fields = consolidate_one_to_n_fields(
|
||||
"contract_name_cleaned", all_fields_df, ["LOB", "SERVICE_TERM"]
|
||||
)
|
||||
df = pd.merge(
|
||||
one_to_one_df,
|
||||
consolidated_one_to_n_fields,
|
||||
on="contract_name_cleaned",
|
||||
how="left",
|
||||
)
|
||||
|
||||
# Use create_group_column for provider name cleaning (from working code)
|
||||
one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col)
|
||||
one_to_one_df = update_group(one_to_one_df)
|
||||
one_to_one_df = standardize_provider_groups(one_to_one_df)
|
||||
|
||||
# Determine which fields to consolidate based on available columns
|
||||
fields_to_consolidate = []
|
||||
if "LOB" in all_fields_df.columns:
|
||||
fields_to_consolidate.append("LOB")
|
||||
if "SERVICE_TERM" in all_fields_df.columns:
|
||||
fields_to_consolidate.append("SERVICE_TERM")
|
||||
|
||||
if fields_to_consolidate:
|
||||
consolidated_one_to_n_fields = consolidate_one_to_n_fields(
|
||||
"contract_name_cleaned", all_fields_df, fields_to_consolidate
|
||||
)
|
||||
df = pd.merge(
|
||||
one_to_one_df,
|
||||
consolidated_one_to_n_fields,
|
||||
on="contract_name_cleaned",
|
||||
how="left",
|
||||
)
|
||||
else:
|
||||
df = one_to_one_df
|
||||
|
||||
df = drop_duplicates(df)
|
||||
df = drop_forms(df, partial_match=True)
|
||||
df = drop_exact_matches(df)
|
||||
df = drop_letter(df)
|
||||
logging.info(f"df after preprocessing: {df.columns}")
|
||||
return df
|
||||
|
||||
return df, original_df
|
||||
File diff suppressed because it is too large
Load Diff
@@ -247,7 +247,7 @@ def main(client: str = "saas", testing=False, test_params={}):
|
||||
io_utils.write_local(ERROR_RESULT_DF, "", run_timestamp, "error")
|
||||
|
||||
if config.PERFORM_PARENT_CHILD_MAPPING:
|
||||
parent_child_main.main(doczy_output_for_pc)
|
||||
parent_child_main.main(doczy_output_for_pc, client=client)
|
||||
|
||||
if not per_file_usage_df.empty and not batch_summary_df.empty:
|
||||
logging.info("Exporting usage and cost tracking data locally...")
|
||||
|
||||
@@ -276,7 +276,14 @@ def main(testing=False, test_params={}):
|
||||
f"Starting Parent-Child mapping with {len(FINAL_RESULT_DF)} records from: {doczy_output_for_pc}"
|
||||
)
|
||||
with timing_utils.timed_block("parent_child_mapping", log_level="INFO"):
|
||||
parent_child_main.main(doczy_output_for_pc)
|
||||
client = (
|
||||
config.CLIENT[0]
|
||||
if config.CLIENT
|
||||
and len(config.CLIENT) == 1
|
||||
and config.CLIENT[0] != "None"
|
||||
else None
|
||||
)
|
||||
parent_child_main.main(doczy_output_for_pc, client=client)
|
||||
|
||||
# Export usage and cost tracking data locally only if both dataframes have data
|
||||
if not per_file_usage_df.empty and not batch_summary_df.empty:
|
||||
|
||||
@@ -0,0 +1,641 @@
|
||||
"""
|
||||
Tests for the parent_child module.
|
||||
|
||||
Tests cover:
|
||||
- Grouping key creation and tier assignment
|
||||
- Grouping compatibility checks
|
||||
- Parent identification (exclude keywords)
|
||||
- Parent ranking
|
||||
- Child assignment logic
|
||||
- Utility functions (main.py)
|
||||
- QC engine utilities
|
||||
"""
|
||||
|
||||
import unittest
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
|
||||
from src.parent_child.pipeline import (
|
||||
create_grouping_key_and_tier,
|
||||
check_grouping_compatibility,
|
||||
filter_on_exclude_keywords,
|
||||
rank_parents,
|
||||
build_grouping_string,
|
||||
build_matching_columns_string,
|
||||
flag_multi_parent_with_early_dates,
|
||||
assign_child_ranks,
|
||||
extract_ordinal,
|
||||
)
|
||||
from src.parent_child.__main__ import extract_client_name
|
||||
from src.parent_child.qc import (
|
||||
to_string,
|
||||
to_datetime,
|
||||
resolve_column,
|
||||
TextProcessor,
|
||||
ConfigFactory,
|
||||
)
|
||||
|
||||
|
||||
class TestGroupingKeyCreation(unittest.TestCase):
|
||||
"""Tests for create_grouping_key_and_tier function."""
|
||||
|
||||
def test_tier_1_all_columns_present(self):
|
||||
"""Tier 1: All 3 columns (TIN, NPI, PROV_NAME) are non-null."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"PROV_GROUP_TIN": ["123456789"],
|
||||
"PROV_GROUP_NPI": ["1234567890"],
|
||||
"PROV_GROUP_NAME_FULL_cleaned": ["Test Hospital"],
|
||||
}
|
||||
)
|
||||
result = create_grouping_key_and_tier(df)
|
||||
|
||||
self.assertEqual(result["grouping_tier"].iloc[0], 1)
|
||||
self.assertIn("TIN:123456789", result["grouping_key"].iloc[0])
|
||||
self.assertIn("NPI:1234567890", result["grouping_key"].iloc[0])
|
||||
self.assertIn("NAME:Test Hospital", result["grouping_key"].iloc[0])
|
||||
|
||||
def test_tier_2_two_columns_present(self):
|
||||
"""Tier 2: Any 2 columns are non-null."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"PROV_GROUP_TIN": ["123456789"],
|
||||
"PROV_GROUP_NPI": ["1234567890"],
|
||||
"PROV_GROUP_NAME_FULL_cleaned": [""], # Empty
|
||||
}
|
||||
)
|
||||
result = create_grouping_key_and_tier(df)
|
||||
|
||||
self.assertEqual(result["grouping_tier"].iloc[0], 2)
|
||||
self.assertIn("TIN:123456789", result["grouping_key"].iloc[0])
|
||||
self.assertIn("NPI:1234567890", result["grouping_key"].iloc[0])
|
||||
|
||||
def test_tier_3_one_column_present(self):
|
||||
"""Tier 3: Only 1 column is non-null."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"PROV_GROUP_TIN": ["123456789"],
|
||||
"PROV_GROUP_NPI": [""],
|
||||
"PROV_GROUP_NAME_FULL_cleaned": [None],
|
||||
}
|
||||
)
|
||||
result = create_grouping_key_and_tier(df)
|
||||
|
||||
self.assertEqual(result["grouping_tier"].iloc[0], 3)
|
||||
self.assertEqual(result["grouping_key"].iloc[0], "TIN:123456789")
|
||||
|
||||
def test_tier_0_no_columns_present(self):
|
||||
"""Tier 0: No columns are non-null (immediate orphan)."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"PROV_GROUP_TIN": [None],
|
||||
"PROV_GROUP_NPI": [""],
|
||||
"PROV_GROUP_NAME_FULL_cleaned": [""],
|
||||
}
|
||||
)
|
||||
result = create_grouping_key_and_tier(df)
|
||||
|
||||
self.assertEqual(result["grouping_tier"].iloc[0], 0)
|
||||
self.assertIsNone(result["grouping_key"].iloc[0])
|
||||
|
||||
|
||||
class TestGroupingCompatibility(unittest.TestCase):
|
||||
"""Tests for check_grouping_compatibility function."""
|
||||
|
||||
def test_compatible_same_tin(self):
|
||||
"""Compatible: Parent and child share same TIN."""
|
||||
parent_key = "TIN:123|NPI:456|NAME:Hospital"
|
||||
child_key = "TIN:123|NPI:789"
|
||||
|
||||
self.assertTrue(check_grouping_compatibility(parent_key, child_key))
|
||||
|
||||
def test_compatible_same_npi(self):
|
||||
"""Compatible: Parent and child share same NPI."""
|
||||
parent_key = "TIN:123|NPI:456"
|
||||
child_key = "NPI:456"
|
||||
|
||||
self.assertTrue(check_grouping_compatibility(parent_key, child_key))
|
||||
|
||||
def test_not_compatible_different_values(self):
|
||||
"""Not compatible: No shared values."""
|
||||
parent_key = "TIN:123|NPI:456"
|
||||
child_key = "TIN:999|NPI:888"
|
||||
|
||||
self.assertFalse(check_grouping_compatibility(parent_key, child_key))
|
||||
|
||||
def test_not_compatible_null_keys(self):
|
||||
"""Not compatible: Null keys."""
|
||||
self.assertFalse(check_grouping_compatibility(None, "TIN:123"))
|
||||
self.assertFalse(check_grouping_compatibility("TIN:123", None))
|
||||
self.assertFalse(check_grouping_compatibility(None, None))
|
||||
|
||||
|
||||
class TestFilterOnExcludeKeywords(unittest.TestCase):
|
||||
"""Tests for filter_on_exclude_keywords function."""
|
||||
|
||||
def test_parent_no_exclude_keywords(self):
|
||||
"""Row without exclude keywords should be marked as parent."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"contract_title_cleaned": ["Provider Agreement"],
|
||||
"grouping_tier": [1],
|
||||
}
|
||||
)
|
||||
result = filter_on_exclude_keywords(df, True)
|
||||
|
||||
self.assertTrue(result["parent"].iloc[0])
|
||||
|
||||
def test_not_parent_with_amendment(self):
|
||||
"""Row with 'amendment' should NOT be marked as parent."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"contract_title_cleaned": ["First Amendment to Agreement"],
|
||||
"grouping_tier": [1],
|
||||
}
|
||||
)
|
||||
result = filter_on_exclude_keywords(df, True)
|
||||
|
||||
self.assertFalse(result["parent"].iloc[0])
|
||||
|
||||
def test_not_parent_with_addendum(self):
|
||||
"""Row with 'addendum' should NOT be marked as parent."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"contract_title_cleaned": ["Addendum A - Rate Schedule"],
|
||||
"grouping_tier": [1],
|
||||
}
|
||||
)
|
||||
result = filter_on_exclude_keywords(df, True)
|
||||
|
||||
self.assertFalse(result["parent"].iloc[0])
|
||||
|
||||
def test_not_parent_with_exhibit(self):
|
||||
"""Row with 'exhibit' should NOT be marked as parent."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"contract_title_cleaned": ["Exhibit B - Service Areas"],
|
||||
"grouping_tier": [1],
|
||||
}
|
||||
)
|
||||
result = filter_on_exclude_keywords(df, True)
|
||||
|
||||
self.assertFalse(result["parent"].iloc[0])
|
||||
|
||||
|
||||
class TestRankParents(unittest.TestCase):
|
||||
"""Tests for rank_parents function."""
|
||||
|
||||
def test_single_parent_gets_rank_1(self):
|
||||
"""Single parent in group gets rank 1."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123"],
|
||||
"parent": [True],
|
||||
"fixed_effective_date": ["2024-01-01"],
|
||||
}
|
||||
)
|
||||
result = rank_parents(df)
|
||||
|
||||
self.assertEqual(result["parent_rank"].iloc[0], 1)
|
||||
|
||||
def test_multiple_parents_ranked_by_date(self):
|
||||
"""Multiple parents ranked by effective date (earliest = rank 1)."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123", "TIN:123"],
|
||||
"parent": [True, True, False],
|
||||
"fixed_effective_date": ["2024-03-01", "2024-01-01", "2024-02-01"],
|
||||
}
|
||||
)
|
||||
result = rank_parents(df)
|
||||
|
||||
# Index 1 has earliest date, should be rank 1
|
||||
self.assertEqual(result["parent_rank"].iloc[1], 1)
|
||||
# Index 0 has latest date, should be rank 2
|
||||
self.assertEqual(result["parent_rank"].iloc[0], 2)
|
||||
# Index 2 is not a parent, should have NaN rank
|
||||
self.assertTrue(pd.isna(result["parent_rank"].iloc[2]))
|
||||
|
||||
def test_non_parents_get_no_rank(self):
|
||||
"""Non-parents should not receive a parent rank."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123"],
|
||||
"parent": [False],
|
||||
"fixed_effective_date": ["2024-01-01"],
|
||||
}
|
||||
)
|
||||
result = rank_parents(df)
|
||||
|
||||
self.assertTrue(pd.isna(result["parent_rank"].iloc[0]))
|
||||
|
||||
|
||||
class TestExtractClientName(unittest.TestCase):
|
||||
"""Tests for extract_client_name function from main.py."""
|
||||
|
||||
def test_simple_filename(self):
|
||||
"""Extract client name from simple filename."""
|
||||
result = extract_client_name("Molina_123.xlsx")
|
||||
self.assertEqual(result, "molina")
|
||||
|
||||
def test_filename_with_spaces(self):
|
||||
"""Extract client name with spaces converted to underscores."""
|
||||
result = extract_client_name("Clover Health_132.xlsx")
|
||||
self.assertEqual(result, "clover_health")
|
||||
|
||||
def test_full_path(self):
|
||||
"""Extract client name from full file path."""
|
||||
result = extract_client_name("/data/uploads/BCBS_NC_456.xlsx")
|
||||
self.assertEqual(result, "bcbs_nc")
|
||||
|
||||
def test_s3_path(self):
|
||||
"""Extract client name from S3 path."""
|
||||
result = extract_client_name("s3://bucket/folder/CareSource_789.xlsx")
|
||||
self.assertEqual(result, "caresource")
|
||||
|
||||
def test_empty_returns_output(self):
|
||||
"""Empty filename after cleaning returns 'output'."""
|
||||
result = extract_client_name("123.xlsx")
|
||||
self.assertEqual(result, "output")
|
||||
|
||||
|
||||
class TestQCUtilities(unittest.TestCase):
|
||||
"""Tests for utility functions from generic_hierarchy_lineage_qc.py."""
|
||||
|
||||
def test_to_string_with_values(self):
|
||||
"""to_string converts series to string type."""
|
||||
series = pd.Series(["abc", "def", None, 123])
|
||||
result = to_string(series)
|
||||
|
||||
self.assertEqual(result.dtype, "string")
|
||||
self.assertEqual(result.iloc[0], "abc")
|
||||
self.assertEqual(result.iloc[2], "") # None becomes empty string
|
||||
|
||||
def test_to_string_with_nan(self):
|
||||
"""to_string fills NaN with empty string."""
|
||||
series = pd.Series([np.nan, "value", pd.NA])
|
||||
result = to_string(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], "")
|
||||
self.assertEqual(result.iloc[1], "value")
|
||||
self.assertEqual(result.iloc[2], "")
|
||||
|
||||
def test_to_datetime_valid_dates(self):
|
||||
"""to_datetime parses valid date strings."""
|
||||
series = pd.Series(["2024-01-15", "2024-06-30"])
|
||||
result = to_datetime(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], pd.Timestamp("2024-01-15"))
|
||||
self.assertEqual(result.iloc[1], pd.Timestamp("2024-06-30"))
|
||||
|
||||
def test_to_datetime_invalid_dates(self):
|
||||
"""to_datetime coerces invalid dates to NaT."""
|
||||
series = pd.Series(["2024-01-15", "not a date", None])
|
||||
result = to_datetime(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], pd.Timestamp("2024-01-15"))
|
||||
self.assertTrue(pd.isna(result.iloc[1]))
|
||||
self.assertTrue(pd.isna(result.iloc[2]))
|
||||
|
||||
def test_resolve_column_string_exists(self):
|
||||
"""resolve_column finds column by string name."""
|
||||
df = pd.DataFrame({"col_a": [1], "col_b": [2]})
|
||||
result = resolve_column(df, "col_a")
|
||||
|
||||
self.assertEqual(result, "col_a")
|
||||
|
||||
def test_resolve_column_string_not_exists(self):
|
||||
"""resolve_column returns None for non-existent column."""
|
||||
df = pd.DataFrame({"col_a": [1]})
|
||||
result = resolve_column(df, "col_x")
|
||||
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_resolve_column_list_first_match(self):
|
||||
"""resolve_column returns first matching column from list."""
|
||||
df = pd.DataFrame({"col_b": [1], "col_c": [2]})
|
||||
result = resolve_column(df, ["col_a", "col_b", "col_c"])
|
||||
|
||||
self.assertEqual(result, "col_b")
|
||||
|
||||
def test_resolve_column_required_raises(self):
|
||||
"""resolve_column raises ValueError when required column missing."""
|
||||
df = pd.DataFrame({"col_a": [1]})
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_column(df, "col_x", required=True, label="test_col")
|
||||
|
||||
|
||||
class TestTextProcessor(unittest.TestCase):
|
||||
"""Tests for TextProcessor class methods."""
|
||||
|
||||
def test_clean_text_removes_special_chars(self):
|
||||
"""clean_text removes special characters."""
|
||||
series = pd.Series(["Hello, World! @#$%"])
|
||||
result = TextProcessor.clean_text(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], "Hello World")
|
||||
|
||||
def test_clean_text_normalizes_whitespace(self):
|
||||
"""clean_text normalizes multiple spaces."""
|
||||
series = pd.Series(["Hello World"])
|
||||
result = TextProcessor.clean_text(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], "Hello World")
|
||||
|
||||
def test_clean_text_lowercase(self):
|
||||
"""clean_text_lowercase converts to lowercase."""
|
||||
series = pd.Series(["HELLO World"])
|
||||
result = TextProcessor.clean_text_lowercase(series)
|
||||
|
||||
self.assertEqual(result.iloc[0], "hello world")
|
||||
|
||||
def test_normalize_identifier_simple(self):
|
||||
"""normalize_identifier handles simple values."""
|
||||
result = TextProcessor.normalize_identifier("123-45-6789")
|
||||
self.assertEqual(result, "123456789")
|
||||
|
||||
def test_normalize_identifier_pipe_separated(self):
|
||||
"""normalize_identifier handles pipe-separated values."""
|
||||
result = TextProcessor.normalize_identifier("123|456|123")
|
||||
self.assertEqual(result, "123|456") # Deduped
|
||||
|
||||
def test_normalize_identifier_none(self):
|
||||
"""normalize_identifier returns empty string for None."""
|
||||
result = TextProcessor.normalize_identifier(None)
|
||||
self.assertEqual(result, "")
|
||||
|
||||
def test_has_text_match_substring(self):
|
||||
"""has_text_match detects substring matches."""
|
||||
self.assertTrue(TextProcessor.has_text_match("Agreement", "Provider Agreement"))
|
||||
self.assertTrue(TextProcessor.has_text_match("Provider Agreement", "Agreement"))
|
||||
|
||||
def test_has_text_match_no_match(self):
|
||||
"""has_text_match returns False for no match."""
|
||||
self.assertFalse(
|
||||
TextProcessor.has_text_match("Amendment", "Provider Agreement")
|
||||
)
|
||||
|
||||
def test_has_text_match_case_insensitive(self):
|
||||
"""has_text_match is case insensitive."""
|
||||
self.assertTrue(TextProcessor.has_text_match("AGREEMENT", "provider agreement"))
|
||||
|
||||
def test_calculate_text_similarity(self):
|
||||
"""calculate_text_similarity computes overlap ratio."""
|
||||
# Identical texts should have similarity 1.0
|
||||
sim = TextProcessor.calculate_text_similarity("hello world", "hello world")
|
||||
self.assertEqual(sim, 1.0)
|
||||
|
||||
# Completely different texts should have similarity 0.0
|
||||
sim = TextProcessor.calculate_text_similarity("abc def", "xyz uvw")
|
||||
self.assertEqual(sim, 0.0)
|
||||
|
||||
# Partial overlap
|
||||
sim = TextProcessor.calculate_text_similarity("hello world", "hello there")
|
||||
self.assertGreater(sim, 0.0)
|
||||
self.assertLess(sim, 1.0)
|
||||
|
||||
|
||||
class TestBuildStrings(unittest.TestCase):
|
||||
"""Tests for string building helper functions."""
|
||||
|
||||
def test_build_grouping_string_single_col(self):
|
||||
"""build_grouping_string with single column."""
|
||||
child_row = pd.Series(
|
||||
{
|
||||
"PROV_GROUP_TIN": "123456789",
|
||||
"PROV_GROUP_NPI": None,
|
||||
"PROV_GROUP_NAME_FULL": None,
|
||||
}
|
||||
)
|
||||
result = build_grouping_string(child_row, ["TIN"])
|
||||
|
||||
self.assertIn("TIN", result)
|
||||
self.assertIn("123456789", result)
|
||||
|
||||
def test_build_grouping_string_multiple_cols(self):
|
||||
"""build_grouping_string with multiple columns."""
|
||||
child_row = pd.Series(
|
||||
{
|
||||
"PROV_GROUP_TIN": "123456789",
|
||||
"PROV_GROUP_NPI": "9876543210",
|
||||
"PROV_GROUP_NAME_FULL": "Test Hospital",
|
||||
}
|
||||
)
|
||||
result = build_grouping_string(child_row, ["TIN", "NPI"])
|
||||
|
||||
self.assertIn("TIN", result)
|
||||
self.assertIn("NPI", result)
|
||||
self.assertIn("and", result)
|
||||
|
||||
def test_build_matching_columns_string_tin_match(self):
|
||||
"""build_matching_columns_string shows matched TIN."""
|
||||
parent_row = pd.Series(
|
||||
{
|
||||
"PROV_GROUP_TIN": "123456789",
|
||||
"PROV_GROUP_NPI": "111",
|
||||
"PROV_GROUP_NAME_FULL": "Parent Hospital",
|
||||
}
|
||||
)
|
||||
child_row = pd.Series(
|
||||
{
|
||||
"PROV_GROUP_TIN": "123456789",
|
||||
"PROV_GROUP_NPI": "222",
|
||||
"PROV_GROUP_NAME_FULL": "Child Clinic",
|
||||
}
|
||||
)
|
||||
result = build_matching_columns_string(parent_row, child_row)
|
||||
|
||||
self.assertIn("TIN", result)
|
||||
self.assertNotIn("NPI", result) # NPI didn't match
|
||||
|
||||
|
||||
class TestFlagMultiParentWithEarlyDates(unittest.TestCase):
|
||||
"""Tests for flag_multi_parent_with_early_dates function."""
|
||||
|
||||
def test_flags_group_with_multiple_early_parents(self):
|
||||
"""Flags groups where multiple parents have dates before children."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123", "TIN:123", "TIN:123"],
|
||||
"parent": [True, True, False, False],
|
||||
"parent_rank": [1.0, 2.0, np.nan, np.nan],
|
||||
"fixed_effective_date": [
|
||||
"2024-01-01",
|
||||
"2024-02-01",
|
||||
"2024-06-01",
|
||||
"2024-07-01",
|
||||
],
|
||||
}
|
||||
)
|
||||
result = flag_multi_parent_with_early_dates(df)
|
||||
|
||||
# Group should be flagged because both parents have dates before children
|
||||
self.assertEqual(
|
||||
result["flag_multi_parent_with_early_dates"].iloc[0],
|
||||
"multiple_parents_early_children",
|
||||
)
|
||||
|
||||
def test_no_flag_single_parent(self):
|
||||
"""No flag when only one parent in group."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123"],
|
||||
"parent": [True, False],
|
||||
"parent_rank": [1.0, np.nan],
|
||||
"fixed_effective_date": ["2024-01-01", "2024-06-01"],
|
||||
}
|
||||
)
|
||||
result = flag_multi_parent_with_early_dates(df)
|
||||
|
||||
self.assertTrue(pd.isna(result["flag_multi_parent_with_early_dates"].iloc[0]))
|
||||
|
||||
def test_no_flag_no_children(self):
|
||||
"""No flag when group has no children."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123"],
|
||||
"parent": [True, True],
|
||||
"parent_rank": [1.0, 2.0],
|
||||
"fixed_effective_date": ["2024-01-01", "2024-02-01"],
|
||||
}
|
||||
)
|
||||
result = flag_multi_parent_with_early_dates(df)
|
||||
|
||||
self.assertTrue(pd.isna(result["flag_multi_parent_with_early_dates"].iloc[0]))
|
||||
|
||||
|
||||
class TestAssignChildRanks(unittest.TestCase):
|
||||
"""Tests for assign_child_ranks function."""
|
||||
|
||||
def test_orphan_children_get_0_prefix(self):
|
||||
"""Orphan children (no parent) get ranks starting with 0.0."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123"],
|
||||
"parent": [False, False],
|
||||
"assigned_parent_rank": ["no_parent", "no_parent"],
|
||||
"fixed_effective_date": ["2024-01-01", "2024-02-01"],
|
||||
}
|
||||
)
|
||||
result = assign_child_ranks(df)
|
||||
|
||||
# Both should have ranks starting with "0.0"
|
||||
self.assertTrue(result["child_rank"].iloc[0].startswith("0.0"))
|
||||
self.assertTrue(result["child_rank"].iloc[1].startswith("0.0"))
|
||||
|
||||
def test_assigned_children_get_parent_prefix(self):
|
||||
"""Children assigned to parent get ranks with parent's rank prefix."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123", "TIN:123"],
|
||||
"parent": [True, False, False],
|
||||
"parent_rank": [1.0, np.nan, np.nan],
|
||||
"assigned_parent_rank": [np.nan, 1.0, 1.0],
|
||||
"fixed_effective_date": ["2024-01-01", "2024-03-01", "2024-02-01"],
|
||||
}
|
||||
)
|
||||
result = assign_child_ranks(df)
|
||||
|
||||
# Parent should have combined_rank "1"
|
||||
self.assertEqual(result["combined_rank"].iloc[0], "1")
|
||||
# Children should have ranks like "1.0.1", "1.0.2"
|
||||
child_ranks = result[result["parent"] == False]["child_rank"].tolist()
|
||||
self.assertTrue(all(r.startswith("1.0.") for r in child_ranks))
|
||||
|
||||
def test_children_ranked_by_effective_date(self):
|
||||
"""Children within same parent group ranked by effective date."""
|
||||
df = pd.DataFrame(
|
||||
{
|
||||
"grouping_key": ["TIN:123", "TIN:123", "TIN:123"],
|
||||
"parent": [True, False, False],
|
||||
"parent_rank": [1.0, np.nan, np.nan],
|
||||
"assigned_parent_rank": [np.nan, 1.0, 1.0],
|
||||
"fixed_effective_date": ["2024-01-01", "2024-06-01", "2024-03-01"],
|
||||
}
|
||||
)
|
||||
result = assign_child_ranks(df)
|
||||
|
||||
# Child with earlier date (index 2) should have lower rank number
|
||||
child_df = result[result["parent"] == False].sort_values("fixed_effective_date")
|
||||
ranks = child_df["child_rank"].tolist()
|
||||
# Earlier date should have .1, later date should have .2
|
||||
self.assertIn(".1", ranks[0])
|
||||
self.assertIn(".2", ranks[1])
|
||||
|
||||
|
||||
class TestExtractOrdinal(unittest.TestCase):
|
||||
"""Tests for extract_ordinal function."""
|
||||
|
||||
def test_extract_numeric_ordinal(self):
|
||||
"""Extracts numeric ordinals like '1st', '2nd', '3rd'."""
|
||||
row = pd.Series({"contract_title_cleaned": "1st Amendment to Agreement"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
row = pd.Series({"contract_title_cleaned": "2nd Amendment"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertEqual(result, 2)
|
||||
|
||||
row = pd.Series({"contract_title_cleaned": "3rd Addendum"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertEqual(result, 3)
|
||||
|
||||
def test_extract_word_ordinal(self):
|
||||
"""Extracts word ordinals like 'first', 'second', 'third'."""
|
||||
row = pd.Series({"contract_title_cleaned": "First Amendment to Agreement"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
row = pd.Series({"contract_title_cleaned": "Second Amendment"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertEqual(result, 2)
|
||||
|
||||
def test_no_ordinal_returns_none(self):
|
||||
"""Returns None when no ordinal found."""
|
||||
row = pd.Series({"contract_title_cleaned": "Provider Agreement"})
|
||||
result = extract_ordinal(row)
|
||||
self.assertIsNone(result)
|
||||
|
||||
|
||||
class TestConfigFactory(unittest.TestCase):
|
||||
"""Tests for ConfigFactory class."""
|
||||
|
||||
def test_get_molina_config(self):
|
||||
"""ConfigFactory returns Molina configuration."""
|
||||
config = ConfigFactory.get_config("molina")
|
||||
|
||||
self.assertEqual(config.name, "molina")
|
||||
self.assertIsNotNone(config.parent_identification)
|
||||
|
||||
def test_get_caresource_config(self):
|
||||
"""ConfigFactory returns CareSource configuration."""
|
||||
config = ConfigFactory.get_config("caresource")
|
||||
|
||||
self.assertEqual(config.name, "caresource")
|
||||
self.assertIsNotNone(config.parent_identification)
|
||||
|
||||
def test_get_bcbs_config(self):
|
||||
"""ConfigFactory returns BCBS configuration."""
|
||||
config = ConfigFactory.get_config("bcbs")
|
||||
|
||||
self.assertEqual(config.name, "bcbs")
|
||||
self.assertIsNotNone(config.parent_identification)
|
||||
|
||||
def test_get_clover_config(self):
|
||||
"""ConfigFactory returns Clover Health configuration."""
|
||||
config = ConfigFactory.get_config("clover_health")
|
||||
|
||||
self.assertEqual(config.name, "clover_health")
|
||||
|
||||
def test_unknown_client_returns_molina_default(self):
|
||||
"""Unknown client falls back to Molina config."""
|
||||
config = ConfigFactory.get_config("unknown_client")
|
||||
|
||||
# Should return a valid config (Molina as default)
|
||||
self.assertIsNotNone(config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user