Merged in BCBS/PC_RUN (pull request #916)
BCBS/PC RUN * Bcbs_pc_fix_saas_output * black format * Fallback orphans * black format fix Approved-by: Katon Minhas
This commit is contained in:
Rahul Ailaboina
committed by
Katon Minhas
Katon Minhas
parent
d32b89a9f7
commit
1bda35e7c0
@@ -10,6 +10,7 @@ from src.constants.parent_child.generic import (
|
||||
ASSIGNMENT_NO_PARENT,
|
||||
DATE_FORMAT,
|
||||
LEGAL_SUFFIXES,
|
||||
FILENAME_EXCLUDE_WORDS,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
@@ -22,4 +23,5 @@ __all__ = [
|
||||
"ASSIGNMENT_NO_PARENT",
|
||||
"DATE_FORMAT",
|
||||
"LEGAL_SUFFIXES",
|
||||
"FILENAME_EXCLUDE_WORDS",
|
||||
]
|
||||
|
||||
@@ -70,3 +70,60 @@ FILE_EXTENSIONS = [
|
||||
".htm",
|
||||
".xps",
|
||||
]
|
||||
|
||||
# Words/patterns to exclude when deriving provider group names from file names
|
||||
FILENAME_EXCLUDE_WORDS = [
|
||||
"amendment",
|
||||
"amend",
|
||||
"amd",
|
||||
"addendum",
|
||||
"med",
|
||||
"medicare",
|
||||
"medi-cal",
|
||||
"medical",
|
||||
"medicaid",
|
||||
"rate",
|
||||
"rates",
|
||||
"effective",
|
||||
"ecm",
|
||||
"ecf",
|
||||
"inc",
|
||||
"llc",
|
||||
"corp",
|
||||
"corporation",
|
||||
"incorporated",
|
||||
"ltd",
|
||||
"lp",
|
||||
"llp",
|
||||
"pty",
|
||||
"company",
|
||||
"contract",
|
||||
"agreement",
|
||||
"exhibit",
|
||||
"appendix",
|
||||
"attachment",
|
||||
"schedule",
|
||||
"renewal",
|
||||
"extension",
|
||||
"termination",
|
||||
"notice",
|
||||
"letter",
|
||||
"form",
|
||||
"template",
|
||||
"draft",
|
||||
"final",
|
||||
"revised",
|
||||
"version",
|
||||
"copy",
|
||||
"original",
|
||||
"signed",
|
||||
"executed",
|
||||
"fully",
|
||||
"page",
|
||||
"pages",
|
||||
"section",
|
||||
"part",
|
||||
"number",
|
||||
"no",
|
||||
"num",
|
||||
]
|
||||
|
||||
+255
-11
@@ -18,6 +18,7 @@ from src.constants.parent_child import (
|
||||
TIER_ONE_IDENTIFIER,
|
||||
TIER_NO_IDENTIFIERS,
|
||||
ASSIGNMENT_NO_PARENT,
|
||||
FILENAME_EXCLUDE_WORDS,
|
||||
)
|
||||
from typing import Tuple, Optional, List, Dict, Any
|
||||
|
||||
@@ -95,6 +96,184 @@ def create_grouping_key_and_tier(df):
|
||||
return df
|
||||
|
||||
|
||||
def _clean_filename_for_grouping(filename):
|
||||
"""
|
||||
Cleans a filename to derive a potential provider group name.
|
||||
|
||||
Removes file extensions, legal suffixes, contract/amendment words,
|
||||
numeric noise, and other non-name tokens defined in FILENAME_EXCLUDE_WORDS.
|
||||
|
||||
Returns:
|
||||
str or None: Cleaned name, or None if nothing meaningful remains.
|
||||
"""
|
||||
if pd.isna(filename) or not filename:
|
||||
return None
|
||||
|
||||
name = str(filename).strip()
|
||||
|
||||
# Remove common file extensions
|
||||
name = re.sub(
|
||||
r"\.(txt|pdf|msg|xlsx|docx|tif|mht|doc|htm|xps|csv|eml)$",
|
||||
"",
|
||||
name,
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Remove "Filename: " prefix
|
||||
name = re.sub(r"^Filename:\s*", "", name, flags=re.IGNORECASE)
|
||||
|
||||
# Replace underscores, hyphens, dots with spaces for word-level processing
|
||||
name = re.sub(r"[_\-\.]+", " ", name)
|
||||
|
||||
# Remove non-alphanumeric characters except spaces
|
||||
name = re.sub(r"[^a-zA-Z0-9\s]", " ", name)
|
||||
|
||||
# Lowercase for matching
|
||||
name = name.lower().strip()
|
||||
|
||||
# Split into words and filter
|
||||
words = name.split()
|
||||
cleaned_words = []
|
||||
exclude_set = set(FILENAME_EXCLUDE_WORDS)
|
||||
|
||||
for word in words:
|
||||
# Skip excluded words
|
||||
if word in exclude_set:
|
||||
continue
|
||||
# Skip purely numeric tokens
|
||||
if re.match(r"^\d+$", word):
|
||||
continue
|
||||
# Skip tokens that end with numbers (e.g., "v2", "rev3")
|
||||
if re.match(r"^[a-z]+\d+$", word):
|
||||
continue
|
||||
cleaned_words.append(word)
|
||||
|
||||
if not cleaned_words:
|
||||
return None
|
||||
|
||||
result = " ".join(cleaned_words).strip()
|
||||
|
||||
# If the result is too short (single char) or just whitespace, treat as unusable
|
||||
if len(result) <= 1:
|
||||
return None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _assign_filename_based_grouping(df):
|
||||
"""
|
||||
For Tier-0 (immediate orphan) rows where TIN, NPI, and PROV_NAME are all blank,
|
||||
attempts to assign a grouping using the file name as a fallback.
|
||||
|
||||
Tier 2 fallback: Match filename against existing provider group names.
|
||||
Tier 3 fallback: Derive a temporary group name from the cleaned filename.
|
||||
Tier 4: If filename is too generic, keep as orphan.
|
||||
|
||||
Args:
|
||||
df: DataFrame with grouping_key, grouping_tier, and contract_name_cleaned columns.
|
||||
|
||||
Returns:
|
||||
DataFrame with updated grouping_key, grouping_tier, grouping_columns,
|
||||
and assignment_reasoning for rows that were re-grouped.
|
||||
"""
|
||||
df = df.copy()
|
||||
|
||||
# Identify Tier 0 rows (all grouping fields blank)
|
||||
tier0_mask = df["grouping_tier"] == TIER_NO_IDENTIFIERS
|
||||
if tier0_mask.sum() == 0:
|
||||
logging.info("Filename fallback: No Tier-0 orphans to process.")
|
||||
return df
|
||||
|
||||
logging.info(f"Filename fallback: Processing {tier0_mask.sum()} Tier-0 orphan rows")
|
||||
|
||||
# Collect all existing provider group names (from non-orphan rows)
|
||||
existing_groups = set(
|
||||
df.loc[df["grouping_tier"] > 0, "PROV_GROUP_NAME_FULL_cleaned"]
|
||||
.dropna()
|
||||
.unique()
|
||||
)
|
||||
existing_groups.discard("")
|
||||
logging.info(
|
||||
f"Filename fallback: {len(existing_groups)} existing provider group names available for matching"
|
||||
)
|
||||
|
||||
# Build a lookup: lowercase group name -> original group name
|
||||
group_lookup = {}
|
||||
for g in existing_groups:
|
||||
group_lookup[g.lower().strip()] = g
|
||||
|
||||
reassigned_match = 0
|
||||
reassigned_derived = 0
|
||||
kept_orphan = 0
|
||||
|
||||
for idx in df.index[tier0_mask]:
|
||||
row = df.loc[idx]
|
||||
|
||||
# Use contract_name_cleaned (already cleaned filename) if available
|
||||
filename = row.get("contract_name_cleaned", row.get("FILE_NAME", ""))
|
||||
if pd.isna(filename) or not filename:
|
||||
kept_orphan += 1
|
||||
continue
|
||||
|
||||
filename_lower = str(filename).lower().strip()
|
||||
|
||||
# --- Tier 2 fallback: Match against existing provider group names ---
|
||||
best_match = None
|
||||
best_match_len = 0
|
||||
|
||||
for group_lower, group_original in group_lookup.items():
|
||||
if not group_lower:
|
||||
continue
|
||||
# Check if the existing group name appears in the filename
|
||||
if group_lower in filename_lower:
|
||||
# Prefer longer matches (more specific)
|
||||
if len(group_lower) > best_match_len:
|
||||
best_match = group_original
|
||||
best_match_len = len(group_lower)
|
||||
|
||||
if best_match:
|
||||
# Assign grouping using the matched existing provider group name
|
||||
new_key = f"{GROUPING_KEY_PREFIXES['NAME']}{best_match}"
|
||||
df.at[idx, "grouping_key"] = new_key
|
||||
df.at[idx, "grouping_tier"] = TIER_ONE_IDENTIFIER
|
||||
df.at[idx, "grouping_columns"] = ["PROV_NAME"]
|
||||
df.at[idx, "PROV_GROUP_NAME_FULL_cleaned"] = best_match
|
||||
df.at[idx, "assignment_reasoning"] = (
|
||||
f"Filename fallback (Tier 2): matched existing provider group "
|
||||
f"'{best_match}' from filename '{filename}'"
|
||||
)
|
||||
reassigned_match += 1
|
||||
continue
|
||||
|
||||
# --- Tier 3 fallback: Derive temporary group name from filename ---
|
||||
derived_name = _clean_filename_for_grouping(filename)
|
||||
|
||||
if derived_name:
|
||||
new_key = f"{GROUPING_KEY_PREFIXES['NAME']}{derived_name}"
|
||||
df.at[idx, "grouping_key"] = new_key
|
||||
df.at[idx, "grouping_tier"] = TIER_ONE_IDENTIFIER
|
||||
df.at[idx, "grouping_columns"] = ["PROV_NAME"]
|
||||
df.at[idx, "PROV_GROUP_NAME_FULL_cleaned"] = derived_name
|
||||
df.at[idx, "assignment_reasoning"] = (
|
||||
f"Filename fallback (Tier 3): derived group name "
|
||||
f"'{derived_name}' from filename '{filename}'"
|
||||
)
|
||||
reassigned_derived += 1
|
||||
continue
|
||||
|
||||
# --- Tier 4: Filename too generic, keep as orphan ---
|
||||
kept_orphan += 1
|
||||
|
||||
logging.info(
|
||||
f"Filename fallback results: "
|
||||
f"matched existing group={reassigned_match}, "
|
||||
f"derived from filename={reassigned_derived}, "
|
||||
f"kept as orphan={kept_orphan}"
|
||||
)
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def build_grouping_string(child_row, grouping_cols):
|
||||
"""
|
||||
Builds a formatted string showing which grouping columns and their values were used.
|
||||
@@ -392,7 +571,7 @@ def assign_parents_with_state_and_payer_cross_tier(
|
||||
)
|
||||
continue
|
||||
|
||||
# Step 3: Apply effective date constraint
|
||||
# Step 3: Apply effective date constraint (with fallback)
|
||||
if pd.notna(child["fixed_effective_date"]):
|
||||
valid_date_parents = state_parents[
|
||||
pd.to_datetime(state_parents["fixed_effective_date"], errors="coerce")
|
||||
@@ -400,12 +579,49 @@ def assign_parents_with_state_and_payer_cross_tier(
|
||||
]
|
||||
|
||||
if valid_date_parents.empty:
|
||||
assignments.append((ASSIGNMENT_NO_PARENT, idx))
|
||||
grouping_str = build_grouping_string(child, child_grouping_cols)
|
||||
reasoning_map[idx] = (
|
||||
f"No parent with effective date on or before child date ({child['fixed_effective_date'].date()})"
|
||||
)
|
||||
continue
|
||||
# Fallback: amendments can have earlier dates than base contracts
|
||||
# If there's a compatible parent in the same group, assign anyway
|
||||
if len(state_parents) == 1:
|
||||
parent = state_parents.iloc[0]
|
||||
parent_rank = parent["parent_rank"]
|
||||
parent_name = parent["FILE_NAME"]
|
||||
assignments.append((parent_rank, idx))
|
||||
grouping_str = build_matching_columns_string(parent, child)
|
||||
reasoning_map[idx] = (
|
||||
f"Matched on {grouping_str}; state match, date relaxed "
|
||||
f"(child date {child['fixed_effective_date'].date()} before "
|
||||
f"parent date, single parent in group)"
|
||||
)
|
||||
parent_name_map[idx] = parent_name
|
||||
continue
|
||||
elif len(state_parents) > 1:
|
||||
# Multiple parents but none with valid date - pick closest date
|
||||
state_parents_sorted = state_parents.copy()
|
||||
state_parents_sorted["_date_diff"] = abs(
|
||||
pd.to_datetime(
|
||||
state_parents_sorted["fixed_effective_date"],
|
||||
errors="coerce",
|
||||
)
|
||||
- child["fixed_effective_date"]
|
||||
)
|
||||
closest = state_parents_sorted.sort_values("_date_diff").iloc[0]
|
||||
parent_rank = closest["parent_rank"]
|
||||
parent_name = closest["FILE_NAME"]
|
||||
assignments.append((parent_rank, idx))
|
||||
grouping_str = build_matching_columns_string(closest, child)
|
||||
reasoning_map[idx] = (
|
||||
f"Matched on {grouping_str}; state match, date relaxed "
|
||||
f"(closest parent by date among {len(state_parents)})"
|
||||
)
|
||||
parent_name_map[idx] = parent_name
|
||||
continue
|
||||
else:
|
||||
assignments.append((ASSIGNMENT_NO_PARENT, idx))
|
||||
grouping_str = build_grouping_string(child, child_grouping_cols)
|
||||
reasoning_map[idx] = (
|
||||
f"No parent with effective date on or before child date ({child['fixed_effective_date'].date()})"
|
||||
)
|
||||
continue
|
||||
|
||||
state_parents = valid_date_parents
|
||||
|
||||
@@ -695,12 +911,21 @@ def assign_orphans_based_on_unique_parent_and_payer_cross_tier(
|
||||
)
|
||||
df_work.at[idx, "parent_name"] = parent_filename
|
||||
assignments_made += 1
|
||||
else:
|
||||
# Relaxed date: single parent in group, assign even if child date is earlier
|
||||
df_work.at[idx, "assigned_parent_rank"] = parent_rank
|
||||
df_work.at[idx, "assignment_reasoning"] = (
|
||||
f"Matched on {grouping_str} and payer ({payer_display}) with unique parent in group; date relaxed"
|
||||
)
|
||||
df_work.at[idx, "parent_name"] = parent_filename
|
||||
assignments_made += 1
|
||||
|
||||
elif unique_parent_ranks > 1:
|
||||
sorted_parents = possible_parents.sort_values(
|
||||
"parent_rank", ascending=False
|
||||
)
|
||||
|
||||
assigned_in_loop = False
|
||||
for _, parent in sorted_parents.iterrows():
|
||||
parent_effective_date = pd.to_datetime(
|
||||
parent["fixed_effective_date"], errors="coerce"
|
||||
@@ -717,8 +942,20 @@ def assign_orphans_based_on_unique_parent_and_payer_cross_tier(
|
||||
)
|
||||
df_work.at[idx, "parent_name"] = parent["FILE_NAME"]
|
||||
assignments_made += 1
|
||||
assigned_in_loop = True
|
||||
break
|
||||
|
||||
# Fallback: no date-valid parent, pick closest by date
|
||||
if not assigned_in_loop:
|
||||
closest = sorted_parents.iloc[0] # already sorted by rank desc
|
||||
grouping_str = build_matching_columns_string(closest, orphan)
|
||||
df_work.at[idx, "assigned_parent_rank"] = closest["parent_rank"]
|
||||
df_work.at[idx, "assignment_reasoning"] = (
|
||||
f"Matched on {grouping_str} and payer ({payer_display}); date relaxed, selected latest parent among {unique_parent_ranks}"
|
||||
)
|
||||
df_work.at[idx, "parent_name"] = closest["FILE_NAME"]
|
||||
assignments_made += 1
|
||||
|
||||
logging.info(
|
||||
f"Tier {tier} - Assigned {assignments_made} orphans based on unique parent and payer logic"
|
||||
)
|
||||
@@ -903,13 +1140,20 @@ def parent_child_mapping(
|
||||
f"Grouping keys created. Tier distribution:\n{cleaned_df['grouping_tier'].value_counts().sort_index()}"
|
||||
)
|
||||
|
||||
# Phase 1: Separate immediate orphans (tier 0)
|
||||
# Phase 0.5: Filename-based fallback grouping for Tier-0 orphans
|
||||
# Before separating orphans, try to rescue Tier-0 rows using filename matching
|
||||
cleaned_df = _assign_filename_based_grouping(cleaned_df)
|
||||
logging.info(
|
||||
f"After filename fallback - Tier distribution:\n{cleaned_df['grouping_tier'].value_counts().sort_index()}"
|
||||
)
|
||||
|
||||
# Phase 1: Separate immediate orphans (tier 0) — only true orphans remain
|
||||
immediate_orphans = cleaned_df[cleaned_df["grouping_tier"] == 0].copy()
|
||||
immediate_orphans["parent"] = False
|
||||
immediate_orphans["assigned_parent_rank"] = ASSIGNMENT_NO_PARENT
|
||||
immediate_orphans["assignment_reasoning"] = (
|
||||
"Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null"
|
||||
)
|
||||
immediate_orphans["assignment_reasoning"] = immediate_orphans[
|
||||
"assignment_reasoning"
|
||||
].fillna("Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null")
|
||||
|
||||
workable_df = cleaned_df[cleaned_df["grouping_tier"] > 0].copy()
|
||||
logging.info(
|
||||
|
||||
+201
-198
@@ -1148,7 +1148,7 @@ class ChildAssigner:
|
||||
|
||||
child_eff = row.get(col_eff, pd.NaT) if col_eff else pd.NaT
|
||||
|
||||
# Filter by date
|
||||
# Filter by date, fallback to all candidates if none are date-valid
|
||||
date_valid_cand = [
|
||||
pidx
|
||||
for pidx in cand
|
||||
@@ -1289,225 +1289,210 @@ class BCBSChildAssigner:
|
||||
col_lob = constraints.get("lob", "_lob")
|
||||
col_provider_type = constraints.get("provider_type", "_provider_type")
|
||||
|
||||
# Group by Folder + IRS_Group (grouping_key)
|
||||
for gk, group in df.groupby("grouping_key", dropna=False):
|
||||
if pd.isna(gk):
|
||||
# Build parent index for cross-tier candidate lookup
|
||||
index_fields = [
|
||||
f
|
||||
for f in (config.get("candidate_index_fields", []) or [])
|
||||
if isinstance(f, str) and f
|
||||
]
|
||||
child_assigner = ChildAssigner()
|
||||
parent_index = child_assigner.build_parent_index(df, index_fields)
|
||||
|
||||
all_parents = df[df["is_parent"]].copy()
|
||||
all_children = df[~df["is_parent"] & df["processable"]].copy()
|
||||
|
||||
if all_parents.empty or all_children.empty:
|
||||
return df
|
||||
|
||||
# Build parent cache for all parents
|
||||
parent_cache = {}
|
||||
for pidx in all_parents.index:
|
||||
parent_cache[pidx] = {
|
||||
"title": (
|
||||
self.get_str(all_parents.at[pidx, col_title])
|
||||
if col_title in all_parents.columns
|
||||
else ""
|
||||
),
|
||||
"eff_date": (
|
||||
all_parents.at[pidx, col_eff]
|
||||
if col_eff in all_parents.columns
|
||||
else pd.NaT
|
||||
),
|
||||
"lob": (
|
||||
self.get_str(all_parents.at[pidx, col_lob])
|
||||
if col_lob in all_parents.columns
|
||||
else ""
|
||||
),
|
||||
"provider_type": (
|
||||
self.get_str(all_parents.at[pidx, col_provider_type])
|
||||
if col_provider_type in all_parents.columns
|
||||
else ""
|
||||
),
|
||||
"parent_rank": (
|
||||
int(all_parents.at[pidx, "parent_rank"])
|
||||
if "parent_rank" in all_parents.columns
|
||||
else 0
|
||||
),
|
||||
}
|
||||
|
||||
# Process each child — find candidates via index lookup (cross-tier)
|
||||
for cidx in all_children.index:
|
||||
child_row = df.loc[cidx]
|
||||
child_title = (
|
||||
self.get_str(df.at[cidx, col_title]) if col_title in df.columns else ""
|
||||
)
|
||||
child_eff = df.at[cidx, col_eff] if col_eff in df.columns else pd.NaT
|
||||
child_lob = (
|
||||
self.get_str(df.at[cidx, col_lob]) if col_lob in df.columns else ""
|
||||
)
|
||||
child_ptype = (
|
||||
self.get_str(df.at[cidx, col_provider_type])
|
||||
if col_provider_type in df.columns
|
||||
else ""
|
||||
)
|
||||
|
||||
candidate_idx = None
|
||||
rule_used = ""
|
||||
|
||||
# Find eligible parents via cross-tier candidate lookup
|
||||
cand_indices = child_assigner.find_candidates(
|
||||
parent_index, child_row, index_fields
|
||||
)
|
||||
eligible_parents = [p for p in cand_indices if p in parent_cache]
|
||||
|
||||
if not eligible_parents:
|
||||
continue
|
||||
|
||||
parents = group[group["is_parent"]].copy()
|
||||
children = group[~group["is_parent"]].copy()
|
||||
# Rule 0: Professional logic
|
||||
if "professional" in child_title and candidate_idx is None:
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p_title = parent_cache[pidx]["title"]
|
||||
if p_title and len(p_title) > 0 and p_title in child_title:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 0a: Professional subset match"
|
||||
break
|
||||
|
||||
if parents.empty or children.empty:
|
||||
continue
|
||||
|
||||
# Sort parents by effective date
|
||||
parents = parents.sort_values(col_eff, na_position="last")
|
||||
|
||||
# Build parent cache for this group
|
||||
parent_cache = {}
|
||||
for pidx in parents.index:
|
||||
parent_cache[pidx] = {
|
||||
"title": (
|
||||
self.get_str(parents.at[pidx, col_title])
|
||||
if col_title in parents.columns
|
||||
else ""
|
||||
),
|
||||
"eff_date": (
|
||||
parents.at[pidx, col_eff]
|
||||
if col_eff in parents.columns
|
||||
else pd.NaT
|
||||
),
|
||||
"lob": (
|
||||
self.get_str(parents.at[pidx, col_lob])
|
||||
if col_lob in parents.columns
|
||||
else ""
|
||||
),
|
||||
"provider_type": (
|
||||
self.get_str(parents.at[pidx, col_provider_type])
|
||||
if col_provider_type in parents.columns
|
||||
else ""
|
||||
),
|
||||
"parent_rank": (
|
||||
int(parents.at[pidx, "parent_rank"])
|
||||
if "parent_rank" in parents.columns
|
||||
else 0
|
||||
),
|
||||
}
|
||||
|
||||
# Process each child
|
||||
for cidx in children.index:
|
||||
child_title = (
|
||||
self.get_str(df.at[cidx, col_title])
|
||||
if col_title in df.columns
|
||||
else ""
|
||||
)
|
||||
child_eff = df.at[cidx, col_eff] if col_eff in df.columns else pd.NaT
|
||||
child_lob = (
|
||||
self.get_str(df.at[cidx, col_lob]) if col_lob in df.columns else ""
|
||||
)
|
||||
child_ptype = (
|
||||
self.get_str(df.at[cidx, col_provider_type])
|
||||
if col_provider_type in df.columns
|
||||
else ""
|
||||
)
|
||||
|
||||
candidate_idx = None
|
||||
rule_used = ""
|
||||
|
||||
# Get date-eligible parents (parent_eff <= child_eff)
|
||||
def is_date_eligible(pidx: int) -> bool:
|
||||
p_eff = parent_cache[pidx]["eff_date"]
|
||||
if pd.isna(child_eff):
|
||||
return True # No child date, all parents eligible
|
||||
if pd.isna(p_eff):
|
||||
return False # Parent has no date, not eligible
|
||||
return p_eff <= child_eff
|
||||
|
||||
eligible_parents = [
|
||||
pidx for pidx in parent_cache.keys() if is_date_eligible(pidx)
|
||||
]
|
||||
|
||||
# Rule 0: Professional logic
|
||||
if "professional" in child_title and candidate_idx is None:
|
||||
# 0a: Parent title fully contained in child title
|
||||
if candidate_idx is None:
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p_title = parent_cache[pidx]["title"]
|
||||
if p_title and len(p_title) > 0 and p_title in child_title:
|
||||
if "professional" in parent_cache[pidx]["title"]:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 0a: Professional subset match"
|
||||
rule_used = "Rule 0b: Professional keyword"
|
||||
break
|
||||
|
||||
# 0b: Parent title contains "professional"
|
||||
if candidate_idx is None:
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
if "professional" in parent_cache[pidx]["title"]:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 0b: Professional keyword"
|
||||
break
|
||||
|
||||
# Rule 1: LOB + Provider Type match
|
||||
if (
|
||||
candidate_idx is None
|
||||
and not self.is_missing_lob(child_lob)
|
||||
and child_ptype
|
||||
# Rule 1: LOB + Provider Type match
|
||||
if (
|
||||
candidate_idx is None
|
||||
and not self.is_missing_lob(child_lob)
|
||||
and child_ptype
|
||||
):
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p = parent_cache[pidx]
|
||||
if (
|
||||
p["lob"] == child_lob
|
||||
and p["provider_type"]
|
||||
and p["provider_type"] == child_ptype
|
||||
):
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 1: LOB + Provider Type match"
|
||||
break
|
||||
|
||||
# Rule 2: Title substring match (for missing LOB)
|
||||
if candidate_idx is None and self.is_missing_lob(child_lob):
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p_title = parent_cache[pidx]["title"]
|
||||
if p_title and len(p_title) > 0 and p_title in child_title:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 2: Title substring (missing LOB)"
|
||||
break
|
||||
|
||||
# Rule 3: Provider Type logic
|
||||
if candidate_idx is None:
|
||||
if child_ptype:
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p = parent_cache[pidx]
|
||||
if (
|
||||
p["lob"] == child_lob
|
||||
and p["provider_type"]
|
||||
and p["provider_type"] == child_ptype
|
||||
):
|
||||
if p["lob"] == child_lob and p["provider_type"] == child_ptype:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 1: LOB + Provider Type match"
|
||||
rule_used = "Rule 3a: Provider Type match"
|
||||
break
|
||||
|
||||
# Rule 2: Title substring match (for missing LOB)
|
||||
if candidate_idx is None and self.is_missing_lob(child_lob):
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p_title = parent_cache[pidx]["title"]
|
||||
if p_title and len(p_title) > 0 and p_title in child_title:
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 2: Title substring (missing LOB)"
|
||||
break
|
||||
|
||||
# Rule 3: Provider Type logic
|
||||
if candidate_idx is None:
|
||||
if child_ptype:
|
||||
# 3a: Match by LOB + Provider Type
|
||||
for pidx in sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
):
|
||||
p = parent_cache[pidx]
|
||||
if (
|
||||
p["lob"] == child_lob
|
||||
and p["provider_type"] == child_ptype
|
||||
):
|
||||
candidate_idx = pidx
|
||||
rule_used = "Rule 3a: Provider Type match"
|
||||
break
|
||||
else:
|
||||
# 3b: Fallback - find parent with same LOB, prefer 'facility' provider type
|
||||
lob_matches = [
|
||||
pidx
|
||||
for pidx in eligible_parents
|
||||
if parent_cache[pidx]["lob"] == child_lob
|
||||
]
|
||||
if lob_matches:
|
||||
# Prefer facility
|
||||
facility_matches = [
|
||||
pidx
|
||||
for pidx in lob_matches
|
||||
if "facility" in parent_cache[pidx]["provider_type"]
|
||||
]
|
||||
if facility_matches:
|
||||
candidate_idx = sorted(
|
||||
facility_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 3b: Provider Type Count (facility)"
|
||||
else:
|
||||
candidate_idx = sorted(
|
||||
lob_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 3b: Provider Type Count fallback"
|
||||
|
||||
# Rule 4: Commercial LOB fallback
|
||||
if candidate_idx is None:
|
||||
comm_matches = [
|
||||
else:
|
||||
lob_matches = [
|
||||
pidx
|
||||
for pidx in eligible_parents
|
||||
if "comm" in parent_cache[pidx]["lob"]
|
||||
if parent_cache[pidx]["lob"] == child_lob
|
||||
]
|
||||
if comm_matches:
|
||||
candidate_idx = sorted(
|
||||
comm_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 4: Commercial fallback"
|
||||
if lob_matches:
|
||||
facility_matches = [
|
||||
pidx
|
||||
for pidx in lob_matches
|
||||
if "facility" in parent_cache[pidx]["provider_type"]
|
||||
]
|
||||
if facility_matches:
|
||||
candidate_idx = sorted(
|
||||
facility_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 3b: Provider Type Count (facility)"
|
||||
else:
|
||||
candidate_idx = sorted(
|
||||
lob_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"]
|
||||
or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 3b: Provider Type Count fallback"
|
||||
|
||||
# Rule 5: Latest parent fallback
|
||||
if candidate_idx is None and eligible_parents:
|
||||
# Rule 4: Commercial LOB fallback
|
||||
if candidate_idx is None:
|
||||
comm_matches = [
|
||||
pidx
|
||||
for pidx in eligible_parents
|
||||
if "comm" in parent_cache[pidx]["lob"]
|
||||
]
|
||||
if comm_matches:
|
||||
candidate_idx = sorted(
|
||||
eligible_parents,
|
||||
comm_matches,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 5: Latest parent fallback"
|
||||
rule_used = "Rule 4: Commercial fallback"
|
||||
|
||||
# Assign the candidate
|
||||
if candidate_idx is not None:
|
||||
p_rank = parent_cache[candidate_idx]["parent_rank"]
|
||||
df.at[cidx, "assigned_parent_idx"] = candidate_idx
|
||||
df.at[cidx, "assignment_pass"] = 1 # All BCBS rules are pass 1
|
||||
df.at[cidx, "assignment_reasoning"] = f"{rule_used} (rank={p_rank})"
|
||||
# Rule 5: Latest parent fallback
|
||||
if candidate_idx is None and eligible_parents:
|
||||
candidate_idx = sorted(
|
||||
eligible_parents,
|
||||
key=lambda x: parent_cache[x]["eff_date"] or pd.Timestamp.min,
|
||||
reverse=True,
|
||||
)[0]
|
||||
rule_used = "Rule 5: Latest parent fallback"
|
||||
|
||||
# Assign the candidate
|
||||
if candidate_idx is not None:
|
||||
p_rank = parent_cache[candidate_idx]["parent_rank"]
|
||||
df.at[cidx, "assigned_parent_idx"] = candidate_idx
|
||||
df.at[cidx, "assignment_pass"] = 1
|
||||
df.at[cidx, "assignment_reasoning"] = f"{rule_used} (rank={p_rank})"
|
||||
|
||||
return df
|
||||
|
||||
@@ -1703,20 +1688,27 @@ class ConfigFactory:
|
||||
"contract_title_clean",
|
||||
"fixed_contract_name",
|
||||
"Contract Name",
|
||||
"contract_title_cleaned",
|
||||
],
|
||||
"effective_date": [
|
||||
"final_effective_date",
|
||||
"Contract Effective Date",
|
||||
"fixed_effective_date",
|
||||
],
|
||||
"effective_date": ["final_effective_date", "Contract Effective Date"],
|
||||
},
|
||||
prep_fields={
|
||||
# Grouping fields - match original: Folder + IRS_Group
|
||||
# Grouping fields - old Camelot columns primary, generic pipeline as fallback
|
||||
"_folder": {
|
||||
"source": "Folder",
|
||||
"mode": "copy_clean",
|
||||
"cleaner": "upper_clean",
|
||||
"fallback_sources": ["PROV_GROUP_NAME_FULL_cleaned"],
|
||||
},
|
||||
"_irs_group": {
|
||||
"source": "IRS_Group",
|
||||
"mode": "copy_clean",
|
||||
"cleaner": "lower_clean",
|
||||
"fallback_sources": ["PROV_GROUP_TIN"],
|
||||
},
|
||||
# Additional fields for matching
|
||||
"_provider_type": {
|
||||
@@ -1728,9 +1720,10 @@ class ConfigFactory:
|
||||
"source": "extracted_lob",
|
||||
"mode": "copy_clean",
|
||||
"cleaner": "lower_clean",
|
||||
"fallback_sources": ["consolidated_aarete_derived_lob"],
|
||||
},
|
||||
},
|
||||
# Match original grouping: Folder + IRS_Group
|
||||
# Match original grouping: Folder + IRS_Group (now mapped to provider name + TIN)
|
||||
grouping={
|
||||
"strategy_type": "multi_identifier",
|
||||
"identifiers": ["_folder", "_irs_group"],
|
||||
@@ -1905,12 +1898,22 @@ class ParentChildEngine:
|
||||
src = spec.get("source")
|
||||
mode = spec.get("mode", "copy_clean")
|
||||
cleaner = spec.get("cleaner", "raw")
|
||||
fallbacks = spec.get("fallback_sources", [])
|
||||
|
||||
# Try primary source, then fallbacks
|
||||
if not src or src not in df.columns:
|
||||
df[internal_col] = pd.Series(
|
||||
[pd.NA] * len(df), index=df.index, dtype="string"
|
||||
)
|
||||
continue
|
||||
resolved = None
|
||||
for fb in fallbacks:
|
||||
if fb in df.columns:
|
||||
resolved = fb
|
||||
break
|
||||
if resolved:
|
||||
src = resolved
|
||||
else:
|
||||
df[internal_col] = pd.Series(
|
||||
[pd.NA] * len(df), index=df.index, dtype="string"
|
||||
)
|
||||
continue
|
||||
|
||||
if mode == "copy_clean":
|
||||
if cleaner == "upper_clean":
|
||||
|
||||
Reference in New Issue
Block a user