Merged in bugfix/PC_health_partners (pull request #921)
Bugfix/PC health partners * Parent_child_fix * black format fix Approved-by: Katon Minhas
This commit is contained in:
Rahul Ailaboina
committed by
Katon Minhas
Katon Minhas
parent
1dd81c4bcc
commit
f53fd3273c
@@ -45,6 +45,9 @@ class ColumnMapper:
|
||||
r"^insurance.*name$",
|
||||
r"^carrier.*name$",
|
||||
],
|
||||
"AARETE_DERIVED_PROVIDER_NAME": [
|
||||
r"^aarete_derived_provider_name$",
|
||||
],
|
||||
"PROV_GROUP_NAME_FULL": [
|
||||
r"^prov.*group.*name.*full$",
|
||||
r"^irs_name",
|
||||
@@ -120,6 +123,7 @@ class ColumnMapper:
|
||||
"PROV_GROUP_TIN",
|
||||
"PROV_GROUP_NPI",
|
||||
"AARETE_DERIVED_AMENDMENT_NUM",
|
||||
"AARETE_DERIVED_PROVIDER_NAME",
|
||||
]
|
||||
|
||||
def __init__(self, df: pd.DataFrame):
|
||||
|
||||
@@ -1151,9 +1151,16 @@ def parent_child_mapping(
|
||||
immediate_orphans = cleaned_df[cleaned_df["grouping_tier"] == 0].copy()
|
||||
immediate_orphans["parent"] = False
|
||||
immediate_orphans["assigned_parent_rank"] = ASSIGNMENT_NO_PARENT
|
||||
immediate_orphans["assignment_reasoning"] = immediate_orphans[
|
||||
"assignment_reasoning"
|
||||
].fillna("Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null")
|
||||
if "assignment_reasoning" not in immediate_orphans.columns:
|
||||
immediate_orphans["assignment_reasoning"] = (
|
||||
"Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null"
|
||||
)
|
||||
else:
|
||||
immediate_orphans["assignment_reasoning"] = immediate_orphans[
|
||||
"assignment_reasoning"
|
||||
].fillna(
|
||||
"Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null"
|
||||
)
|
||||
|
||||
workable_df = cleaned_df[cleaned_df["grouping_tier"] > 0].copy()
|
||||
logging.info(
|
||||
|
||||
@@ -137,6 +137,93 @@ def clean_payer_name(text, hit_words):
|
||||
return text
|
||||
|
||||
|
||||
def lightweight_clean_provider_name(
|
||||
df,
|
||||
provider_col="AARETE_DERIVED_PROVIDER_NAME",
|
||||
group_col="PROV_GROUP_NAME_FULL_cleaned",
|
||||
):
|
||||
"""
|
||||
Lightweight cleaning for the Aarete-derived provider name.
|
||||
Since the upstream pipeline already standardized/canonicalized the name,
|
||||
this only handles minor normalization:
|
||||
1. Lowercase
|
||||
2. Remove non-alphanumeric characters (except spaces)
|
||||
3. Remove legal suffixes (llc, inc, ltd, corp, etc.)
|
||||
4. Remove the word 'and'
|
||||
5. Normalize plural/abbreviation variants
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df : pandas.DataFrame
|
||||
The dataframe containing the provider data
|
||||
provider_col : str
|
||||
Name of the aarete-derived provider name column
|
||||
group_col : str
|
||||
Name of the new cleaned group column to create
|
||||
|
||||
Returns:
|
||||
--------
|
||||
pandas.DataFrame
|
||||
The dataframe with the cleaned group column added
|
||||
"""
|
||||
|
||||
def clean_derived_name(name):
|
||||
if pd.isna(name) or not str(name).strip():
|
||||
return ""
|
||||
|
||||
cleaned = str(name).strip()
|
||||
|
||||
# Remove non-alphanumeric characters except spaces
|
||||
cleaned = re.sub(r"[^a-zA-Z0-9\s]", "", cleaned)
|
||||
|
||||
# Lowercase
|
||||
cleaned = cleaned.lower()
|
||||
|
||||
# Remove standalone 's' left over from possessives (e.g., "children s" -> "childrens")
|
||||
cleaned = re.sub(r"\b(\w+)\s+s\b", r"\1s", cleaned)
|
||||
|
||||
# Remove legal suffixes (as complete words)
|
||||
suffix_patterns = [
|
||||
"dba",
|
||||
"inc",
|
||||
"ltd",
|
||||
"corp",
|
||||
"corporation",
|
||||
"incorporated",
|
||||
"lp",
|
||||
"llc",
|
||||
"llp",
|
||||
"pty",
|
||||
]
|
||||
for pattern in suffix_patterns:
|
||||
cleaned = re.sub(r"\b" + pattern + r"\b", "", cleaned)
|
||||
|
||||
# Remove 'and' as a standalone word
|
||||
cleaned = re.sub(r"\band\b", "", cleaned)
|
||||
|
||||
# Normalize common abbreviation/plural variants
|
||||
cleaned = re.sub(r"\bassoc\b", "associates", cleaned)
|
||||
cleaned = re.sub(r"\bcenters\b", "center", cleaned)
|
||||
cleaned = re.sub(r"\bsolutions\b", "solution", cleaned)
|
||||
cleaned = re.sub(r"\bsystems\b", "system", cleaned)
|
||||
cleaned = re.sub(r"\bplans\b", "plan", cleaned)
|
||||
cleaned = re.sub(r"\bservices\b", "service", cleaned)
|
||||
cleaned = re.sub(r"\bcasemanagement\b", "case management", cleaned)
|
||||
|
||||
# Remove extra whitespace
|
||||
cleaned = " ".join(cleaned.split())
|
||||
|
||||
return cleaned.strip()
|
||||
|
||||
df[group_col] = df[provider_col].apply(clean_derived_name)
|
||||
|
||||
# Create empty DBA_Name column for compatibility with downstream code
|
||||
if "DBA_Name" not in df.columns:
|
||||
df["DBA_Name"] = ""
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def create_group_column(
|
||||
df,
|
||||
provider_col="PROV_GROUP_NAME_FULL",
|
||||
@@ -144,13 +231,16 @@ def create_group_column(
|
||||
dba_col="DBA_Name",
|
||||
):
|
||||
"""
|
||||
Creates a Group column from the provider column by:
|
||||
Creates a Group column from the raw provider column by:
|
||||
1. Removing non-alphanumeric characters (except spaces)
|
||||
2. Converting to lowercase
|
||||
3. Removing anything after 'llc', 'inc', 'dba', or 'd/b/a' (only as complete words)
|
||||
|
||||
Also creates a DBA_Name column that extracts text after 'dba' or 'd/b/a'
|
||||
|
||||
NOTE: This is the legacy/fallback path. When AARETE_DERIVED_PROVIDER_NAME is
|
||||
available, lightweight_clean_provider_name() is used instead.
|
||||
|
||||
Parameters:
|
||||
-----------
|
||||
df : pandas.DataFrame
|
||||
@@ -755,10 +845,24 @@ def parent_child_preprocessing(
|
||||
lambda x: clean_payer_name(str(x), ["inc", "llc", "dba"])
|
||||
)
|
||||
|
||||
# Use create_group_column for provider name cleaning (from working code)
|
||||
one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col)
|
||||
one_to_one_df = update_group(one_to_one_df)
|
||||
one_to_one_df = standardize_provider_groups(one_to_one_df)
|
||||
# Use AARETE_DERIVED_PROVIDER_NAME if available (already cleaned/standardized upstream)
|
||||
# Otherwise fall back to raw PROV_GROUP_NAME_FULL with full cleaning
|
||||
if "AARETE_DERIVED_PROVIDER_NAME" in one_to_one_df.columns:
|
||||
logging.info(
|
||||
"Using AARETE_DERIVED_PROVIDER_NAME for provider grouping (lightweight cleaning)"
|
||||
)
|
||||
one_to_one_df = lightweight_clean_provider_name(
|
||||
one_to_one_df,
|
||||
provider_col="AARETE_DERIVED_PROVIDER_NAME",
|
||||
)
|
||||
one_to_one_df = standardize_provider_groups(one_to_one_df)
|
||||
else:
|
||||
logging.info(
|
||||
"AARETE_DERIVED_PROVIDER_NAME not found, falling back to raw provider name cleaning"
|
||||
)
|
||||
one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col)
|
||||
one_to_one_df = update_group(one_to_one_df)
|
||||
one_to_one_df = standardize_provider_groups(one_to_one_df)
|
||||
|
||||
# Determine which fields to consolidate based on available columns
|
||||
fields_to_consolidate = []
|
||||
|
||||
@@ -1158,7 +1158,10 @@ class ChildAssigner:
|
||||
]
|
||||
|
||||
if not date_valid_cand:
|
||||
continue
|
||||
# Fallback: use all candidates when date filtering eliminates
|
||||
# everyone — the child may predate the parent (e.g., an older
|
||||
# amendment linked to a newer base agreement)
|
||||
date_valid_cand = cand
|
||||
|
||||
# Unique parent
|
||||
if pass3.get("unique_parent") and len(date_valid_cand) == 1:
|
||||
|
||||
Reference in New Issue
Block a user