diff --git a/src/parent_child/column_mapper.py b/src/parent_child/column_mapper.py index c77abe0..3d6d4c5 100644 --- a/src/parent_child/column_mapper.py +++ b/src/parent_child/column_mapper.py @@ -45,6 +45,9 @@ class ColumnMapper: r"^insurance.*name$", r"^carrier.*name$", ], + "AARETE_DERIVED_PROVIDER_NAME": [ + r"^aarete_derived_provider_name$", + ], "PROV_GROUP_NAME_FULL": [ r"^prov.*group.*name.*full$", r"^irs_name", @@ -120,6 +123,7 @@ class ColumnMapper: "PROV_GROUP_TIN", "PROV_GROUP_NPI", "AARETE_DERIVED_AMENDMENT_NUM", + "AARETE_DERIVED_PROVIDER_NAME", ] def __init__(self, df: pd.DataFrame): diff --git a/src/parent_child/pipeline.py b/src/parent_child/pipeline.py index 82c1fa2..d5ad7b6 100644 --- a/src/parent_child/pipeline.py +++ b/src/parent_child/pipeline.py @@ -1151,9 +1151,16 @@ def parent_child_mapping( immediate_orphans = cleaned_df[cleaned_df["grouping_tier"] == 0].copy() immediate_orphans["parent"] = False immediate_orphans["assigned_parent_rank"] = ASSIGNMENT_NO_PARENT - immediate_orphans["assignment_reasoning"] = immediate_orphans[ - "assignment_reasoning" - ].fillna("Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null") + if "assignment_reasoning" not in immediate_orphans.columns: + immediate_orphans["assignment_reasoning"] = ( + "Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null" + ) + else: + immediate_orphans["assignment_reasoning"] = immediate_orphans[ + "assignment_reasoning" + ].fillna( + "Immediate orphan: all grouping columns (TIN, NPI, PROV_NAME) are null" + ) workable_df = cleaned_df[cleaned_df["grouping_tier"] > 0].copy() logging.info( diff --git a/src/parent_child/preprocessing.py b/src/parent_child/preprocessing.py index 9730df1..558ed3b 100644 --- a/src/parent_child/preprocessing.py +++ b/src/parent_child/preprocessing.py @@ -137,6 +137,93 @@ def clean_payer_name(text, hit_words): return text +def lightweight_clean_provider_name( + df, + provider_col="AARETE_DERIVED_PROVIDER_NAME", + group_col="PROV_GROUP_NAME_FULL_cleaned", +): + """ + Lightweight cleaning for the Aarete-derived provider name. + Since the upstream pipeline already standardized/canonicalized the name, + this only handles minor normalization: + 1. Lowercase + 2. Remove non-alphanumeric characters (except spaces) + 3. Remove legal suffixes (llc, inc, ltd, corp, etc.) + 4. Remove the word 'and' + 5. Normalize plural/abbreviation variants + + Parameters: + ----------- + df : pandas.DataFrame + The dataframe containing the provider data + provider_col : str + Name of the aarete-derived provider name column + group_col : str + Name of the new cleaned group column to create + + Returns: + -------- + pandas.DataFrame + The dataframe with the cleaned group column added + """ + + def clean_derived_name(name): + if pd.isna(name) or not str(name).strip(): + return "" + + cleaned = str(name).strip() + + # Remove non-alphanumeric characters except spaces + cleaned = re.sub(r"[^a-zA-Z0-9\s]", "", cleaned) + + # Lowercase + cleaned = cleaned.lower() + + # Remove standalone 's' left over from possessives (e.g., "children s" -> "childrens") + cleaned = re.sub(r"\b(\w+)\s+s\b", r"\1s", cleaned) + + # Remove legal suffixes (as complete words) + suffix_patterns = [ + "dba", + "inc", + "ltd", + "corp", + "corporation", + "incorporated", + "lp", + "llc", + "llp", + "pty", + ] + for pattern in suffix_patterns: + cleaned = re.sub(r"\b" + pattern + r"\b", "", cleaned) + + # Remove 'and' as a standalone word + cleaned = re.sub(r"\band\b", "", cleaned) + + # Normalize common abbreviation/plural variants + cleaned = re.sub(r"\bassoc\b", "associates", cleaned) + cleaned = re.sub(r"\bcenters\b", "center", cleaned) + cleaned = re.sub(r"\bsolutions\b", "solution", cleaned) + cleaned = re.sub(r"\bsystems\b", "system", cleaned) + cleaned = re.sub(r"\bplans\b", "plan", cleaned) + cleaned = re.sub(r"\bservices\b", "service", cleaned) + cleaned = re.sub(r"\bcasemanagement\b", "case management", cleaned) + + # Remove extra whitespace + cleaned = " ".join(cleaned.split()) + + return cleaned.strip() + + df[group_col] = df[provider_col].apply(clean_derived_name) + + # Create empty DBA_Name column for compatibility with downstream code + if "DBA_Name" not in df.columns: + df["DBA_Name"] = "" + + return df + + def create_group_column( df, provider_col="PROV_GROUP_NAME_FULL", @@ -144,13 +231,16 @@ def create_group_column( dba_col="DBA_Name", ): """ - Creates a Group column from the provider column by: + Creates a Group column from the raw provider column by: 1. Removing non-alphanumeric characters (except spaces) 2. Converting to lowercase 3. Removing anything after 'llc', 'inc', 'dba', or 'd/b/a' (only as complete words) Also creates a DBA_Name column that extracts text after 'dba' or 'd/b/a' + NOTE: This is the legacy/fallback path. When AARETE_DERIVED_PROVIDER_NAME is + available, lightweight_clean_provider_name() is used instead. + Parameters: ----------- df : pandas.DataFrame @@ -755,10 +845,24 @@ def parent_child_preprocessing( lambda x: clean_payer_name(str(x), ["inc", "llc", "dba"]) ) - # Use create_group_column for provider name cleaning (from working code) - one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col) - one_to_one_df = update_group(one_to_one_df) - one_to_one_df = standardize_provider_groups(one_to_one_df) + # Use AARETE_DERIVED_PROVIDER_NAME if available (already cleaned/standardized upstream) + # Otherwise fall back to raw PROV_GROUP_NAME_FULL with full cleaning + if "AARETE_DERIVED_PROVIDER_NAME" in one_to_one_df.columns: + logging.info( + "Using AARETE_DERIVED_PROVIDER_NAME for provider grouping (lightweight cleaning)" + ) + one_to_one_df = lightweight_clean_provider_name( + one_to_one_df, + provider_col="AARETE_DERIVED_PROVIDER_NAME", + ) + one_to_one_df = standardize_provider_groups(one_to_one_df) + else: + logging.info( + "AARETE_DERIVED_PROVIDER_NAME not found, falling back to raw provider name cleaning" + ) + one_to_one_df = create_group_column(one_to_one_df, provider_col=prov_group_col) + one_to_one_df = update_group(one_to_one_df) + one_to_one_df = standardize_provider_groups(one_to_one_df) # Determine which fields to consolidate based on available columns fields_to_consolidate = [] diff --git a/src/parent_child/qc.py b/src/parent_child/qc.py index 3bb2f99..929c302 100644 --- a/src/parent_child/qc.py +++ b/src/parent_child/qc.py @@ -1158,7 +1158,10 @@ class ChildAssigner: ] if not date_valid_cand: - continue + # Fallback: use all candidates when date filtering eliminates + # everyone — the child may predate the parent (e.g., an older + # amendment linked to a newer base agreement) + date_valid_cand = cand # Unique parent if pass3.get("unique_parent") and len(date_valid_cand) == 1: