Merged in feature/DAIP2-1803-add-aarete-derived-provider-name (pull request #884)
Feature/DAIP2-1803 add aarete derived provider name * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * state logic added in clustering * removed print statements * pull request updates * black format fix * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * added scalability feature and optimization * black format * Pull Request Changes * generalised funcs from aarete derived payer name * updated remove states and updated docstring for build similarity matrix * derived payer name functionality added * config and main file changes * sync dev into ADD AARETE DERIVED PROVIDER GROUP NAME FULL * updated derived_provider name * pipeline fixes * Merged DEV into feature/DAIP2-1803-add-aarete-derived-provider-name * config changes * field name updated to AARETE_DERIVED_PROVIDER_NAME * Merged DEV into feature/DAIP2-1803-add-aarete-derived-provider-name * black format fix * Fix docstring and add empty column when not found * Rename test file and add missing column test case - Renamed test_derived_payer_name.py to test_derived_names.py since file now covers both payer and provider name functionality - Added test_derived_provider_names_column_missing to verify empty column is added when AARETE_DERIVED_PROVIDER_NAME is not present * Remove trailing whitespace * Merge remote-tracking branch 'origin/DEV' into feature/DAIP2-1803-add-aarete-derived-provider-name Approved-by: Siddhant Medar
This commit is contained in:
VenkataKrishna Reddy Avula
committed by
Siddhant Medar
Siddhant Medar
parent
a417540ee9
commit
a4c5e80161
@@ -413,3 +413,17 @@ RUN_DASHBOARD_POSTPROCESSING = get_arg_value("run_dashboard", "False") == "True"
|
||||
############## QC/QA VALIDATION SETTINGS ##############
|
||||
# Enable QC/QA validation in pipeline
|
||||
ENABLE_QC_QA = get_arg_value("enable_qc_qa", "True") == "True"
|
||||
|
||||
############## ADD_AARETE_DERIVED_PAYER_NAME SETTINGS ############
|
||||
# STATE_FLAG controls whether payer names are treated as state-specific.
|
||||
#
|
||||
# True → Preserve state distinctions.
|
||||
# Example: "Molina Healthcare of Texas" and
|
||||
# "Molina Healthcare of Utah" remain separate.
|
||||
#
|
||||
# False → Collapse regional state suffixes into national brand.
|
||||
# Example: "Molina Healthcare of Texas" and
|
||||
# "Molina Healthcare of Utah" both become "Molina Healthcare".
|
||||
# Leading state-based brand names (e.g., "Oklahoma Complete Health")
|
||||
# are preserved.
|
||||
STATE_FLAG = True
|
||||
|
||||
@@ -38,6 +38,7 @@ FIELD_FORMAT_MAPPING = {
|
||||
"PROV_GROUP_TIN": "list[str]",
|
||||
"PROV_GROUP_NPI": "list[str]",
|
||||
"PROV_GROUP_NAME_FULL": "list[str]",
|
||||
"AARETE_DERIVED_PROVIDER_NAME": "str",
|
||||
"PROV_OTHER_TIN": "list[str]",
|
||||
"PROV_OTHER_NPI": "list[str]",
|
||||
"PROV_OTHER_NAME_FULL": "list[str]",
|
||||
|
||||
@@ -224,19 +224,29 @@ def main(testing=False, test_params={}):
|
||||
FINAL_RESULT_DF_CC = pd.DataFrame()
|
||||
FINAL_RESULT_DF_DASHBOARD = pd.DataFrame()
|
||||
|
||||
# Add derived payer name and reorder columns for both CC and dashboard results
|
||||
FINAL_RESULT_DF_CC = one_to_one_funcs.add_aarete_derived_payer_name(
|
||||
FINAL_RESULT_DF_CC, state_flag=True
|
||||
)
|
||||
if not FINAL_RESULT_DF_CC.empty:
|
||||
FINAL_RESULT_DF_CC = one_to_one_funcs.add_aarete_derived_payer_name(
|
||||
FINAL_RESULT_DF_CC, config.STATE_FLAG
|
||||
) # add aarete derived payer name to cc version
|
||||
FINAL_RESULT_DF_CC = one_to_one_funcs.add_aarete_derived_provider_name(
|
||||
FINAL_RESULT_DF_CC, config.STATE_FLAG
|
||||
) # add aarete derived provider name to cc version
|
||||
if not FINAL_RESULT_DF_DASHBOARD.empty:
|
||||
FINAL_RESULT_DF_DASHBOARD = one_to_one_funcs.add_aarete_derived_payer_name(
|
||||
FINAL_RESULT_DF_DASHBOARD, config.STATE_FLAG
|
||||
) # add aarete derived payer name to dashboard version
|
||||
FINAL_RESULT_DF_DASHBOARD = (
|
||||
one_to_one_funcs.add_aarete_derived_provider_name(
|
||||
FINAL_RESULT_DF_DASHBOARD, config.STATE_FLAG
|
||||
)
|
||||
) # add aarete derived provider name to dashboard version
|
||||
|
||||
FINAL_RESULT_DF_CC = postprocessing_funcs.reorder_columns(
|
||||
FINAL_RESULT_DF_CC, FIELD_FORMAT_MAPPING
|
||||
)
|
||||
FINAL_RESULT_DF_DASHBOARD = one_to_one_funcs.add_aarete_derived_payer_name(
|
||||
FINAL_RESULT_DF_DASHBOARD, state_flag=True
|
||||
)
|
||||
) # Reorder columns in CC version only
|
||||
FINAL_RESULT_DF_DASHBOARD = postprocessing_funcs.reorder_columns(
|
||||
FINAL_RESULT_DF_DASHBOARD, FIELD_FORMAT_MAPPING
|
||||
)
|
||||
) # Reorder columns in dashboard version only
|
||||
|
||||
if len(error_results) > 0:
|
||||
ERROR_RESULT_DF = pd.concat(error_results, ignore_index=True)
|
||||
|
||||
@@ -1096,7 +1096,7 @@ def prompt_aarete_derived_payer_name(cluster_rows: list[dict], state_flag: bool)
|
||||
|
||||
# If only one record in cluster, avoid LLM call
|
||||
if len(cluster_rows) == 1:
|
||||
base_name = cluster_rows[0].get("base_payer_name", "UNKNOWN")
|
||||
base_name = cluster_rows[0].get("base_name", "UNKNOWN")
|
||||
return str(base_name).upper()
|
||||
|
||||
prompt, _parser = prompt_templates.AARETE_DERIVED_PAYER_NAME(
|
||||
@@ -1114,7 +1114,11 @@ def prompt_aarete_derived_payer_name(cluster_rows: list[dict], state_flag: bool)
|
||||
logging.debug(f"LLM Response for Aarete Derived Payer Name: {llm_answer_raw}")
|
||||
try:
|
||||
llm_answer_final = _parser(llm_answer_raw)
|
||||
if llm_answer_final and not string_utils.is_empty(llm_answer_final):
|
||||
if (
|
||||
llm_answer_final
|
||||
and "AARETE_DERIVED_PAYER_NAME" in llm_answer_final
|
||||
and not string_utils.is_empty(llm_answer_final["AARETE_DERIVED_PAYER_NAME"])
|
||||
):
|
||||
derived_name = (
|
||||
llm_answer_final["AARETE_DERIVED_PAYER_NAME"]
|
||||
.replace("```", "")
|
||||
@@ -1124,12 +1128,67 @@ def prompt_aarete_derived_payer_name(cluster_rows: list[dict], state_flag: bool)
|
||||
)
|
||||
return derived_name
|
||||
else:
|
||||
# If LLM returns N/A or empty, fallback to the programmatic base_payer_name
|
||||
fallback = cluster_rows[0].get("base_payer_name", "UNKNOWN")
|
||||
# If LLM returns N/A or empty, fallback to the programmatic base_name
|
||||
fallback = cluster_rows[0].get("base_name", "UNKNOWN")
|
||||
return str(fallback).upper()
|
||||
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
f"Error parsing LLM response for Aarete Derived Payer Name: {str(e)}"
|
||||
)
|
||||
return str(cluster_rows[0].get("base_payer_name", "UNKNOWN")).upper()
|
||||
return str(cluster_rows[0].get("base_name", "UNKNOWN")).upper()
|
||||
|
||||
|
||||
def prompt_aarete_derived_provider_name(
|
||||
cluster_rows: list[dict], state_flag: bool
|
||||
) -> str:
|
||||
"""
|
||||
Select canonical provider name using LLM.
|
||||
"""
|
||||
|
||||
# Skip LLM if only one record
|
||||
if len(cluster_rows) == 1:
|
||||
base_name = cluster_rows[0].get("base_name", "UNKNOWN")
|
||||
return str(base_name).upper()
|
||||
|
||||
prompt, _parser = prompt_templates.AARETE_DERIVED_PROVIDER_NAME(
|
||||
cluster_rows, state_flag
|
||||
)
|
||||
logging.debug(f"Aarete Derived Provider Name Prompt: {prompt}")
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
"all_files",
|
||||
cache=True,
|
||||
instruction=prompt_templates.AARETE_DERIVED_PROVIDER_NAME_INSTRUCTION(),
|
||||
usage_label="AARETE_DERIVED_PROVIDER_NAME",
|
||||
)
|
||||
logging.debug(f"LLM Response for Aarete Derived Provider Name: {llm_answer_raw}")
|
||||
try:
|
||||
llm_answer_final = _parser(llm_answer_raw)
|
||||
if (
|
||||
llm_answer_final
|
||||
and "AARETE_DERIVED_PROVIDER_NAME" in llm_answer_final
|
||||
and not string_utils.is_empty(
|
||||
llm_answer_final["AARETE_DERIVED_PROVIDER_NAME"]
|
||||
)
|
||||
):
|
||||
derived_name = (
|
||||
llm_answer_final["AARETE_DERIVED_PROVIDER_NAME"]
|
||||
.replace("```", "")
|
||||
.replace("**", "")
|
||||
.strip()
|
||||
.upper()
|
||||
)
|
||||
return derived_name
|
||||
|
||||
# Explicit fallback
|
||||
fallback = cluster_rows[0].get("base_name", "UNKNOWN")
|
||||
return str(fallback).upper()
|
||||
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
f"Error parsing LLM response for Aarete Derived Provider Name: {str(e)}"
|
||||
)
|
||||
fallback = cluster_rows[0].get("base_name", "UNKNOWN")
|
||||
return str(fallback).upper()
|
||||
|
||||
@@ -424,11 +424,6 @@ def add_aarete_derived_payer_name(df: pd.DataFrame, state_flag: bool) -> pd.Data
|
||||
Returns:
|
||||
DataFrame with added AARETE_DERIVED_PAYER_NAME column.
|
||||
"""
|
||||
|
||||
if df is None or df.empty:
|
||||
logging.warning("Input DataFrame is empty. Returning empty DataFrame.")
|
||||
return df
|
||||
|
||||
df_payer = df.copy()
|
||||
# Check if PAYER_NAME column exists in the DataFrame, if not add an empty AARETE_DERIVED_PAYER_NAME column and return the DataFrame
|
||||
if "PAYER_NAME" not in df_payer.columns:
|
||||
@@ -469,7 +464,7 @@ def add_aarete_derived_payer_name(df: pd.DataFrame, state_flag: bool) -> pd.Data
|
||||
)
|
||||
|
||||
for record in cluster:
|
||||
mapping[record["original_payer_name"]] = derived_payer_name
|
||||
mapping[record["original_name"]] = derived_payer_name
|
||||
|
||||
df_payer["AARETE_DERIVED_PAYER_NAME"] = df_payer["PAYER_NAME"].map(mapping)
|
||||
|
||||
@@ -493,7 +488,7 @@ def standardize_unique_names(
|
||||
|
||||
state = string_utils.detect_state(clean)
|
||||
|
||||
legal, dba = string_utils.extract_dba_payer_name(clean)
|
||||
legal, dba = string_utils.extract_dba_name(clean)
|
||||
base = dba if dba else clean
|
||||
legal = legal if legal else clean
|
||||
|
||||
@@ -510,12 +505,12 @@ def standardize_unique_names(
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"original_payer_name": name,
|
||||
"base_payer_name": base,
|
||||
"original_name": name,
|
||||
"base_name": base,
|
||||
"signature": signature if signature else (),
|
||||
"has_dba_payer_name": dba is not None,
|
||||
"dba_payer_name": dba,
|
||||
"legal_payer_name": legal,
|
||||
"has_dba_name": dba is not None,
|
||||
"dba_name": dba,
|
||||
"legal_name": legal,
|
||||
"state": state,
|
||||
"block_key": block_key,
|
||||
}
|
||||
@@ -526,20 +521,21 @@ def standardize_unique_names(
|
||||
|
||||
def build_similarity_matrix(rows: list[dict]) -> np.ndarray:
|
||||
"""
|
||||
Compute a pairwise similarity matrix for base payer names.
|
||||
Create a pairwise similarity matrix for standardized entity names.
|
||||
|
||||
This function calculates fuzzy string similarity scores between
|
||||
all pairs of 'base_payer_name' values in the input rows using
|
||||
multiple fuzzy matching strategies. The highest score is used
|
||||
as the final similarity value.
|
||||
Computes fuzzy similarity scores between each pair of rows using
|
||||
both base_name and legal_name fields. The maximum score from multiple
|
||||
fuzzy comparisons is used as the final similarity value.
|
||||
|
||||
Args:
|
||||
rows (list[dict]): A list of dictionaries, each containing
|
||||
a 'base_payer_name' key.
|
||||
rows : list[dict]
|
||||
List of standardized entity dictionaries containing
|
||||
"base_name" and "legal_name".
|
||||
|
||||
Returns:
|
||||
np.ndarray: A square (n x n) matrix of similarity scores
|
||||
ranging from 0 to 100.
|
||||
np.ndarray
|
||||
An (n x n) symmetric matrix of similarity scores (0–100),
|
||||
where n is the number of rows.
|
||||
"""
|
||||
# Number of payer name records
|
||||
n = len(rows)
|
||||
@@ -556,18 +552,12 @@ def build_similarity_matrix(rows: list[dict]) -> np.ndarray:
|
||||
else:
|
||||
# Compute similarity using multiple fuzzy matching methods and different combinations
|
||||
score = max(
|
||||
fuzz.ratio(rows[i]["base_payer_name"], rows[j]["base_payer_name"]),
|
||||
fuzz.token_set_ratio(
|
||||
rows[i]["base_payer_name"], rows[j]["base_payer_name"]
|
||||
),
|
||||
fuzz.ratio(rows[i]["legal_payer_name"], rows[j]["base_payer_name"]),
|
||||
fuzz.token_set_ratio(
|
||||
rows[i]["legal_payer_name"], rows[j]["base_payer_name"]
|
||||
),
|
||||
fuzz.ratio(rows[i]["base_payer_name"], rows[j]["legal_payer_name"]),
|
||||
fuzz.token_set_ratio(
|
||||
rows[i]["base_payer_name"], rows[j]["legal_payer_name"]
|
||||
),
|
||||
fuzz.ratio(rows[i]["base_name"], rows[j]["base_name"]),
|
||||
fuzz.token_set_ratio(rows[i]["base_name"], rows[j]["base_name"]),
|
||||
fuzz.ratio(rows[i]["legal_name"], rows[j]["base_name"]),
|
||||
fuzz.token_set_ratio(rows[i]["legal_name"], rows[j]["base_name"]),
|
||||
fuzz.ratio(rows[i]["base_name"], rows[j]["legal_name"]),
|
||||
fuzz.token_set_ratio(rows[i]["base_name"], rows[j]["legal_name"]),
|
||||
)
|
||||
|
||||
# Fill both symmetric positions in the matrix
|
||||
@@ -683,22 +673,102 @@ def propagate_dba_to_legal(rows: list[dict]) -> list[dict]:
|
||||
|
||||
# First pass: build mapping
|
||||
for row in rows:
|
||||
legal = row.get("legal_payer_name")
|
||||
dba = row.get("dba_payer_name")
|
||||
legal = row.get("legal_name")
|
||||
dba = row.get("dba_name")
|
||||
|
||||
if legal and dba:
|
||||
legal_to_dba[legal] = dba
|
||||
|
||||
# Second pass: propagate
|
||||
for row in rows:
|
||||
legal = row.get("legal_payer_name")
|
||||
legal = row.get("legal_name")
|
||||
|
||||
if legal in legal_to_dba:
|
||||
new_base = legal_to_dba[legal]
|
||||
row["base_payer_name"] = new_base
|
||||
row["base_name"] = new_base
|
||||
|
||||
# Update block key
|
||||
tokens = new_base.split()
|
||||
row["block_key"] = tokens[0] if tokens else ""
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def add_aarete_derived_provider_name(
|
||||
df: pd.DataFrame, state_flag: bool
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Normalize extracted AARETE_DERIVED_PROVIDER_NAME values into canonical form.
|
||||
"""
|
||||
df_copy = df.copy()
|
||||
|
||||
# Check if AARETE_DERIVED_PROVIDER_NAME column exists, if not add empty column and return
|
||||
if "AARETE_DERIVED_PROVIDER_NAME" not in df_copy.columns:
|
||||
logging.warning(
|
||||
"AARETE_DERIVED_PROVIDER_NAME column not found in DataFrame. Adding empty column."
|
||||
)
|
||||
df_copy["AARETE_DERIVED_PROVIDER_NAME"] = ""
|
||||
return df_copy
|
||||
|
||||
# 1. Extract unique provider group names
|
||||
raw_values = df_copy["AARETE_DERIVED_PROVIDER_NAME"].dropna().tolist()
|
||||
|
||||
flattened_names = []
|
||||
|
||||
for value in raw_values:
|
||||
if isinstance(value, list):
|
||||
flattened_names.extend(value)
|
||||
else:
|
||||
flattened_names.append(value)
|
||||
|
||||
# Remove empties
|
||||
unique_names = list(
|
||||
{name for name in flattened_names if not string_utils.is_empty(name)}
|
||||
)
|
||||
|
||||
if not unique_names:
|
||||
df_copy["AARETE_DERIVED_PROVIDER_NAME"] = ""
|
||||
return df_copy
|
||||
|
||||
# 2. Standardize
|
||||
standardized_rows = standardize_unique_names(unique_names, state_flag)
|
||||
|
||||
# 3. DBA propagation (kept separate in case rules diverge later)
|
||||
standardized_rows = propagate_dba_to_legal(standardized_rows)
|
||||
|
||||
# 4. Cluster with blocking
|
||||
clusters = cluster_with_blocking(
|
||||
standardized_rows,
|
||||
state_flag=state_flag,
|
||||
threshold=90,
|
||||
)
|
||||
|
||||
# 5. Build mapping
|
||||
mapping = {}
|
||||
|
||||
for cluster in clusters:
|
||||
derived_name = prompt_calls.prompt_aarete_derived_provider_name(
|
||||
cluster, state_flag
|
||||
)
|
||||
|
||||
for record in cluster:
|
||||
mapping[record["original_name"]] = derived_name
|
||||
|
||||
def map_value(value):
|
||||
if isinstance(value, list):
|
||||
derived_values = {mapping.get(v) for v in value if mapping.get(v)}
|
||||
|
||||
# If all collapse to same canonical, return single string
|
||||
if len(derived_values) == 1:
|
||||
return derived_values.pop()
|
||||
|
||||
# If multiple canonicals (rare), return list
|
||||
return list(derived_values)
|
||||
|
||||
return mapping.get(value)
|
||||
|
||||
df_copy["AARETE_DERIVED_PROVIDER_NAME"] = df_copy[
|
||||
"AARETE_DERIVED_PROVIDER_NAME"
|
||||
].apply(map_value)
|
||||
|
||||
return df_copy
|
||||
|
||||
@@ -399,7 +399,7 @@ def prompt_hsc_single_field(
|
||||
else "N/A"
|
||||
)
|
||||
)
|
||||
if field_value_str == "N/A":
|
||||
if string_utils.is_empty(field_value_str):
|
||||
field.field_type = "full_context"
|
||||
if field.field_name == "PAYER_NAME":
|
||||
field.prompt = (
|
||||
|
||||
@@ -268,6 +268,13 @@
|
||||
"prompt": "Extract the PROVIDER organization name(s) from the retrieved contract text.\n\nThe provider is the healthcare organization entering into the agreement (NOT the payer/insurance company like Highmark, Aetna, UnitedHealthcare, Blue Cross Blue Shield, etc.).\n\nGUIDELINES:\n1. Look for the provider name in phrases like:\n - 'This Agreement is between [PAYER] and [PROVIDER]'\n - 'Agreement between [PAYER] and [PROVIDER]'\n - 'by and between [PAYER] and [PROVIDER]'\n - '[PROVIDER] (hereinafter referred to as \"Provider\")'\n - 'Provider: [PROVIDER]'\n2. The provider name often appears in the opening paragraph or preamble of the contract.\n3. Return the EXACT provider name as it appears in the document, including legal suffixes (LLC, Inc., P.C., etc.).\n4. If the provider name includes 'd/b/a' (doing business as), return the FULL name including the d/b/a portion.\n5. If MULTIPLE provider organization names appear in the contract (e.g., multiple entities entering the agreement), return ALL of them as a JSON list.\n - Example: [\"Provider Group A, LLC\", \"Provider Group B, P.C.\", \"Provider Group C d/b/a City Medical\"]\n6. Do NOT return:\n - The payer/insurance company name\n - Individual physician names (unless that is the practice name)\n - Department names only\n - Addresses or other metadata\n7. If the provider name cannot be determined from the retrieved text, return 'N/A'.\n\nBriefly explain your reasoning. Return your answer as a valid JSON dictionary with the key \"PROVIDER_NAME\". If multiple provider names are found, the value should be a JSON list. If a single provider name is found, the value should be a string. Example: {\"PROVIDER_NAME\": [\"Provider Group A, LLC\", \"Provider Group B, P.C.\"]} or {\"PROVIDER_NAME\": \"Provider Group A, LLC\"} or {\"PROVIDER_NAME\": \"N/A\"}",
|
||||
"retrieval_question": "What is the provider organization name? Agreement between payer and provider. This agreement is made between. By and between. Hereinafter referred to as Provider. Provider name in contract preamble. Provider party to the agreement. Healthcare organization entering agreement."
|
||||
},
|
||||
{
|
||||
"field_name": "AARETE_DERIVED_PROVIDER_NAME",
|
||||
"relationship": "one_to_one",
|
||||
"field_type": "full_context",
|
||||
"prompt": "Extract the PROVIDER organization name from the retrieved contract text.\n\nThe provider is the healthcare organization entering into the agreement (NOT the payer/insurance company like Highmark, Aetna, UnitedHealthcare, Blue Cross Blue Shield, etc.).\n\nGUIDELINES:\n1. Look for the provider name in phrases like:\n - 'This Agreement is between [PAYER] and [PROVIDER]'\n - 'Agreement between [PAYER] and [PROVIDER]'\n - 'by and between [PAYER] and [PROVIDER]'\n - '[PROVIDER] (hereinafter referred to as \"Provider\")'\n - 'Provider: [PROVIDER]'\n2. The provider name often appears in the opening paragraph or preamble of the contract.\n3. Return the EXACT provider name as it appears in the document, including legal suffixes (LLC, Inc., P.C., etc.).\n4. If the provider name includes 'd/b/a' (doing business as), return the FULL name including the d/b/a portion.\n5. If multiple provider organization names appear in the contract (for example, if multiple entities are entering into the agreement), identify and return the single most appropriate provider name based on the overall contract context. Carefully analyze all provider names mentioned in the contract. If the extracted provider name contains the word 'and', split the name into separate entities and determine which one is the most relevant primary provider for the agreement. Always return only one provider name in the output. The final result must not contain the word 'and'.\n6. Do NOT return:\n - The payer/insurance company name\n - Individual physician names (unless that is the practice name)\n - Department names only\n - Addresses or other metadata\n7. If the provider name cannot be determined from the retrieved text, return 'N/A'.\n\nBriefly explain your reasoning.",
|
||||
"retrieval_question": "What is the provider organization name? Agreement between payer and provider. This agreement is made between. By and between. Hereinafter referred to as Provider. Provider name in contract preamble. Provider party to the agreement. Healthcare organization entering agreement."
|
||||
},
|
||||
{
|
||||
"field_name": "REIMB_PROV_TIN",
|
||||
"relationship": "one_to_n",
|
||||
|
||||
@@ -2438,9 +2438,7 @@ def AARETE_DERIVED_PAYER_NAME(
|
||||
variant_list = []
|
||||
for r in cluster_rows:
|
||||
# Using your preferred logic to show the LLM both states of the data
|
||||
variant = (
|
||||
f"- Original: {r['original_payer_name']} (Cleaned: {r['base_payer_name']})"
|
||||
)
|
||||
variant = f"- Original: {r['original_name']} (Cleaned: {r['base_name']})"
|
||||
variant_list.append(variant)
|
||||
|
||||
variants_string = "\n".join(variant_list)
|
||||
@@ -2451,3 +2449,63 @@ def AARETE_DERIVED_PAYER_NAME(
|
||||
{state_flag}
|
||||
"""
|
||||
return (prompt, _json_dict_parser)
|
||||
|
||||
|
||||
def AARETE_DERIVED_PROVIDER_NAME_INSTRUCTION():
|
||||
return f"""You are an expert in healthcare provider group name normalization.
|
||||
|
||||
Your goal is to take a list of similar provider group names and determine
|
||||
the single most accurate canonical provider group name.
|
||||
|
||||
The given variants refer to the same Provider Group entity.
|
||||
|
||||
TASK:
|
||||
1. Select or derive the most professional canonical name.
|
||||
2. If given STATE_FLAG is FALSE:
|
||||
- Collapse regional entities into the national brand (e.g., 'Molina Healthcare').
|
||||
- DO NOT remove the word 'State' if it is part of a proper noun (e.g., 'New York State Catholic Health Plan').
|
||||
3. If given STATE_FLAG is TRUE:
|
||||
- Maintain state distinctions (e.g., 'Molina Healthcare of Texas').
|
||||
4. Prefer the 'DBA' (Doing Business As) name if it is present.
|
||||
5. Cleaned name should be preferred over original name, but use your judgment to determine if the cleaned name is actually more accurate or if it has removed important context.
|
||||
6. Remove excessive noise like "a corporation of...", "an Illinois company", or "d/b/a" prefixes.
|
||||
7. Maintain essential legal identifiers only if they are part of the standard brand identity.
|
||||
8. Return only the cleaned canonical name without any explanation.
|
||||
|
||||
Example Variants Input:
|
||||
VARIANTS:
|
||||
- Texas Health Physicians Group
|
||||
- THPG
|
||||
Example Output:
|
||||
{{"AARETE_DERIVED_PROVIDER_NAME":"Texas Health Physicians Group"}}
|
||||
|
||||
[OUTPUT FORMAT]
|
||||
Return ONLY the JSON dictionary with key:
|
||||
"AARETE_DERIVED_PROVIDER_NAME"
|
||||
{JSON_DICT_FORMAT_INSTRUCTIONS}
|
||||
"""
|
||||
|
||||
|
||||
def AARETE_DERIVED_PROVIDER_NAME(
|
||||
cluster_rows: list[dict], state_flag: bool
|
||||
) -> Tuple[str, Callable[[str], dict]]:
|
||||
"""
|
||||
Construct LLM prompt for provider group canonical selection.
|
||||
"""
|
||||
|
||||
variant_list = []
|
||||
|
||||
for r in cluster_rows:
|
||||
variant = f"- Original: {r['original_name']} (Cleaned: {r['base_name']})"
|
||||
variant_list.append(variant)
|
||||
|
||||
variants_string = "\n".join(variant_list)
|
||||
|
||||
prompt = f"""
|
||||
VARIANTS:
|
||||
{variants_string}
|
||||
STATE_FLAG:
|
||||
{state_flag}
|
||||
"""
|
||||
|
||||
return (prompt, _json_dict_parser)
|
||||
|
||||
@@ -4,13 +4,14 @@ import numpy as np
|
||||
from unittest.mock import patch
|
||||
|
||||
from src.utils.string_utils import (
|
||||
extract_dba_name,
|
||||
normalize_text,
|
||||
extract_dba_payer_name,
|
||||
remove_legal_suffixes,
|
||||
remove_states,
|
||||
token_signature,
|
||||
)
|
||||
from src.pipelines.shared.extraction.one_to_one_funcs import (
|
||||
add_aarete_derived_provider_name,
|
||||
standardize_unique_names,
|
||||
add_aarete_derived_payer_name,
|
||||
cluster_from_similarity,
|
||||
@@ -30,7 +31,7 @@ class TestPayerNameTextUtilities(unittest.TestCase):
|
||||
def test_extract_dba_payer_name_from_text(self):
|
||||
text = "ABC Corp d/b/a XYZ Health"
|
||||
normalized = normalize_text(text)
|
||||
result = extract_dba_payer_name(normalized)
|
||||
result = extract_dba_name(normalized)
|
||||
self.assertEqual(result, ("abc corp", "xyz health"))
|
||||
|
||||
def test_remove_legal_suffixes_from_payer_name(self):
|
||||
@@ -51,11 +52,11 @@ class TestPayerNameStandardization(unittest.TestCase):
|
||||
rows = standardize_unique_names(names, state_flag=True)
|
||||
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["original_payer_name"], names[0])
|
||||
self.assertIn("base_payer_name", rows[0])
|
||||
self.assertEqual(rows[0]["original_name"], names[0])
|
||||
self.assertIn("base_name", rows[0])
|
||||
self.assertIn("signature", rows[0])
|
||||
self.assertIn("block_key", rows[0])
|
||||
self.assertFalse(rows[0]["has_dba_payer_name"])
|
||||
self.assertFalse(rows[0]["has_dba_name"])
|
||||
|
||||
|
||||
class TestRemoveStatesFunction(unittest.TestCase):
|
||||
@@ -78,11 +79,11 @@ class TestRemoveStatesFunction(unittest.TestCase):
|
||||
|
||||
def test_remove_of_state_pattern(self):
|
||||
self.assertEqual(
|
||||
remove_states("molina healthcare of utah"), "molina healthcare of"
|
||||
remove_states("molina healthcare of utah"), "molina healthcare"
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
remove_states("molina healthcare of texas"), "molina healthcare of"
|
||||
remove_states("molina healthcare of texas"), "molina healthcare"
|
||||
)
|
||||
|
||||
def test_no_state_present(self):
|
||||
@@ -94,26 +95,26 @@ class TestPropagateDbaToLegal(unittest.TestCase):
|
||||
def test_propagate_dba_to_matching_legal_entity(self):
|
||||
rows = [
|
||||
{
|
||||
"original_payer_name": "NY State Catholic Health Plan dba Fidelis Care New York",
|
||||
"base_payer_name": "fidelis care new york",
|
||||
"legal_payer_name": "new york state catholic health plan",
|
||||
"dba_payer_name": "fidelis care new york",
|
||||
"original_name": "NY State Catholic Health Plan dba Fidelis Care New York",
|
||||
"base_name": "fidelis care new york",
|
||||
"legal_name": "new york state catholic health plan",
|
||||
"dba_name": "fidelis care new york",
|
||||
"block_key": "fidelis",
|
||||
},
|
||||
{
|
||||
"original_payer_name": "New York State Catholic Health Plan",
|
||||
"base_payer_name": "new york state catholic health plan",
|
||||
"legal_payer_name": "new york state catholic health plan",
|
||||
"dba_payer_name": None,
|
||||
"original_name": "New York State Catholic Health Plan",
|
||||
"base_name": "new york state catholic health plan",
|
||||
"legal_name": "new york state catholic health plan",
|
||||
"dba_name": None,
|
||||
"block_key": "new",
|
||||
},
|
||||
]
|
||||
|
||||
updated = propagate_dba_to_legal(rows)
|
||||
|
||||
# Both should now share same base_payer_name
|
||||
self.assertEqual(updated[0]["base_payer_name"], "fidelis care new york")
|
||||
self.assertEqual(updated[1]["base_payer_name"], "fidelis care new york")
|
||||
# Both should now share same base_name
|
||||
self.assertEqual(updated[0]["base_name"], "fidelis care new york")
|
||||
self.assertEqual(updated[1]["base_name"], "fidelis care new york")
|
||||
|
||||
# Block key should also update
|
||||
self.assertEqual(updated[1]["block_key"], "fidelis")
|
||||
@@ -121,17 +122,17 @@ class TestPropagateDbaToLegal(unittest.TestCase):
|
||||
def test_no_propagation_when_no_dba_present(self):
|
||||
rows = [
|
||||
{
|
||||
"original_payer_name": "ODS Health Plan",
|
||||
"base_payer_name": "ods health plan",
|
||||
"legal_payer_name": "ods health plan",
|
||||
"dba_payer_name": None,
|
||||
"original_name": "ODS Health Plan",
|
||||
"base_name": "ods health plan",
|
||||
"legal_name": "ods health plan",
|
||||
"dba_name": None,
|
||||
"block_key": "ods",
|
||||
},
|
||||
{
|
||||
"original_payer_name": "Moda Health Plan",
|
||||
"base_payer_name": "moda health plan",
|
||||
"legal_payer_name": "moda health plan",
|
||||
"dba_payer_name": None,
|
||||
"original_name": "Moda Health Plan",
|
||||
"base_name": "moda health plan",
|
||||
"legal_name": "moda health plan",
|
||||
"dba_name": None,
|
||||
"block_key": "moda",
|
||||
},
|
||||
]
|
||||
@@ -139,8 +140,8 @@ class TestPropagateDbaToLegal(unittest.TestCase):
|
||||
updated = propagate_dba_to_legal(rows)
|
||||
|
||||
# Should remain unchanged
|
||||
self.assertEqual(updated[0]["base_payer_name"], "ods health plan")
|
||||
self.assertEqual(updated[1]["base_payer_name"], "moda health plan")
|
||||
self.assertEqual(updated[0]["base_name"], "ods health plan")
|
||||
self.assertEqual(updated[1]["base_name"], "moda health plan")
|
||||
|
||||
|
||||
class TestPayerNameClustering(unittest.TestCase):
|
||||
@@ -202,5 +203,49 @@ class TestPayerNameEndToEndPipeline(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestProvGroupEndToEnd(unittest.TestCase):
|
||||
|
||||
def test_derived_provider_names_column_missing(self):
|
||||
"""Test that missing column is added as empty and function returns early."""
|
||||
data = {"OTHER_COLUMN": ["value1", "value2"]}
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
result = add_aarete_derived_provider_name(df, state_flag=True)
|
||||
|
||||
self.assertIn("AARETE_DERIVED_PROVIDER_NAME", result.columns)
|
||||
self.assertEqual(result.loc[0, "AARETE_DERIVED_PROVIDER_NAME"], "")
|
||||
self.assertEqual(result.loc[1, "AARETE_DERIVED_PROVIDER_NAME"], "")
|
||||
|
||||
@patch(
|
||||
"src.pipelines.saas.prompts.prompt_calls.prompt_aarete_derived_provider_name"
|
||||
)
|
||||
def test_derived_provider_names_end_to_end(self, mock_llm):
|
||||
|
||||
data = {
|
||||
"AARETE_DERIVED_PROVIDER_NAME": [
|
||||
"Texas Health Physicians Group, Inc.",
|
||||
"Texas Health Physicians Group",
|
||||
]
|
||||
}
|
||||
|
||||
mock_llm.return_value = "TEXAS HEALTH PHYSICIANS GROUP"
|
||||
|
||||
df = pd.DataFrame(data)
|
||||
|
||||
result = add_aarete_derived_provider_name(df, state_flag=True)
|
||||
|
||||
self.assertIn("AARETE_DERIVED_PROVIDER_NAME", result.columns)
|
||||
|
||||
self.assertEqual(
|
||||
result.loc[0, "AARETE_DERIVED_PROVIDER_NAME"],
|
||||
"TEXAS HEALTH PHYSICIANS GROUP",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
result.loc[1, "AARETE_DERIVED_PROVIDER_NAME"],
|
||||
"TEXAS HEALTH PHYSICIANS GROUP",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -932,7 +932,7 @@ def remove_legal_suffixes(text: str) -> str:
|
||||
return " ".join(tokens)
|
||||
|
||||
|
||||
def extract_dba_payer_name(text: str) -> tuple[str | None, str | None]:
|
||||
def extract_dba_name(text: str) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Extract a DBA (Doing Business As) name from a text string.
|
||||
|
||||
@@ -1001,8 +1001,10 @@ def remove_states(text: str) -> str:
|
||||
if tokens[0] == state_tokens[0]:
|
||||
return text
|
||||
|
||||
# Remove trailing state tokens
|
||||
# Remove state AND trailing "of" if present
|
||||
new_tokens = tokens[: -len(state_tokens)]
|
||||
if new_tokens and new_tokens[-1] == "of":
|
||||
new_tokens = new_tokens[:-1]
|
||||
return " ".join(new_tokens).strip()
|
||||
|
||||
# Also handle "of <state>" pattern anywhere in the string, as it often indicates a regional suffixe.g., "Health Plan of New York" should become "Health Plan"
|
||||
|
||||
Reference in New Issue
Block a user