Merged in feature/update-claim-type (pull request #869)
Feature/update claim type * Update CLAIM_TYPE_CD extraction to prioritize title/header - Modified retrieval_question to search for title, header, and agreement name - Updated prompt to check title/header first before analyzing body text - Enhanced FULL_CONTEXT_CLAIM_TYPES_ADDITIONAL_INSTRUCTION to emphasize title/header as primary source * Fix CLAIM_TYPE_CD extraction with 3-step fallback 1. Exhibit Level: Updated retrieval_question and prompt to prioritize title/header 2. Contract Title fallback: Added infer_claim_type_from_title() function that extracts claim type from CONTRACT_TITLE when exhibit-level extraction returns empty 3. Postprocessing: fill_claim_type() now uses CONTRACT_TITLE inference when all AARETE_DERIVED_CLAIM_TYPE_CD values are empty Keywords mapped: - Professional/Ancillary -> M (Physician, Professional, Ancillary, Home Health, DME, etc.) - Institutional -> H (Hospital, Facility, Surgery Center, SNF, etc.) * Add more professional keywords for claim type inference Added PROVIDER SERVICE, PROVIDER AGREEMENT, PARTICIPATING PROVIDER to handle titles like 'PROVIDER SERVICES AGREEMENT' * Add postprocessing safety net for claim type inference - Added fill_claim_type_from_title() in postprocessing_funcs.py that infers AARETE_DERIVED_CLAIM_TYPE_CD from CONTRACT_TITLE if still empty after code_breakout - Added call in postprocess.py before attach_sid_column - This ensures claim type is populated even if earlier extraction steps miss it * Improve CLAIM_TYPE_CD extraction context and postprocessing 1. Added keywords to CLAIM_TYPE_CD field (same as CONTRACT_TITLE) so it retrieves from the same document header/title context 2. Updated fill_claim_type_from_title() to first fill from other exhibits in the same file (mode within file) before falling back to CONTRACT_TITLE keyword matching * Merge DEV into feature/update-claim-type Resolved conflicts: - code_funcs.py: Kept DEV's code_breakout (claim type now handled in postprocessing) - postprocess.py: Added fill_claim_type_from_title call - prompt_templates.py: Kept DEV's VALIDATE_REIMBURSEMENTS_PROMPT signature * Merge DEV into feature/update-claim-type * move keywords to ancillary Approved-by: Katon Minhas
This commit is contained in:
Faizan Mohiuddin
committed by
Katon Minhas
Katon Minhas
parent
9e4b88395b
commit
49407bfb93
@@ -82,6 +82,9 @@ def postprocess(df, constants: Constants):
|
||||
|
||||
df = postprocessing_funcs.update_grouper_base_rate_and_grouper_pct_rate(df)
|
||||
|
||||
# Fill empty claim type from CONTRACT_TITLE as a safety net
|
||||
df = postprocessing_funcs.fill_claim_type_from_title(df)
|
||||
|
||||
df = postprocessing_funcs.attach_sid_column(df)
|
||||
|
||||
# Standardize output column order - this should ALWAYS be the final postprocessing step
|
||||
|
||||
@@ -853,3 +853,117 @@ def add_aarete_derived_signatory_complete_ind(df: pd.DataFrame) -> pd.DataFrame:
|
||||
except Exception:
|
||||
# Return df unchanged on any exception
|
||||
return df
|
||||
|
||||
|
||||
def fill_claim_type_from_title(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""
|
||||
Fill empty AARETE_DERIVED_CLAIM_TYPE_CD values using two strategies:
|
||||
1. First, fill from other rows in the same file that have populated values (mode within file)
|
||||
2. Then, infer from CONTRACT_TITLE using keyword matching
|
||||
|
||||
This is a postprocessing safety net to ensure claim type is populated even if
|
||||
earlier extraction steps failed to set it.
|
||||
|
||||
Args:
|
||||
df (pd.DataFrame): Input DataFrame with FILE_NAME, CONTRACT_TITLE and AARETE_DERIVED_CLAIM_TYPE_CD columns.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: DataFrame with AARETE_DERIVED_CLAIM_TYPE_CD filled where possible.
|
||||
"""
|
||||
if "AARETE_DERIVED_CLAIM_TYPE_CD" not in df.columns:
|
||||
df["AARETE_DERIVED_CLAIM_TYPE_CD"] = ""
|
||||
|
||||
# Step 1: Fill from other rows in the same file (mode within file)
|
||||
if "FILE_NAME" in df.columns:
|
||||
for file_name in df["FILE_NAME"].unique():
|
||||
file_mask = df["FILE_NAME"] == file_name
|
||||
file_rows = df.loc[file_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"]
|
||||
|
||||
# Get non-empty values for this file
|
||||
non_empty_values = [
|
||||
v for v in file_rows if not string_utils.is_empty(v)
|
||||
]
|
||||
|
||||
if non_empty_values:
|
||||
# Use mode (most common value) to fill empty rows in this file
|
||||
mode_value = max(set(non_empty_values), key=non_empty_values.count)
|
||||
empty_mask = file_mask & df["AARETE_DERIVED_CLAIM_TYPE_CD"].apply(
|
||||
string_utils.is_empty
|
||||
)
|
||||
df.loc[empty_mask, "AARETE_DERIVED_CLAIM_TYPE_CD"] = mode_value
|
||||
|
||||
# Step 2: If still empty, infer from CONTRACT_TITLE
|
||||
if "CONTRACT_TITLE" not in df.columns:
|
||||
return df
|
||||
|
||||
# Professional indicators -> M
|
||||
professional_keywords = [
|
||||
"PHYSICIAN",
|
||||
"PROFESSIONAL",
|
||||
"PROVIDER SERVICE",
|
||||
"PROVIDER GROUP",
|
||||
"PROVIDER AGREEMENT",
|
||||
"PARTICIPATING PROVIDER",
|
||||
"MEDICAL GROUP",
|
||||
"PRACTITIONER",
|
||||
]
|
||||
|
||||
# Ancillary indicators -> M
|
||||
ancillary_keywords = [
|
||||
"ANCILLARY",
|
||||
"HOME HEALTH",
|
||||
"DME",
|
||||
"DURABLE MEDICAL",
|
||||
"LABORATORY",
|
||||
"LAB SERVICE",
|
||||
"RADIOLOGY",
|
||||
"DIALYSIS",
|
||||
"IMAGING",
|
||||
"BEHAVIORAL",
|
||||
"MENTAL HEALTH",
|
||||
"TELEMEDICINE",
|
||||
"TELEHEALTH",
|
||||
"SURGERY CENTER",
|
||||
"AMBULATORY SURGICAL",
|
||||
"SKILLED NURSING",
|
||||
"REHAB",
|
||||
]
|
||||
|
||||
# Institutional/Hospital/Facility indicators -> H
|
||||
institutional_keywords = [
|
||||
"HOSPITAL",
|
||||
"INSTITUTIONAL",
|
||||
"FACILITY",
|
||||
"INPATIENT",
|
||||
]
|
||||
|
||||
def infer_from_title(row):
|
||||
# If already has a value, keep it
|
||||
current_val = row.get("AARETE_DERIVED_CLAIM_TYPE_CD", "")
|
||||
if not string_utils.is_empty(current_val):
|
||||
return current_val
|
||||
|
||||
title = str(row.get("CONTRACT_TITLE", "")).upper()
|
||||
if not title:
|
||||
return ""
|
||||
|
||||
# Check Professional keywords
|
||||
for kw in professional_keywords:
|
||||
if kw in title:
|
||||
return "M"
|
||||
|
||||
# Check Ancillary keywords
|
||||
for kw in ancillary_keywords:
|
||||
if kw in title:
|
||||
return "M"
|
||||
|
||||
# Check Institutional keywords
|
||||
for kw in institutional_keywords:
|
||||
if kw in title:
|
||||
return "H"
|
||||
|
||||
return ""
|
||||
|
||||
df["AARETE_DERIVED_CLAIM_TYPE_CD"] = df.apply(infer_from_title, axis=1)
|
||||
|
||||
return df
|
||||
|
||||
@@ -156,10 +156,20 @@
|
||||
"field_name": "CLAIM_TYPE_CD",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "exhibit_level",
|
||||
"prompt": "What is the Claim Type of the contract, exhibit, or services mentioned in the text? Choose ONLY from the following: {valid_values}.\n\nGuidelines:\n- Return 'Professional' if the text refers to services rendered by individual healthcare providers or provider groups (e.g., physician services, office visits, outpatient procedures).\n- Return 'Institutional' if the text refers to services provided by hospitals, facilities, or institutions (e.g., inpatient, outpatient facility, or surgery centers).\n- Return 'Ancillary' if the text refers to services such as Home Health, Durable Medical Equipment (DME), Laboratory, Radiology, Dialysis, Diagnostic Imaging, Genetic Testing, Blood Testing, Sleep Lab Services, Telemedicine, Behavioral or Mental Health, or other ancillary settings.\n- If multiple claim types are mentioned, return the one most closely associated with the compensation or service description in the context.\n- If no claim type is mentioned or cannot be determined, return 'N/A'.\nDo not infer or guess.",
|
||||
"prompt": "What is the Claim Type of the contract, exhibit, or services mentioned in the text? Choose ONLY from the following: {valid_values}.\n\nPRIORITY: First check the title or header for explicit claim type indicators before analyzing the body text.\n\nGuidelines:\n- Return 'Professional' if the title/header indicates physician, professional services, or provider group agreement, OR if the text refers to services rendered by individual healthcare providers or provider groups (e.g., physician services, office visits, outpatient procedures).\n- Return 'Institutional' if the title/header indicates hospital, facility, or institutional agreement, OR if the text refers to services provided by hospitals, facilities, or institutions (e.g., inpatient, outpatient facility, or surgery centers).\n- Return 'Ancillary' if the title/header indicates ancillary services, OR if the text refers to services such as Home Health, Durable Medical Equipment (DME), Laboratory, Radiology, Dialysis, Diagnostic Imaging, Genetic Testing, Blood Testing, Sleep Lab Services, Telemedicine, Behavioral or Mental Health, or other ancillary settings.\n- If multiple claim types are mentioned, return the one most closely associated with the title/header first, then the compensation or service description in the context.\n- If no claim type is mentioned or cannot be determined, return 'N/A'.\nDo not infer or guess.",
|
||||
"valid_values": "VALID_CLAIM_TYPE",
|
||||
"keywords": [],
|
||||
"retrieval_question": "Who is the provider in the contract? Provider is usually a physician practice, medical group, hospital, surgery center, ambulatory facility, institutional provider, home health agency, DME supplier, laboratory, radiology center, or ancillary service provider delivering healthcare services",
|
||||
"keywords": [
|
||||
"AGREEMENT",
|
||||
"AMENDMENT",
|
||||
"ADDENDUM",
|
||||
"PROVIDER",
|
||||
"CONTRACT",
|
||||
"MEMORANDUM",
|
||||
"LETTER",
|
||||
"REQUEST",
|
||||
"EFFECTIVE"
|
||||
],
|
||||
"retrieval_question": "What is the title of this agreement? Look for the header, title, agreement name, or exhibit title. Common titles include Professional Services Agreement, Physician Agreement, Hospital Agreement, Institutional Provider Agreement, Facility Agreement, Ancillary Services Agreement, DME Provider Agreement, Home Health Agreement, Laboratory Services Agreement.",
|
||||
"allow_na": true
|
||||
},
|
||||
{
|
||||
|
||||
@@ -1518,7 +1518,7 @@ Contract text:
|
||||
|
||||
|
||||
def FULL_CONTEXT_CLAIM_TYPES_ADDITIONAL_INSTRUCTION():
|
||||
return "Additionally Check the contract text for the provider name from the preamble or signature section to help determine the claim type."
|
||||
return " IMPORTANT: Prioritize the title and header as the primary source for determining claim type. Look for terms like 'Professional', 'Physician', 'Hospital', 'Facility', 'Institutional', 'Ancillary', 'DME', 'Home Health', 'Laboratory' in the title or header. If the title/header clearly indicates the claim type, use that. Otherwise, check the preamble, provider name, or signature section to help determine the claim type."
|
||||
|
||||
|
||||
def FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(allow_na):
|
||||
|
||||
Reference in New Issue
Block a user