Merged main into bugfix/UT-special-cases

fieldExtraction/src/investment/postprocess.py

@@ -49,6 +49,9 @@ def postprocess(df, constants: Constants):
             df[col] = df[col].apply(postprocessing_funcs.validate_and_reformat_date)
         if "CPT" in col:
             df[col] = df[col].apply(postprocessing_funcs.normalize_cpt_fields)
+    
+    # Replace PROV_GROUP_NAME_FULL with PROVIDER_NAME extracted from Hybrid Smart Chunking when not empty/N/A
+    df = postprocessing_funcs.replace_provider_name_from_hybrid_smart_chunking(df)
 
     # Deduplicate provider fields - remove GROUP values from OTHER fields and deduplicate OTHER lists
     df = postprocessing_funcs.deduplicate_provider_columns(df)

fieldExtraction/src/investment/postprocessing_funcs.py

@@ -621,6 +621,50 @@ def remove_redundant_reimb_info(df):
             df.loc[mask, reimb_col] = ""
     return df
 
+def replace_provider_name_from_hybrid_smart_chunking(df):
+    """
+    Replaces PROV_GROUP_NAME_FULL with PROVIDER_NAME for rows where PROVIDER_NAME is not empty/N/A,
+    then removes the PROVIDER_NAME column.
+    
+    This function is used in postprocessing after the Hybrid Smart Chunking provider name extraction.
+    The RAG method may produce a more accurate provider name than the initial extraction,
+    so we use it to override PROV_GROUP_NAME_FULL when available.
+    
+    Args:
+        df (pd.DataFrame): DataFrame containing contract data
+    
+    Returns:
+        pd.DataFrame: DataFrame with PROV_GROUP_NAME_FULL updated and PROVIDER_NAME removed
+    """
+    # Check if both columns exist
+    if "PROVIDER_NAME" not in df.columns:
+        logging.debug("PROVIDER_NAME column not found in DataFrame, skipping replacement")
+        return df
+    
+    if "PROV_GROUP_NAME_FULL" not in df.columns:
+        logging.debug("PROV_GROUP_NAME_FULL column not found in DataFrame")
+        # Just remove PROVIDER_NAME if it exists
+        df = df.drop(columns=["PROVIDER_NAME"])
+        return df
+    
+    # Create a mask for rows where PROVIDER_NAME should replace PROV_GROUP_NAME_FULL
+    # Replace when PROVIDER_NAME is not empty
+    mask = ~string_utils.is_empty(df["PROVIDER_NAME"], pd_mask=True)
+    
+    # Count how many rows will be updated
+    num_replacements = mask.sum()
+    
+    if num_replacements > 0:
+        # Replace PROV_GROUP_NAME_FULL with PROVIDER_NAME for matching rows
+        df.loc[mask, "PROV_GROUP_NAME_FULL"] = df.loc[mask, "PROVIDER_NAME"]
+        logging.debug(f"Replaced PROV_GROUP_NAME_FULL with PROVIDER_NAME for {num_replacements} rows")
+    else:
+        logging.debug("No valid PROVIDER_NAME values to replace PROV_GROUP_NAME_FULL")
+    
+    # Remove the PROVIDER_NAME column
+    df = df.drop(columns=["PROVIDER_NAME"])
+    
+    return df
 
 def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
     """Deduplicate provider fields in the DataFrame.

fieldExtraction/src/investment/prompt_calls.py

@@ -768,3 +768,54 @@ def prompt_lesser_of_check(service_term: str, reimb_term: str, filename: str):
         return True # Default to True - prefer extra rows to missing
 
 
+def extract_provider_name_from_first_page(text_dict: dict, filename: str) -> str:
+    """
+    Extracts the provider group name from the first page of the contract using LLM.
+    
+    This is an additional check to identify provider group names that may be present
+    on the first page but not captured through other extraction methods.
+    
+    Args:
+        text_dict (dict): Dictionary containing the contract text with page numbers as keys.
+        filename (str): The name of the file being processed.
+    
+    Returns:
+        str: The extracted provider name or "N/A" if not found.
+    """
+    # Get the first page
+    pages = list(text_dict.keys())
+    pages.sort(key=string_utils.page_key_sort)
+    
+    if not pages:
+        logging.warning(f"No pages found in text_dict for {filename}")
+        return "N/A"
+    
+    first_page = pages[0]
+    first_page_text = text_dict[first_page]
+    
+    # Create prompt for LLM
+    FIRST_PAGE_PROVIDER_NAME_PROMPT = prompt_templates.FIRST_PAGE_PROVIDER_NAME_TEMPLATE(
+        first_page_text=first_page_text
+    )
+    
+    # Get response from LLM
+    response = llm_utils.invoke_claude(FIRST_PAGE_PROVIDER_NAME_PROMPT, "legacy_sonnet", filename)
+    
+    try:
+        # Extract the provider name from delimiters
+        provider_name = string_utils.extract_text_from_delimiters(
+            response, Delimiter.PIPE
+        ).strip()
+        
+        # Validate the response
+        if provider_name and not string_utils.is_empty(provider_name):
+            logging.info(f"First page provider name extracted for {filename}: {provider_name}")
+            return provider_name
+        else:
+            logging.debug(f"No valid provider name found on first page for {filename}")
+            return "N/A"
+            
+    except Exception as e:
+        logging.error(f"Error extracting provider name from first page: {str(e)}")
+        logging.error(f"Raw response: {response}")
+        return "N/A"

fieldExtraction/src/investment/tin_npi_funcs.py

@@ -1,4 +1,5 @@
 import json
+
 import logging
 import re
 from typing import Optional
@@ -6,11 +7,11 @@ from typing import Optional
 import constants.regex_patterns as regex_patterns
 import src.config as config
 import src.prompts.prompt_templates as prompt_templates
-import src.utils.llm_utils as llm_utils
-import src.utils.string_utils as string_utils
+from src.utils import llm_utils, string_utils
+from src.investment import prompt_calls
 from constants.delimiters import Delimiter
 from src.prompts.fieldset import Field, FieldSet
-
+from difflib import SequenceMatcher
 
 def get_all_matches(text: str, pattern: str) -> list[str]:
     """
@@ -479,7 +480,88 @@ def deduplicate_providers(
     return valid_providers
 
 
-def get_provider_info(text_dict: dict, page_num: str, filename: str) -> list:
+def is_provider_name_match(
+    provider_name: str, 
+    first_page_provider_group_name: str, 
+    threshold: float = 0.75
+) -> bool:
+    """Check if provider_name matches first_page_provider_group_name (handles d/b/a)."""
+    if string_utils.is_empty(provider_name) or string_utils.is_empty(first_page_provider_group_name):
+        return False
+    
+    if first_page_provider_group_name == "N/A":
+        return False
+    
+    provider_clean = provider_name.lower().strip()
+    expected_clean = first_page_provider_group_name.lower().strip()
+    
+    # Strategy 1: Exact match
+    if provider_clean == expected_clean:
+        logging.debug(f"[is_provider_name_match] ✓ Exact match")
+        return True
+    
+    # Strategy 2: Fuzzy similarity
+    similarity = SequenceMatcher(None, provider_clean, expected_clean).ratio()
+    if similarity >= threshold:
+        logging.debug(f"[is_provider_name_match] ✓ Fuzzy match ({similarity:.2f})")
+        return True
+    
+    # Strategy 3: Contains match
+    if expected_clean in provider_clean or provider_clean in expected_clean:
+        logging.debug(f"[is_provider_name_match] ✓ Contains match")
+        return True
+    
+    # Strategy 4: Word overlap
+    common_words = {"llc", "inc", "corp", "ltd", "the", "and", "dba", "d/b/a", "pa", "pc"}
+    provider_words = set([w for w in provider_clean.split() if len(w) > 3 and w not in common_words])
+    expected_words = set([w for w in expected_clean.split() if len(w) > 3 and w not in common_words])
+    
+    if len(provider_words) > 0 and len(expected_words) > 0:
+        overlap = len(provider_words & expected_words)
+        overlap_ratio = overlap / min(len(provider_words), len(expected_words))
+        
+        if overlap_ratio >= 0.6:
+            logging.debug(f"[is_provider_name_match] ✓ Word overlap ({overlap_ratio:.2f})")
+            return True
+    
+    # Strategy 5: D/B/A parsing
+    if "d/b/a" in expected_clean or "dba" in expected_clean:
+        dba_pattern = r'd\.?/?b\.?/?a\.?|doing\s+business\s+as'
+        parts = re.split(dba_pattern, expected_clean, flags=re.IGNORECASE)
+        
+        if len(parts) >= 2:
+            # Check legal name (before d/b/a)
+            legal_name = parts[0].strip()
+            if provider_clean in legal_name or legal_name in provider_clean:
+                logging.debug(f"[is_provider_name_match] ✓ Legal name match")
+                return True
+            
+            # Check d/b/a section
+            dba_section = parts[1].strip()
+            if provider_clean in dba_section or dba_section in provider_clean:
+                logging.debug(f"[is_provider_name_match] ✓ D/B/A section match")
+                return True
+            
+            # Word overlap with d/b/a section
+            dba_words = set([w for w in dba_section.split() if len(w) > 3 and w not in common_words])
+            if len(provider_words) > 0 and len(dba_words) > 0:
+                overlap = len(provider_words & dba_words)
+                overlap_ratio = overlap / min(len(provider_words), len(dba_words))
+                
+                if overlap_ratio >= 0.6:
+                    logging.debug(f"[is_provider_name_match] ✓ D/B/A word overlap ({overlap_ratio:.2f})")
+                    return True
+    
+    logging.debug(f"[is_provider_name_match] ✗ No match")
+    return False
+
+
+def get_provider_info(
+    text_dict: dict, 
+    page_num: str, 
+    filename: str,
+    first_page_provider_group_name: str = None
+) -> list:
     """
     Extracts provider information from a specified page in a document using a language model.
 
@@ -487,6 +569,7 @@ def get_provider_info(text_dict: dict, page_num: str, filename: str) -> list:
         text_dict (dict): Dictionary of text content with page numbers as keys.
         page_num (str): The page number to extract provider information from.
         filename (str): The name of the file being processed, used for logging or tracking purposes.
+        first_page_provider_group_name (str, optional): Expected group name from page 1 for name verification.
 
     Returns:
         list: A list of extracted provider information dictionaries, each containing TIN, NPI,
@@ -583,16 +666,23 @@ def get_provider_info(text_dict: dict, page_num: str, filename: str) -> list:
                 f"LLM missed identifiers on page {page_num} of {filename}. Missed TINs: {missed_tins}, Missed NPIs: {missed_npis}"
             )
 
-        # Add IS_GROUP flag based on ON_SIGNATURE_PAGE  and page_num
         for provider in providers:
-            if float(page_num) <= 2.0 or provider.get("ON_SIGNATURE_PAGE", "N") == "Y":
+            provider_name = provider.get("NAME", "")
+            
+            name_matches = False
+            if first_page_provider_group_name and first_page_provider_group_name != "N/A" and not string_utils.is_empty(provider_name):
+                name_matches = is_provider_name_match(provider_name, first_page_provider_group_name)
+            
+            if name_matches:
                 provider["IS_GROUP"] = "Y"
             else:
                 provider["IS_GROUP"] = "N"
+            
             if "ON_SIGNATURE_PAGE" in provider:
                 del provider["ON_SIGNATURE_PAGE"]
 
         return providers
+    
     except ValueError as e:  # Handle JSON parsing error
         logging.error(f"Error parsing JSON response from LLM: {str(e)}")
         logging.error(f"Raw response: {claude_answer_raw}")
@@ -605,7 +695,7 @@ def get_provider_info(text_dict: dict, page_num: str, filename: str) -> list:
                 "IS_GROUP": "N",
             }
         ]
-
+    
 
 def run_provider_info_fields(
     contract_text: str,
@@ -648,6 +738,10 @@ def run_provider_info_fields(
     logging.debug(f"Extracted TINs: {all_tins}")
     logging.debug(f"Extracted NPIs: {all_npis}")
 
+    # Extract expected group name from first page
+    first_page_provider_group_name = prompt_calls.extract_provider_name_from_first_page(text_dict, filename)
+    logging.info(f"[run_provider_info_fields] Expected group name: '{first_page_provider_group_name}'")
+
     # Log OCR corrections for monitoring
     exact_tins = [tin for tin, quality in tin_matches if quality == "EXACT"]
     ocr_corrected_tins = [
@@ -703,17 +797,18 @@ def run_provider_info_fields(
                 "NAME": "NO_IDENTIFIERS_FOUND",
                 "IS_GROUP": "N",
             }
-        ]       
-    # Process each page separately and combine results
-    combined_provider_info = []
-    for page_num in relevant_pages:
-        page_provider_info = get_provider_info(text_dict, page_num, filename)
-        combined_provider_info.extend(page_provider_info)
+        ]
+    else:
+        # Process each page separately and combine results
+        combined_provider_info = []
+        for page_num in relevant_pages:
+            page_provider_info = get_provider_info(text_dict, page_num, filename, first_page_provider_group_name)
+            combined_provider_info.extend(page_provider_info)
 
-    # Clean and standardize results
-    cleaned_provider_info = clean_provider_info(combined_provider_info)
-    # Deduplicate providers based on TIN and NPI
-    deduplicated_provider_info = deduplicate_providers(cleaned_provider_info)
+        # Clean and standardize results
+        cleaned_provider_info = clean_provider_info(combined_provider_info)
+        # Deduplicate providers based on TIN and NPI
+        deduplicated_provider_info = deduplicate_providers(cleaned_provider_info)
 
     # Create columns
     one_to_one_results = {}
@@ -730,7 +825,13 @@ def run_provider_info_fields(
     one_to_one_results = merge_provider_info(
         one_to_one_results, deduplicated_provider_info
     )  # Add Group TIN
-    # What if the TIN/NPI regexes don't find anything?  Add PROV_GROUP_NAME_FULL to full context
+
+    # Override PROV_GROUP_NAME_FULL with first page extraction if found
+    if first_page_provider_group_name != "N/A":
+        logging.info(f"Overriding PROV_GROUP_NAME_FULL with first page extraction: {first_page_provider_group_name}")
+        one_to_one_results["PROV_GROUP_NAME_FULL"] = first_page_provider_group_name
+
+    # Fallback: What if the TIN/NPI regexes don't find anything? Add PROV_GROUP_NAME_FULL to full context
     if (
         string_utils.is_empty(one_to_one_results.get("PROV_OTHER_NAME_FULL")) and
         string_utils.is_empty(one_to_one_results.get("PROV_GROUP_NAME_FULL"))

fieldExtraction/src/prompts/investment_prompts.json

@@ -21,7 +21,7 @@
         "field_name": "AARETE_DERIVED_REIMB_METHOD",
         "relationship": "one_to_n",
         "field_type": "methodology_breakout",
-        "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'\n- In case of DRG, MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC(Ambulatory Payment Classification)/OPPS(Outpatient Prospective Payment System) or Ambulatory Surgical Center (ASC), return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP)\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'\n- Provider interest rates are not considered 'Flat Rate'.\n- Charge master increase caps or limitations on provider pricing behavior are NOT reimbursement methodologies and should be ignored.\n- Always return 'Grouper' as the priority value if any grouper-related codes or references (e.g., MS-DRG, APR-DRG, DRG, etc.) are present in the SERVICE or METHODOLOGY fields — regardless of any other reimbursement methods (such as case rate, per diem, or lesser-of comparisons) mentioned. \n\nIf no specific rule is matched, map to the most appropriate value from the given valid values.",
+        "prompt": "What is the method of reimbursement? Choose only from the following valid values: {valid_values}.\nFollow these mapping rules:\n- If any of Medicare, Medicaid, Average Sales Price (ASP), AWP (Average Wholesale Price), WAC (Wholesale Acquisition Cost), RBRVS (Resource-Based Relative Value Scale), RVU (Relative Value Unit) or ASA (American Society of Anesthesiology) or any type of Fee Schedule is mentioned, return 'Fee Schedule'.\n- In case of DRG, MS-DRG, AP-DRG, SDA (Standard Dollar Amount), APR-DRG, APG, EAPG, APC (Ambulatory Payment Classification)/OPPS (Outpatient Prospective Payment System) or Ambulatory Surgical Center (ASC), return 'Grouper'.\n- In case of Percent of Charge or Percentage of Billed, return 'Billed Charges'.\n- When reimbursement is defined on a per-day basis, the applicable method is Per Diem.\n- For mixed methodologies (e.g., 'cost plus 5% based on AWP'), prioritize the base method (AWP).\n\nImportant exceptions and additional rules:\n- Reimbursements with specific dollar values cannot be 'Fee Schedule'.\n- Provider interest rates are not considered 'Flat Rate'.\n- Charge master increase caps or limitations on provider pricing behavior are NOT reimbursement methodologies and should be ignored.\n- **Precedence rule (final):** If any grouper-related codes or references (e.g., DRG, MS-DRG, APR-DRG, APG, EAPG, APC/OPPS, SDA, ASC grouping, etc.) are present in the SERVICE or METHODOLOGY fields, return 'Grouper' — This **overrides all other mapping rules**, including explicit dollar amounts (normally 'Flat Rate'), Fee Schedule indicators, case rates, per diem references, or cost-based methodologies..\n- **Exception:** If the methodology explicitly maps to 'Billed Charges' (e.g., Percent of Charge, Percentage of Billed), then return 'Billed Charges' even if grouper codes are present.\n\nIf no specific rule is matched, map to the most appropriate value from the given valid values.",
         "valid_values": "VALID_REIMB_METHODOLOGY"
     },
     {
@@ -257,6 +257,13 @@
         "field_type": "provider_info",
         "prompt": "Return ONLY the full name of the provider associated with this TIN and NPI as it appears in the document. If no actual name is given, return 'N/A' (do not return placeholders like 'provider' or 'provider name'). Do not extract any other names apart from the provider name."
     },
+    {
+        "field_name": "PROVIDER_NAME",
+        "relationship": "one_to_one",
+        "field_type": "smart_chunked",
+        "prompt": "Extract the PROVIDER organization name from the retrieved contract text.\n\nThe provider is the healthcare organization entering into the agreement (NOT the payer/insurance company like Highmark, Aetna, UnitedHealthcare, Blue Cross Blue Shield, etc.).\n\nGUIDELINES:\n1. Look for the provider name in phrases like:\n   - 'This Agreement is between [PAYER] and [PROVIDER]'\n   - 'Agreement between [PAYER] and [PROVIDER]'\n   - 'by and between [PAYER] and [PROVIDER]'\n   - '[PROVIDER] (hereinafter referred to as \"Provider\")'\n   - 'Provider: [PROVIDER]'\n2. The provider name often appears in the opening paragraph or preamble of the contract.\n3. Return the EXACT provider name as it appears in the document, including legal suffixes (LLC, Inc., P.C., etc.).\n4. If the provider name includes 'd/b/a' (doing business as), return only the legal entity name before 'd/b/a'.\n   - Example: 'Middletown Universal, LLC d/b/a American Kidney Care' → return 'Middletown Universal, LLC'\n5. Do NOT return:\n   - The payer/insurance company name\n   - Individual physician names (unless that is the practice name)\n   - Department names only\n   - Addresses or other metadata\n6. If multiple potential provider names appear, return the one from the primary agreement clause (usually in the preamble).\n7. If the provider name cannot be determined from the retrieved text, return 'N/A'.\n\nReturn ONLY the provider name, nothing else.",
+        "retrieval_question": "What is the provider organization name? Agreement between payer and provider. This agreement is made between. By and between. Hereinafter referred to as Provider. Provider name in contract preamble. Provider party to the agreement. Healthcare organization entering agreement."
+    },
     {
         "field_name": "ON_SIGNATURE_PAGE",
         "relationship": "one_to_one",

fieldExtraction/src/prompts/prompt_templates.py

@@ -874,6 +874,18 @@ def FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(allow_na):
     else:
         return " Do NOT leave this field N/A. Choose the most appropriate answer based on the context. If there are multiple answers, return them in a pipe (|) separated list."
 
+def FIRST_PAGE_PROVIDER_NAME_TEMPLATE(first_page_text: str) -> str:
+    return f"""Extract the PROVIDER organization name from this contract first page.
+
+The provider is the healthcare organization entering into the agreement (NOT the payer/insurance company like Highmark, Aetna, UnitedHealthcare, etc.). Look for patterns like 'This AGREEMENT is made between [PAYER] and [PROVIDER]'. Return the EXACT provider name as it appears in the contract.
+FIRST PAGE TEXT:
+{first_page_text}
+
+CRITICAL: Return ONLY the provider name wrapped in || delimiters, with NO explanation, NO preamble, NO additional text.
+
+If no provider found, return ONLY: ||N/A||
+
+Your response must start with || and end with ||"""
 
 def ONE_TO_ONE_MULTI_FIELD_TEMPLATE(context, questions: dict[str, str]):
     return f"""The following is a contract (or excerpt thereof).  Answer the following questions: {questions}

fieldExtraction/tests/test_tin_npi_funcs.py

@@ -111,13 +111,15 @@ class TestTinNpiFuncs:
                     "NPI": "1234567890",
                     "NAME": "Test Provider",
                     "ON_SIGNATURE_PAGE": "N",
+                    "IS_GROUP": "Y",
+
                 }
             ]
         )
 
         result = tin_npi_funcs.get_provider_info(sample_text_dict, "1", "test.pdf")
         assert len(result) == 1
-        assert result[0]["IS_GROUP"] == "Y"  # Page 1 should be marked as intro
+        assert result[0]["IS_GROUP"] == "N"  # Page 1 should not be marked as intro
 
     @patch("src.utils.llm_utils.invoke_claude")
     def test_run_provider_info_fields(self, mock_invoke_claude, sample_text_dict):