Merged in feature/DAIP2-1562-add-aarete_derived_payer_name (pull request #867)

Feature/DAIP2-1562 add aarete derived payer name * black format * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * llm_choose_derived_payer_name function added * threshold updated * Merge branch 'DEV' into feature/DAIP2-1562-add-aarete_derived_payer_name * aarete_derived_payer_name column added * prompt structure updated * state_flag added * pipeline error fixed * pipeline error fixed * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * remove debug print statement for similarity matrix in clustering function * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * state logic added in clustering * removed print statements * pull request updates * black format fix * Merged DEV into feature/DAIP2-1562-add-aarete_derived_payer_name * added scalability feature and optimization * black format * Pull Request Changes * updated config parameters * updated main * no payer name column changes * Business team feedback: always run derived payer name with state_flag=True - Add empty DataFrame guard and proper docstring - Clean up not needed flags Approved-by: Siddhant Medar
2026-02-18 17:17:30 +00:00
parent a0c7e7a738
commit f4457abf35
9 changed files with 881 additions and 1 deletions
@@ -14,8 +14,79 @@ from src.constants.regex_patterns import (
    BACKTICK_PATTERN,
    PIPE_PATTERN,
    TRIPLE_BACKTICK_PATTERN,
+    DBA_PATTERNS,
 )

+LEGAL_SUFFIXES = {
+    "inc",
+    "inc.",
+    "incorporated",
+    "corp",
+    "corporation",
+    "llc",
+    "l.l.c",
+    "ltd",
+    "limited",
+    "co",
+    "company",
+    "affiliates",
+}
+
+STOPWORDS = {"of", "the", "and", "its"}
+
+US_STATES = [
+    "alabama",
+    "alaska",
+    "arizona",
+    "arkansas",
+    "california",
+    "colorado",
+    "connecticut",
+    "delaware",
+    "florida",
+    "georgia",
+    "hawaii",
+    "idaho",
+    "illinois",
+    "indiana",
+    "iowa",
+    "kansas",
+    "kentucky",
+    "louisiana",
+    "maine",
+    "maryland",
+    "massachusetts",
+    "michigan",
+    "minnesota",
+    "mississippi",
+    "missouri",
+    "montana",
+    "nebraska",
+    "nevada",
+    "new hampshire",
+    "new jersey",
+    "new mexico",
+    "new york",
+    "north carolina",
+    "north dakota",
+    "ohio",
+    "oklahoma",
+    "oregon",
+    "pennsylvania",
+    "rhode island",
+    "south carolina",
+    "south dakota",
+    "tennessee",
+    "texas",
+    "utah",
+    "vermont",
+    "virginia",
+    "washington",
+    "west virginia",
+    "wisconsin",
+    "wyoming",
+]
+

 def extract_text_from_delimiters(
    raw_output: str, delimiter: Delimiter, match_index: int = -1
@@ -806,3 +877,167 @@ def normalize_to_json_list(val):
        result = json.dumps(expanded)
        return result
    return val
+
+
+def normalize_text(text: str) -> str:
+    """
+    Normalize a text string by applying common preprocessing steps.
+
+    This function:
+    - Converts all characters to lowercase
+    - Removes punctuation and special characters
+    - Replaces multiple whitespace characters with a single space
+    - Strips leading and trailing whitespace
+
+    Args:
+        text (str): The input text to normalize.
+
+    Returns:
+        str: The normalized text.
+    """
+    # Convert the text to lowercase for case-insensitive processing
+    text = text.lower()
+
+    # Replace all characters that are not letters, numbers, or whitespace with a space
+    text = re.sub(r"[^\w\s]", " ", text)
+
+    # Replace multiple whitespace characters with a single space
+    # and remove leading/trailing spaces
+    text = re.sub(r"\s+", " ", text).strip()
+
+    return text
+
+
+def remove_legal_suffixes(text: str) -> str:
+    """
+    Remove legal entity suffixes from a text string.
+
+    This function splits the input text into tokens (words) and
+    removes any tokens that match known legal suffixes
+    (e.g., "llc", "inc", "corp").
+
+    Args:
+        text (str): The input text containing a business name.
+
+    Returns:
+        str: The text with legal suffixes removed.
+    """
+    # Split the text into individual words
+    tokens = text.split()
+
+    # Filter out any tokens that are legal suffixes
+    tokens = [t for t in tokens if t not in LEGAL_SUFFIXES]
+
+    # Rejoin the remaining tokens into a single string
+    return " ".join(tokens)
+
+
+def extract_dba_payer_name(text: str) -> tuple[str | None, str | None]:
+    """
+    Extract a DBA (Doing Business As) name from a text string.
+
+    The function checks the input text against a list of predefined
+    DBA patterns (e.g., "dba", "doing business as"). If a pattern is found,
+    it returns the text that appears after the pattern.
+
+    Ex:
+    Input: NEW YORK STATE CATHOLIC HEALTH PLAN, INC. d/b/a FIDELIS CARE NEW YORK
+
+    Output: FIDELIS CARE NEW YORK
+
+    Args:
+        text (str): The input text that may contain a DBA reference.
+
+    Returns:
+        tuple[str | None, str | None]: A tuple containing:
+            - The text before the DBA pattern (potentially the legal name)
+            - The text after the DBA pattern (the extracted DBA name)
+            If no DBA pattern is found, both elements of the tuple will be None.
+    """
+    # Iterate through all known DBA patterns
+    for pattern in DBA_PATTERNS:
+        # Split the text using the current pattern (case-insensitive)
+        match = re.split(pattern, text, flags=re.I)
+
+        # If the pattern was found, the split will produce more than one part
+        if len(match) > 1:
+            # Return the text after the matched pattern, trimmed of whitespace
+            return match[0].strip(), match[-1].strip()
+
+    # Return None if no DBA pattern is found
+    return None, None
+
+
+def remove_states(text: str) -> str:
+    """
+    Remove U.S. state names or abbreviations from a text string.
+
+    The function iterates through a list of U.S. states and removes
+    any exact (word-boundary matched) occurrences from the input text.
+    Extra whitespace created by removals is cleaned up before returning.
+
+    Remove state names when they appear as a regional suffix
+    (i.e., at the end of the string), but preserve them when
+    they are leading brand identifiers (e.g., 'Oklahoma Complete Health').
+
+    Args:
+        text (str): The input text that may contain U.S. state names.
+
+    Returns:
+        str: The text with U.S. state references removed.
+    """
+
+    tokens = text.split()
+    if not tokens:
+        return text
+
+    for state in US_STATES:
+        state_tokens = state.split()
+
+        # Check if state appears at END of string
+        if tokens[-len(state_tokens) :] == state_tokens:
+
+            # Do NOT remove if state is the first token
+            if tokens[0] == state_tokens[0]:
+                return text
+
+            # Remove trailing state tokens
+            new_tokens = tokens[: -len(state_tokens)]
+            return " ".join(new_tokens).strip()
+
+    # Also handle "of <state>" pattern anywhere in the string, as it often indicates a regional suffixe.g., "Health Plan of New York" should become "Health Plan"
+    # Remove each state using a case-insensitive regex match. It looks for the word "of" followed by the state name.
+    for state in US_STATES:
+        text = re.sub(rf"\bof\s+{state}\b", "", text, flags=re.I)
+
+    # Normalize whitespace after removals
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def detect_state(text: str) -> str | None:
+    """Return state name if found in text."""
+    for state in US_STATES:
+        if re.search(rf"\b{state}\b", text, re.I):
+            return state
+    return None
+
+
+def token_signature(text: str) -> tuple:
+    """
+    Generate a normalized token signature from a text string.
+
+    The function removes stopwords, sorts the remaining tokens,
+    and returns them as an immutable tuple. This signature can be
+    used for fast comparison or deduplication of similar texts.
+
+    Args:
+        text (str): The input text to tokenize.
+
+    Returns:
+        tuple: A sorted tuple of tokens excluding stopwords.
+    """
+    # Split the text into tokens and remove stopwords
+    tokens = [t for t in text.split() if t not in STOPWORDS]
+
+    # Sort tokens to ensure order-independent comparison
+    return tuple(sorted(tokens))