diff --git a/fieldExtraction/constants/lists/valid_unit_of_measure.json b/fieldExtraction/constants/lists/valid_unit_of_measure.json index 0f7e9cb..53f8505 100644 --- a/fieldExtraction/constants/lists/valid_unit_of_measure.json +++ b/fieldExtraction/constants/lists/valid_unit_of_measure.json @@ -11,5 +11,6 @@ "Per Session", "Per Training Session", "Per Visit", - "Per Case" + "Per Case", + "Per Procedure" ] \ No newline at end of file diff --git a/fieldExtraction/src/investment/postprocessing_funcs.py b/fieldExtraction/src/investment/postprocessing_funcs.py index 3e23a55..1eedfc3 100644 --- a/fieldExtraction/src/investment/postprocessing_funcs.py +++ b/fieldExtraction/src/investment/postprocessing_funcs.py @@ -485,12 +485,16 @@ def standardize_reimb_method_and_fee_schedule( # remove cases of Per Member Per Month (PMPM) df = df[~df["UNIT_OF_MEASURE"].isin(["Per Member Per Month (PMPM)"])] - # Set UNIT_OF_MEASURE as 'Per Unit' for Flat Rate reimbursement method with empty UNIT_OF_MEASURE - df.loc[ - (df["AARETE_DERIVED_REIMB_METHOD"] == "Flat Rate") - & (df["UNIT_OF_MEASURE"].apply(string_utils.is_empty)), - "UNIT_OF_MEASURE", - ] = "Per Unit" + # Enhanced logic: Set UNIT_OF_MEASURE as 'Per Unit' for Flat Rate reimbursement method with empty UNIT_OF_MEASURE + # This catches all edge cases including NaN, None, empty strings, etc. + if "AARETE_DERIVED_REIMB_METHOD" in df.columns: + flat_rate_mask = df["AARETE_DERIVED_REIMB_METHOD"] == "Flat Rate" + empty_uom_mask = ( + df["UNIT_OF_MEASURE"].isna() + | df["UNIT_OF_MEASURE"].eq("") + | df["UNIT_OF_MEASURE"].apply(string_utils.is_empty) + ) + df.loc[flat_rate_mask & empty_uom_mask, "UNIT_OF_MEASURE"] = "Per Unit" # Ensure UNIT_OF_MEASURE is one of the valid values df["UNIT_OF_MEASURE"] = df["UNIT_OF_MEASURE"].apply( @@ -725,5 +729,7 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame: deduped_list.append(item) # Update the DataFrame - df.at[index, other_col] = " | ".join(deduped_list) if deduped_list else "" - return df \ No newline at end of file + df.at[index, other_col] = ( + " | ".join(deduped_list) if deduped_list else "" + ) + return df diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 585c35a..d197082 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -51,9 +51,9 @@ }, { "field_name": "UNIT_OF_MEASURE", - "relationship": "one_to_n", + "relationship": "one_to_n", "field_type": "methodology_breakout", - "prompt": "If the reimbursement method is a flat rate or follows certain fee schedules (e.g., ASA), extract the unit of measure. Otherwise, return 'N/A'. If a unit of measure is explicitly mentioned but not included in the list of provided valid values, return 'per unit'. Valid values are: {valid_values}", + "prompt": "Extract the unit of measure for flat rate or fee schedule reimbursements. Look for specific units like 'per day', 'per visit', 'per procedure', 'per ASA unit', etc. If a specific unit of measure is found and matches the valid values list, return that exact match. If a unit is mentioned but not in the valid values list, return 'Per Unit'. For flat rate methodologies where no specific unit is mentioned, default to 'Per Unit' regardless of whether procedure codes are present. For non-flat rate methodologies (percentages, bundled payments, etc.), return 'N/A'. Valid values are: {valid_values}", "valid_values": "VALID_UNIT_OF_MEASURE" }, { diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index 8ec82ee..90b03d2 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -7,6 +7,8 @@ from datetime import datetime import numpy as np import pandas as pd +from typing import overload, Literal + from constants.delimiters import Delimiter from constants.regex_patterns import ( BACKTICK_PATTERN, @@ -295,6 +297,23 @@ def count_reimbursements_in_exhibit(exhibit_text: str) -> int: # JUST by regex return len(re.findall(reimb_regex, exhibit_text)) +# Overloads for type checking +@overload +def is_empty(value: str, pd_mask: bool = True) -> bool: ... + + +@overload +def is_empty(value: list, pd_mask: bool = True) -> bool: ... + + +@overload +def is_empty(value: pd.Series, pd_mask: Literal[True] = True) -> pd.Series: ... + + +@overload +def is_empty(value: pd.Series, pd_mask: Literal[False]) -> bool: ... + + def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.Series: """ Checks if a value is considered empty or invalid. @@ -321,14 +340,32 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.S # if it's a pd.Series, return the mask (True for empty values) if isinstance(value, pd.Series): - return ( - value.isna() | (value.isin(empty_values)) if pd_mask else value.isna().all() - ) + if pd_mask: + return ( + value.isna() + | (value.isin(empty_values)) + | (value.astype(str).str.strip() == "") + | ( + value.astype(str) + .str.lower() + .str.strip() + .isin(["n/a", "na", "null", "none", "nan"]) + ) + ) + else: + return value.isna().all() if pd.isna(value): return True - else: - return value in empty_values + + if isinstance(value, str): + if not value or value.isspace(): + return True + lower_stripped = value.lower().strip() + if lower_stripped in ["n/a", "na", "null", "none", "nan"]: + return True + + return value in empty_values def datetime_str(): diff --git a/fieldExtraction/tests/test_string_utils.py b/fieldExtraction/tests/test_string_utils.py index 992a9e2..03aa19f 100644 --- a/fieldExtraction/tests/test_string_utils.py +++ b/fieldExtraction/tests/test_string_utils.py @@ -5,11 +5,15 @@ import pytest import src.utils as utils import src.utils.llm_utils as llm_utils from constants.delimiters import Delimiter -from src.utils.string_utils import (contains_reimbursement, - count_reimbursements_in_exhibit, - extract_signature_page, - extract_text_from_delimiters, is_empty, - json_parsing_search, page_key_sort) +from src.utils.string_utils import ( + contains_reimbursement, + count_reimbursements_in_exhibit, + extract_signature_page, + extract_text_from_delimiters, + is_empty, + json_parsing_search, + page_key_sort, +) class TestStringUtils: @@ -168,6 +172,17 @@ class TestStringUtils: (pd.NA, True), ("This is not empty.", False), (42, False), + (" ", True), # Whitespace-only string + ("\t", True), # Tab character + ("\n", True), # Newline character + ("n/a", True), # Lowercase N/A + ("na", True), # Lowercase NA + ("NULL", True), # Uppercase NULL + ("NONE", True), # Uppercase NONE + ("None", True), # Mixed case None (already in empty_values) + ("UNKNOWN", False), # Should NOT be empty + ("unknown", False), # Should NOT be empty + (" N/A ", True), # N/A with whitespace ], ) def test_is_empty(self, value, expected):