Katon Minhas
ec4617eeee
Feature/context caching * Initial commit - context caching for DYNAMIC_PRIMARY * implement context caching for all relevant prompts * Remove option to not context cache * IndentationError fixed * Merge branch 'DEV' into feature/context-caching * Merge and format * Move documentation * Merged DEV into feature/context-caching * Update unit tests * Merged DEV into feature/context-caching * Update signatures * Fix test coverage gap Approved-by: Praneel Panchigar Approved-by: Karan Desai
991 lines
39 KiB
Python
991 lines
39 KiB
Python
"""
|
|
Comprehensive tests for prompt_calls module.
|
|
|
|
Tests that all prompt functions correctly normalize raw LLM responses according to
|
|
FIELD_FORMAT_MAPPING. This verifies that field-aware parsers work correctly with
|
|
realistic LLM outputs for all prompt types.
|
|
"""
|
|
|
|
import unittest
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
from src.pipelines.saas.prompts import prompt_calls
|
|
from src.prompts import prompt_templates
|
|
from src.prompts.fieldset import FieldSet
|
|
from src.constants.constants import Constants
|
|
from src.constants.investment_columns import FIELD_FORMAT_MAPPING
|
|
import src.config as config
|
|
|
|
|
|
class TestPromptCalls(unittest.TestCase):
|
|
"""Test that all prompt functions correctly normalize raw LLM responses."""
|
|
|
|
def setUp(self):
|
|
"""Set up test fixtures."""
|
|
self.constants = Constants()
|
|
self.filename = "test_contract.pdf"
|
|
self.exhibit_text = "Sample exhibit text for testing."
|
|
self.page_text = "Sample page text for testing."
|
|
|
|
# ==================== EXHIBIT LEVEL PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_exhibit_level_normalizes_fields(self, mock_invoke):
|
|
"""Test that prompt_exhibit_level normalizes fields based on format mapping."""
|
|
exhibit_level_fields = FieldSet(
|
|
config.FIELD_JSON_PATH, field_type="exhibit_level"
|
|
)
|
|
|
|
# Get actual field names from the FieldSet
|
|
actual_field_names = [f.field_name for f in exhibit_level_fields.fields]
|
|
|
|
# Use fields that actually exist in exhibit_level
|
|
test_fields = {}
|
|
if "CLAIM_TYPE_CD" in actual_field_names:
|
|
test_fields["CLAIM_TYPE_CD"] = (
|
|
"Inpatient" # Should normalize to ["Inpatient"]
|
|
)
|
|
if "CONTRACT_TITLE" in actual_field_names:
|
|
test_fields["CONTRACT_TITLE"] = [
|
|
"My Contract Title"
|
|
] # Should normalize to "My Contract Title"
|
|
|
|
if not test_fields:
|
|
self.skipTest("No suitable fields found in exhibit_level FieldSet")
|
|
|
|
# Simulate LLM returning values that need normalization
|
|
import json
|
|
|
|
mock_invoke.return_value = json.dumps(test_fields)
|
|
|
|
result = prompt_calls.prompt_exhibit_level(
|
|
self.exhibit_text, exhibit_level_fields, self.constants, self.filename
|
|
)
|
|
|
|
# Check each field against format mapping
|
|
for field_name, input_value in test_fields.items():
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}: {actual_value}",
|
|
)
|
|
# If input was a string, it should normalize to a list with one element
|
|
if isinstance(input_value, str):
|
|
self.assertEqual(actual_value, [input_value])
|
|
else:
|
|
self.assertEqual(actual_value, input_value)
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}: {actual_value}",
|
|
)
|
|
# If input was a list, it should normalize to a string
|
|
if isinstance(input_value, list) and len(input_value) == 1:
|
|
self.assertEqual(actual_value, str(input_value[0]))
|
|
else:
|
|
self.assertEqual(
|
|
actual_value,
|
|
(
|
|
str(input_value)
|
|
if not isinstance(input_value, str)
|
|
else input_value
|
|
),
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_exhibit_level_breakout_normalizes_fields(self, mock_invoke):
|
|
"""Test that prompt_exhibit_level_breakout normalizes facility adjustment fields."""
|
|
# Simulate LLM returning facility adjustment breakout
|
|
mock_invoke.return_value = """{
|
|
"DSH_IND": "Y",
|
|
"IME_IND": "N",
|
|
"GME_IND": "Y"
|
|
}"""
|
|
|
|
exhibit_level_answers = {"FACILITY_ADJUSTMENT_TERM": "Some term"}
|
|
result = prompt_calls.prompt_exhibit_level_breakout(
|
|
exhibit_level_answers, self.filename
|
|
)
|
|
|
|
# Should return dict with normalized fields
|
|
self.assertIsInstance(result, dict)
|
|
|
|
# Check fields against format mapping
|
|
for field_name in ["DSH_IND", "IME_IND", "GME_IND"]:
|
|
if field_name in result:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
|
|
# ==================== DYNAMIC PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_dynamic_primary_normalizes_list_str_field(self, mock_invoke):
|
|
"""Test that prompt_dynamic_primary normalizes a list[str] field correctly."""
|
|
# Simulate LLM returning a string representation of a list (common edge case)
|
|
mock_invoke.return_value = '["Medicare", "Medicaid"]'
|
|
|
|
# Create a mock field for LOB (which is list[str] in format mapping)
|
|
mock_field = MagicMock()
|
|
mock_field.field_name = "LOB"
|
|
mock_field.get_prompt.return_value = "What LOB values are present?"
|
|
|
|
result = prompt_calls.prompt_dynamic_primary(
|
|
self.exhibit_text,
|
|
mock_field,
|
|
self.constants,
|
|
self.filename,
|
|
prompt_templates.DYNAMIC_PRIMARY,
|
|
)
|
|
|
|
# Check expected format from mapping
|
|
expected_format = FIELD_FORMAT_MAPPING.get("LOB", "str")
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
result,
|
|
list,
|
|
f"LOB should be list[str] (format: {expected_format}), got {type(result)}",
|
|
)
|
|
self.assertEqual(result, ["Medicare", "Medicaid"])
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
result,
|
|
str,
|
|
f"LOB should be str (format: {expected_format}), got {type(result)}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for LOB")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_dynamic_normalizes_multiple_fields(self, mock_invoke):
|
|
"""Test that prompt_dynamic normalizes multiple fields correctly."""
|
|
# Simulate LLM returning mixed formats
|
|
mock_invoke.return_value = """{
|
|
"PRODUCT": ["Product1", "Product2"],
|
|
"PROGRAM": "Program1",
|
|
"NETWORK": ["Network1", "Network2"]
|
|
}"""
|
|
|
|
field_prompts = {
|
|
"PRODUCT": "What products?",
|
|
"PROGRAM": "What program?",
|
|
"NETWORK": "What networks?",
|
|
}
|
|
|
|
result = prompt_calls.prompt_dynamic(
|
|
self.exhibit_text, field_prompts, self.filename
|
|
)
|
|
|
|
# Check each field against format mapping
|
|
for field_name, expected_value in [
|
|
("PRODUCT", ["Product1", "Product2"]),
|
|
("PROGRAM", ["Program1"]),
|
|
("NETWORK", ["Network1", "Network2"]),
|
|
]:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(actual_value, expected_value)
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_dynamic_assignment_normalizes_field(self, mock_invoke):
|
|
"""Test that prompt_dynamic_assignment normalizes the assigned field."""
|
|
# Simulate LLM returning a dict with the field name
|
|
mock_invoke.return_value = '{"PRODUCT": ["Product1", "Product2"]}'
|
|
|
|
mock_field = MagicMock()
|
|
mock_field.field_name = "PRODUCT"
|
|
mock_field.get_prompt.return_value = "What product?"
|
|
|
|
result = prompt_calls.prompt_dynamic_assignment(
|
|
"Service Term",
|
|
"Reimbursement Term",
|
|
mock_field,
|
|
self.exhibit_text,
|
|
"1",
|
|
self.constants,
|
|
self.filename,
|
|
)
|
|
|
|
# Check PRODUCT against format mapping
|
|
expected_format = FIELD_FORMAT_MAPPING.get("PRODUCT", "str")
|
|
actual_value = result.get("PRODUCT")
|
|
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"PRODUCT should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(actual_value, ["Product1", "Product2"])
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"PRODUCT should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for PRODUCT")
|
|
|
|
# ==================== REIMBURSEMENT PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_reimbursement_primary_normalizes_service_and_reimb_terms(
|
|
self, mock_invoke
|
|
):
|
|
"""Test that prompt_reimbursement_primary normalizes SERVICE_TERM and REIMB_TERM."""
|
|
# Simulate LLM returning list of dicts
|
|
mock_invoke.return_value = """[
|
|
{"SERVICE_TERM": "Service1", "REIMB_TERM": "100% of Medicare"},
|
|
{"SERVICE_TERM": "Service2", "REIMB_TERM": "Fee Schedule"}
|
|
]"""
|
|
|
|
result = prompt_calls.prompt_reimbursement_primary(
|
|
self.page_text, self.filename
|
|
)
|
|
|
|
# Should return list of dicts
|
|
self.assertIsInstance(result, list)
|
|
self.assertEqual(len(result), 2)
|
|
|
|
# Each dict should have normalized fields with correct types and values
|
|
expected_values = [
|
|
{"SERVICE_TERM": "Service1", "REIMB_TERM": "100% of Medicare"},
|
|
{"SERVICE_TERM": "Service2", "REIMB_TERM": "Fee Schedule"},
|
|
]
|
|
|
|
for i, item in enumerate(result):
|
|
expected_item = expected_values[i]
|
|
# Check SERVICE_TERM and REIMB_TERM against format mapping
|
|
for field_name, expected_value in expected_item.items():
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = item.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(
|
|
actual_value,
|
|
expected_value,
|
|
f"{field_name} value mismatch: expected {expected_value}, got {actual_value}",
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
if isinstance(expected_value, str):
|
|
self.assertEqual(
|
|
actual_value,
|
|
[expected_value],
|
|
f"{field_name} value mismatch: expected [{expected_value}], got {actual_value}",
|
|
)
|
|
else:
|
|
self.assertEqual(
|
|
actual_value,
|
|
expected_value,
|
|
f"{field_name} value mismatch: expected {expected_value}, got {actual_value}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_reimbursement_primary_handles_no_results(self, mock_invoke):
|
|
"""Test that prompt_reimbursement_primary handles NO_REIMBURSEMENT_TERMS_FOUND."""
|
|
mock_invoke.return_value = "NO_REIMBURSEMENT_TERMS_FOUND"
|
|
|
|
result = prompt_calls.prompt_reimbursement_primary(
|
|
self.page_text, self.filename
|
|
)
|
|
|
|
# Should return empty list
|
|
self.assertEqual(result, [])
|
|
|
|
# ==================== METHODOLOGY BREAKOUT PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_methodology_breakout_normalizes_methodology_fields(
|
|
self, mock_invoke
|
|
):
|
|
"""Test that prompt_methodology_breakout normalizes methodology fields."""
|
|
# Simulate LLM returning methodology breakout with mixed formats
|
|
mock_invoke.return_value = """[
|
|
{
|
|
"LESSER_OF_IND": "Y",
|
|
"AARETE_DERIVED_REIMB_METHOD": "Fee Schedule",
|
|
"REIMB_FEE_RATE": "100.00",
|
|
"REIMB_PCT_RATE": "",
|
|
"UNIT_OF_MEASURE": "Per Unit"
|
|
}
|
|
]"""
|
|
|
|
result = prompt_calls.prompt_methodology_breakout(
|
|
"Service Term", "Reimbursement Term", self.filename
|
|
)
|
|
|
|
# Should return list of dicts
|
|
self.assertIsInstance(result, list)
|
|
self.assertEqual(len(result), 1)
|
|
|
|
# Check that fields are normalized correctly against format mapping
|
|
item = result[0]
|
|
expected_values = {
|
|
"LESSER_OF_IND": "Y",
|
|
"AARETE_DERIVED_REIMB_METHOD": "Fee Schedule",
|
|
"REIMB_FEE_RATE": "100.00",
|
|
"REIMB_PCT_RATE": "",
|
|
"UNIT_OF_MEASURE": "Per Unit",
|
|
}
|
|
|
|
for field_name, expected_value in expected_values.items():
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = item.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
# For numeric strings, allow minor variations
|
|
if (
|
|
field_name in ["REIMB_FEE_RATE", "REIMB_PCT_RATE"]
|
|
and expected_value
|
|
):
|
|
try:
|
|
expected_float = float(expected_value)
|
|
actual_float = float(actual_value)
|
|
self.assertEqual(
|
|
actual_float,
|
|
expected_float,
|
|
f"{field_name} numeric value mismatch",
|
|
)
|
|
except (ValueError, TypeError):
|
|
self.assertEqual(
|
|
actual_value, expected_value, f"{field_name} value mismatch"
|
|
)
|
|
else:
|
|
self.assertEqual(
|
|
actual_value, expected_value, f"{field_name} value mismatch"
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
if isinstance(expected_value, str):
|
|
self.assertEqual(
|
|
actual_value, [expected_value], f"{field_name} value mismatch"
|
|
)
|
|
else:
|
|
self.assertEqual(
|
|
actual_value, expected_value, f"{field_name} value mismatch"
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_fee_schedule_breakout_normalizes_fields(self, mock_invoke):
|
|
"""Test that prompt_fee_schedule_breakout normalizes fee schedule fields."""
|
|
# Simulate LLM returning fee schedule breakout
|
|
mock_invoke.return_value = """{
|
|
"FEE_SCHEDULE": "Medicare",
|
|
"FEE_SCHEDULE_VERSION": "2024"
|
|
}"""
|
|
|
|
methodology_breakout_dict = {"FEE_SCHEDULE": "Medicare"}
|
|
result = prompt_calls.prompt_fee_schedule_breakout(
|
|
methodology_breakout_dict, "Fee Schedule", self.filename
|
|
)
|
|
|
|
# Should return dict with normalized fields
|
|
self.assertIsInstance(result, dict)
|
|
|
|
# Check fields against format mapping
|
|
for field_name in ["FEE_SCHEDULE", "FEE_SCHEDULE_VERSION"]:
|
|
if field_name in result:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_grouper_breakout_normalizes_fields(self, mock_invoke):
|
|
"""Test that prompt_grouper_breakout normalizes grouper fields."""
|
|
# Simulate LLM returning grouper breakout
|
|
mock_invoke.return_value = """{
|
|
"GROUPER_TYPE": "DRG",
|
|
"GROUPER_VERSION": "2024"
|
|
}"""
|
|
|
|
result = prompt_calls.prompt_grouper_breakout(
|
|
"Service Term", "Reimbursement Term", self.filename
|
|
)
|
|
|
|
# Should return dict with normalized fields
|
|
self.assertIsInstance(result, dict)
|
|
|
|
# Check fields against format mapping
|
|
for field_name in ["GROUPER_TYPE", "GROUPER_VERSION"]:
|
|
if field_name in result:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
|
|
# ==================== SPECIAL CASE PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_carveout_check_normalizes_carveout_cd(self, mock_invoke):
|
|
"""Test that prompt_carveout_check normalizes CARVEOUT_CD to str."""
|
|
# Simulate LLM returning carveout code as list
|
|
mock_invoke.return_value = '["PAYMENT_CARVEOUT"]'
|
|
|
|
result = prompt_calls.prompt_carveout_check(
|
|
"Service Term", "Reimbursement Term", self.filename
|
|
)
|
|
|
|
# CARVEOUT_CD is str in format mapping, so should normalize list to string
|
|
expected_format = FIELD_FORMAT_MAPPING.get("CARVEOUT_CD", "str")
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
result,
|
|
str,
|
|
f"CARVEOUT_CD should be str (format: {expected_format}), got {type(result)}",
|
|
)
|
|
self.assertEqual(result, "PAYMENT_CARVEOUT")
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
result,
|
|
list,
|
|
f"CARVEOUT_CD should be list[str] (format: {expected_format}), got {type(result)}",
|
|
)
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_special_case_breakout_normalizes_fields(self, mock_invoke):
|
|
"""Test that prompt_special_case_breakout normalizes breakout fields."""
|
|
# Simulate LLM returning facility adjustment breakout
|
|
mock_invoke.return_value = """{
|
|
"DSH_IND": "Y",
|
|
"IME_IND": "N",
|
|
"GME_IND": "Y"
|
|
}"""
|
|
|
|
result = prompt_calls.prompt_special_case_breakout(
|
|
prompt_templates.FACILITY_ADJUSTMENT_BREAKOUT,
|
|
"Facility adjustment term",
|
|
self.filename,
|
|
)
|
|
|
|
# Should return dict with normalized fields
|
|
self.assertIsInstance(result, dict)
|
|
|
|
# Check fields against format mapping
|
|
expected_values = {
|
|
"DSH_IND": "Y",
|
|
"IME_IND": "N",
|
|
"GME_IND": "Y",
|
|
}
|
|
|
|
for field_name, expected_value in expected_values.items():
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(
|
|
actual_value,
|
|
expected_value,
|
|
f"{field_name} value mismatch: expected {expected_value}, got {actual_value}",
|
|
)
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
if isinstance(expected_value, str):
|
|
self.assertEqual(
|
|
actual_value,
|
|
[expected_value],
|
|
f"{field_name} value mismatch: expected [{expected_value}], got {actual_value}",
|
|
)
|
|
else:
|
|
self.assertEqual(
|
|
actual_value,
|
|
expected_value,
|
|
f"{field_name} value mismatch: expected {expected_value}, got {actual_value}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_special_case_assignment_returns_dict(self, mock_invoke):
|
|
"""Test that prompt_special_case_assignment returns a dictionary."""
|
|
# Simulate LLM returning index as list
|
|
mock_invoke.return_value = "[0]"
|
|
|
|
special_case_dicts = [
|
|
{"FACILITY_ADJUSTMENT_TERM": "Term1", "DSH_IND": "Y"},
|
|
{"FACILITY_ADJUSTMENT_TERM": "Term2", "DSH_IND": "N"},
|
|
]
|
|
|
|
result = prompt_calls.prompt_special_case_assignment(
|
|
self.exhibit_text,
|
|
{"SERVICE_TERM": "Service", "REIMB_TERM": "Reimb"},
|
|
special_case_dicts,
|
|
"FACILITY_ADJUSTMENT_TERM",
|
|
self.filename,
|
|
)
|
|
|
|
# Should return one of the special case dicts
|
|
self.assertIsInstance(result, dict)
|
|
self.assertIn("FACILITY_ADJUSTMENT_TERM", result)
|
|
|
|
# ==================== LOB RELATIONSHIP PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_lob_relationship_normalizes_to_str(self, mock_invoke):
|
|
"""Test that prompt_lob_relationship normalizes to str format."""
|
|
# Simulate LLM returning relationship as list
|
|
mock_invoke.return_value = '["Inclusive"]'
|
|
|
|
answer_dict = {
|
|
"AARETE_DERIVED_LOB": "Medicare",
|
|
"AARETE_DERIVED_PROGRAM": "STAR",
|
|
}
|
|
result = prompt_calls.prompt_lob_relationship(
|
|
answer_dict, "AARETE_DERIVED_PROGRAM", self.exhibit_text, self.filename
|
|
)
|
|
|
|
# LOB_PROGRAM_RELATIONSHIP is str in format mapping
|
|
expected_format = FIELD_FORMAT_MAPPING.get("LOB_PROGRAM_RELATIONSHIP", "str")
|
|
if expected_format == "str":
|
|
self.assertIsInstance(
|
|
result,
|
|
str,
|
|
f"LOB_PROGRAM_RELATIONSHIP should be str (format: {expected_format}), got {type(result)}",
|
|
)
|
|
self.assertIn(result, ["Inclusive", "Exclusive"])
|
|
elif expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
result,
|
|
list,
|
|
f"LOB_PROGRAM_RELATIONSHIP should be list[str] (format: {expected_format}), got {type(result)}",
|
|
)
|
|
|
|
# ==================== VALIDATION PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_validate_reimbursements_for_llm_returns_boolean(self, mock_invoke):
|
|
"""Test that validate_reimbursements_for_llm returns a boolean."""
|
|
# Simulate LLM returning YES
|
|
mock_invoke.return_value = '["YES"]'
|
|
|
|
answer_dict = {"SERVICE_TERM": "Service", "REIMB_TERM": "100% of Medicare"}
|
|
result = prompt_calls.validate_reimbursements_for_llm(
|
|
answer_dict, self.filename
|
|
)
|
|
|
|
# Should return boolean
|
|
self.assertIsInstance(result, bool)
|
|
self.assertTrue(result)
|
|
|
|
# Test with NO
|
|
mock_invoke.return_value = '["NO"]'
|
|
result = prompt_calls.validate_reimbursements_for_llm(
|
|
answer_dict, self.filename
|
|
)
|
|
self.assertIsInstance(result, bool)
|
|
self.assertFalse(result)
|
|
|
|
# ==================== EXHIBIT HELPER PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_exhibit_linkage_returns_string(self, mock_invoke):
|
|
"""Test that prompt_exhibit_linkage returns a string."""
|
|
# Simulate LLM returning YES/NO as list
|
|
mock_invoke.return_value = '["YES"]'
|
|
|
|
result = prompt_calls.prompt_exhibit_linkage(
|
|
"Exhibit A", "Exhibit B", self.filename
|
|
)
|
|
|
|
# Should return string (extracted from list)
|
|
self.assertIsInstance(result, str)
|
|
self.assertIn(result, ["YES", "NO"])
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_exhibit_header_returns_parsed_result(self, mock_invoke):
|
|
"""Test that prompt_exhibit_header returns parsed JSON from LLM output."""
|
|
# Simulate LLM returning header as list
|
|
mock_invoke.return_value = '["Exhibit A - Services"]'
|
|
|
|
result = prompt_calls.prompt_exhibit_header(
|
|
"Page content with Exhibit A header", ["Exhibit"], self.filename
|
|
)
|
|
|
|
# universal_json_load parses the JSON string into a Python object
|
|
self.assertIsInstance(result, list)
|
|
self.assertIn("Exhibit A - Services", result)
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_exhibit_title_match_returns_string(self, mock_invoke):
|
|
"""Test that prompt_exhibit_title_match returns YES/NO string."""
|
|
# Simulate LLM returning YES as list
|
|
mock_invoke.return_value = '["YES"]'
|
|
|
|
result = prompt_calls.prompt_exhibit_title_match(
|
|
"Exhibit A", "Exhibit A - Services", self.filename
|
|
)
|
|
|
|
# Should return uppercase YES/NO string
|
|
self.assertIsInstance(result, str)
|
|
self.assertIn(result, ["YES", "NO"])
|
|
|
|
# ==================== DATE PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_date_fix_returns_string(self, mock_invoke):
|
|
"""Test that prompt_date_fix returns a formatted date string."""
|
|
# Simulate LLM returning date as list
|
|
mock_invoke.return_value = '["2024/01/01"]'
|
|
|
|
result = prompt_calls.prompt_date_fix("January 1, 2024")
|
|
|
|
# Should return string (extracted from list)
|
|
self.assertIsInstance(result, str)
|
|
self.assertIn("2024", result)
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_derived_term_date_returns_string(self, mock_invoke):
|
|
"""Test that prompt_derived_term_date returns a date string."""
|
|
# Simulate LLM returning date as list
|
|
mock_invoke.return_value = '["2024/12/31"]'
|
|
|
|
result = prompt_calls.prompt_derived_term_date("2024/01/01", "12 months")
|
|
|
|
# Should return string (extracted from list)
|
|
self.assertIsInstance(result, str)
|
|
self.assertIn("2024", result)
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_split_reimb_dates_normalizes_date_fields(self, mock_invoke):
|
|
"""Test that prompt_split_reimb_dates normalizes date fields."""
|
|
# Simulate LLM returning date range
|
|
mock_invoke.return_value = (
|
|
'{"start_date": "2024/01/01", "end_date": "2024/12/31"}'
|
|
)
|
|
|
|
result = prompt_calls.prompt_split_reimb_dates(
|
|
"January 1, 2024 through December 31, 2024", self.filename
|
|
)
|
|
|
|
# Should return dict with date strings
|
|
self.assertIsInstance(result, dict)
|
|
self.assertIsInstance(result.get("start_date"), str)
|
|
self.assertIsInstance(result.get("end_date"), str)
|
|
self.assertEqual(result["start_date"], "2024/01/01")
|
|
self.assertEqual(result["end_date"], "2024/12/31")
|
|
|
|
# ==================== LESSER-OF PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_lesser_of_distribution_returns_string(self, mock_invoke):
|
|
"""Test that prompt_lesser_of_distribution returns a string."""
|
|
# Simulate LLM returning updated term as list
|
|
mock_invoke.return_value = '["lesser of $100 or billed charges"]'
|
|
|
|
result = prompt_calls.prompt_lesser_of_distribution(
|
|
"Service Term",
|
|
"Reimbursement Term",
|
|
self.exhibit_text,
|
|
"1",
|
|
[],
|
|
self.filename,
|
|
)
|
|
|
|
# Should return string (extracted from list)
|
|
self.assertIsInstance(result, str)
|
|
self.assertIn("lesser", result.lower())
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_lesser_of_check_returns_dict(self, mock_invoke):
|
|
"""Test that prompt_lesser_of_check returns a classification dict."""
|
|
# Simulate LLM returning classification
|
|
mock_invoke.return_value = """{
|
|
"service_term": "Service",
|
|
"reimb_term": "lesser of rates or charges",
|
|
"scope": "STANDALONE",
|
|
"target_exhibit": null,
|
|
"exhibit_reference": null
|
|
}"""
|
|
|
|
result = prompt_calls.prompt_lesser_of_check(
|
|
"Service Term", "Reimbursement Term", "Exhibit A", self.filename
|
|
)
|
|
|
|
# Should return dict with classification
|
|
self.assertIsInstance(result, dict)
|
|
self.assertIn("scope", result)
|
|
self.assertIn("service_term", result)
|
|
self.assertIn("reimb_term", result)
|
|
|
|
# ==================== PROVIDER PROMPTS ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_provider_name_match_check_returns_boolean(self, mock_invoke):
|
|
"""Test that provider_name_match_check returns a boolean."""
|
|
# Simulate LLM returning Y
|
|
mock_invoke.return_value = '["Y"]'
|
|
|
|
result = prompt_calls.provider_name_match_check(
|
|
"Provider Name", "Provider Name", self.filename
|
|
)
|
|
|
|
# Should return boolean
|
|
self.assertIsInstance(result, bool)
|
|
|
|
# Test with N
|
|
mock_invoke.return_value = '["N"]'
|
|
result = prompt_calls.provider_name_match_check(
|
|
"Provider Name", "Different Name", self.filename
|
|
)
|
|
self.assertIsInstance(result, bool)
|
|
self.assertFalse(result)
|
|
|
|
# ==================== EDGE CASES ====================
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_handles_llm_reasoning_text(self, mock_invoke):
|
|
"""Test that prompts handle LLM reasoning text before JSON."""
|
|
# Simulate LLM returning reasoning text before JSON (common pattern)
|
|
mock_invoke.return_value = """Let me analyze this:
|
|
|
|
Based on the contract text, I can see:
|
|
|
|
{"PRODUCT": ["Product1"], "PROGRAM": "Program1"}
|
|
|
|
This is the final answer."""
|
|
|
|
field_prompts = {
|
|
"PRODUCT": "What products?",
|
|
"PROGRAM": "What program?",
|
|
}
|
|
|
|
result = prompt_calls.prompt_dynamic(
|
|
self.exhibit_text, field_prompts, self.filename
|
|
)
|
|
|
|
# Should extract JSON and normalize correctly
|
|
self.assertIsInstance(result, dict)
|
|
|
|
# Check each field against format mapping
|
|
for field_name, expected_value in [
|
|
("PRODUCT", ["Product1"]),
|
|
("PROGRAM", ["Program1"]),
|
|
]:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(actual_value, expected_value)
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_handles_empty_and_none_values(self, mock_invoke):
|
|
"""Test that prompts handle empty and None values correctly."""
|
|
# Simulate LLM returning empty/null values
|
|
mock_invoke.return_value = """{
|
|
"PRODUCT": [],
|
|
"CONTRACT_TITLE": null,
|
|
"NETWORK": ""
|
|
}"""
|
|
|
|
field_prompts = {
|
|
"PRODUCT": "What products?",
|
|
"CONTRACT_TITLE": "What is the contract title?",
|
|
"NETWORK": "What networks?",
|
|
}
|
|
|
|
result = prompt_calls.prompt_dynamic(
|
|
self.exhibit_text, field_prompts, self.filename
|
|
)
|
|
|
|
# Check each field against format mapping
|
|
for field_name, input_value in [
|
|
("PRODUCT", []),
|
|
("CONTRACT_TITLE", None),
|
|
("NETWORK", ""),
|
|
]:
|
|
expected_format = FIELD_FORMAT_MAPPING.get(field_name, "str")
|
|
actual_value = result.get(field_name)
|
|
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
list,
|
|
f"{field_name} should be list[str] (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
self.assertEqual(actual_value, [])
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
actual_value,
|
|
str,
|
|
f"{field_name} should be str (format: {expected_format}), got {type(actual_value)}",
|
|
)
|
|
# null/None should normalize to empty string for str fields
|
|
self.assertEqual(actual_value, "")
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for {field_name}")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_dynamic_primary_handles_string_representation_of_list(
|
|
self, mock_invoke
|
|
):
|
|
"""Test that prompt_dynamic_primary handles string representation of list."""
|
|
# LLM sometimes returns: '["Value1", "Value2"]' as a string
|
|
mock_invoke.return_value = '"["Medicare", "Medicaid"]"'
|
|
|
|
mock_field = MagicMock()
|
|
mock_field.field_name = "LOB"
|
|
mock_field.get_prompt.return_value = "What LOB values are present?"
|
|
|
|
result = prompt_calls.prompt_dynamic_primary(
|
|
self.exhibit_text,
|
|
mock_field,
|
|
self.constants,
|
|
self.filename,
|
|
prompt_templates.DYNAMIC_PRIMARY,
|
|
)
|
|
|
|
# Check expected format from mapping
|
|
expected_format = FIELD_FORMAT_MAPPING.get("LOB", "str")
|
|
if expected_format == "list[str]":
|
|
self.assertIsInstance(
|
|
result,
|
|
list,
|
|
f"LOB should be list[str] (format: {expected_format}), got {type(result)}",
|
|
)
|
|
self.assertEqual(result, ["Medicare", "Medicaid"])
|
|
elif expected_format == "str":
|
|
self.assertIsInstance(
|
|
result,
|
|
str,
|
|
f"LOB should be str (format: {expected_format}), got {type(result)}",
|
|
)
|
|
else:
|
|
self.fail(f"Unexpected format {expected_format} for LOB")
|
|
|
|
@patch("src.utils.llm_utils.invoke_claude")
|
|
def test_prompt_dynamic_primary_supports_non_default_three_tuple_template(
|
|
self, mock_invoke
|
|
):
|
|
"""Test prompt_dynamic_primary supports context-aware templates in fallback branch."""
|
|
mock_invoke.return_value = '["Commercial"]'
|
|
|
|
mock_field = MagicMock()
|
|
mock_field.field_name = "LOB"
|
|
mock_field.get_prompt.return_value = "What LOB values are present?"
|
|
|
|
def custom_template(exhibit_text, field_name, field_prompt):
|
|
return (
|
|
f"[CONTEXT]\n{exhibit_text}",
|
|
f"[ATTRIBUTE TO EXTRACT]\n{field_name}: {field_prompt}",
|
|
prompt_templates._create_json_list_parser(field_name=field_name),
|
|
)
|
|
|
|
result = prompt_calls.prompt_dynamic_primary(
|
|
self.exhibit_text,
|
|
mock_field,
|
|
self.constants,
|
|
self.filename,
|
|
custom_template,
|
|
)
|
|
|
|
self.assertEqual(result, ["Commercial"])
|
|
self.assertEqual(
|
|
mock_invoke.call_args.kwargs.get("context_for_caching"),
|
|
f"[CONTEXT]\n{self.exhibit_text}",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|