Merged in feature/remove-global-lesser-process (pull request #851)
Feature/remove global lesser process * Deprecate GLOBAL_LESSER_OF * remove deprecated * Remove deprecated * Merged main into feature/remove-global-lesser-process * Black formatter * remove deprecated * Remove check redundant lesser * Remove prompt_smart_chunked * Black format * Update logging Approved-by: Sha Brown
This commit is contained in:
Katon Minhas
@@ -72,28 +72,6 @@ def prompt_exhibit_level_breakout(
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_exhibit_lesser(
|
||||
exhibit_text: str,
|
||||
exhibit_level_answers: dict,
|
||||
filename: str,
|
||||
):
|
||||
logging.debug(f"Running Exhibit-Level Lesser for {filename}:")
|
||||
|
||||
prompt = prompt_templates.EXHIBIT_LEVEL_LESSER_OF(exhibit_text)
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, "legacy_sonnet", filename, max_tokens=8192
|
||||
)
|
||||
logging.debug(f"LLM raw output for {filename}: {llm_answer_raw}")
|
||||
try:
|
||||
exhibit_lesser_of_answer = string_utils.extract_text_from_delimiters(
|
||||
llm_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
except:
|
||||
exhibit_lesser_of_answer = "N/A"
|
||||
exhibit_level_answers["EXHIBIT_LESSER_OF_STATEMENT"] = exhibit_lesser_of_answer
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_dynamic_primary(
|
||||
exhibit_text: str, field: Field, constants: Constants, filename: str, TEMPLATE
|
||||
):
|
||||
@@ -399,43 +377,6 @@ def prompt_full_context(
|
||||
return full_context_answers_dict
|
||||
|
||||
|
||||
def is_exhibit_lesser_of_redundant(
|
||||
reimb_term: str, exhibit_lesser_of: str, filename: str
|
||||
) -> bool:
|
||||
"""Use LLM to determine if exhibit-level lesser-of statement is redundant with the reimbursement term.
|
||||
|
||||
This is a second step of filtering, after checking for exact substring matches.
|
||||
Small variations in punctuation may make it past the first filter, so in those cases
|
||||
where it's determined NOT to contain the substring, we use LLM to check if the
|
||||
semantic meaning is the same.
|
||||
|
||||
Args:
|
||||
reimb_term (str): The original reimbursement term.
|
||||
exhibit_lesser_of (str): The exhibit-level lesser-of statement.
|
||||
filename (str): The name of the file being processed, for logging.
|
||||
|
||||
Returns:
|
||||
bool: True if the exhibit-level lesser-of statement is redundant, False otherwise.
|
||||
"""
|
||||
|
||||
prompt = prompt_templates.CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of)
|
||||
|
||||
response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"haiku_latest",
|
||||
filename=filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.CHECK_REDUNDANT_LESSER_INSTRUCTION(),
|
||||
)
|
||||
logging.debug(f"LLM response for redundancy check in {filename}: {response}")
|
||||
return (
|
||||
string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
.strip()
|
||||
.upper()
|
||||
== "YES"
|
||||
)
|
||||
|
||||
|
||||
def validate_reimbursements_for_llm(answer_dict: dict[str, str], filename: str) -> bool:
|
||||
"""Use LLM to determine if a service has actual reimbursement methodology.
|
||||
|
||||
@@ -553,77 +494,6 @@ def prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename):
|
||||
return llm_answer_extracted
|
||||
|
||||
|
||||
def prompt_smart_chunked(
|
||||
answers_dict, new_field_groups, field_group, one_to_one_chunks, filename
|
||||
):
|
||||
"""Extract field values from chunked contract text using appropriate prompt templates.
|
||||
|
||||
Selects single-field or multi-field prompt templates based on the number of fields
|
||||
in the group and processes the corresponding text chunk to extract structured answers.
|
||||
|
||||
Args:
|
||||
answers_dict (dict): Existing answers dictionary, mutated in place with new field values.
|
||||
new_field_groups (dict): Field group definitions containing fields and their prompts.
|
||||
field_group (str): Name of the field group to process.
|
||||
one_to_one_chunks (dict): Text chunks mapped to field groups.
|
||||
filename (str): Source filename for logging and LLM attribution.
|
||||
|
||||
Returns:
|
||||
dict: The mutated answers_dict with extracted field values or error messages.
|
||||
"""
|
||||
fields = new_field_groups[field_group].get("fields", [])
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field.field_name] = field.prompt
|
||||
|
||||
context = one_to_one_chunks[field_group]
|
||||
context = context[0 : min(config.MAX_CONTEXT_LENGTH, len(context)) - 1]
|
||||
|
||||
# prepare prompt
|
||||
if len(fields) == 1:
|
||||
question = questions[
|
||||
fields[0].field_name
|
||||
] # Take the one question for the single field included
|
||||
prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
elif len(fields) > 1:
|
||||
prompt = prompt_templates.ONE_TO_ONE_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
else:
|
||||
raise ValueError("Field group has no defined fields")
|
||||
|
||||
try:
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, "sonnet_latest", filename, 8192
|
||||
)
|
||||
if len(fields) == 1:
|
||||
field_group_answer_parsed = string_utils.extract_text_from_delimiters(
|
||||
field_group_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
field_group_answer_dict = {fields[0].field_name: field_group_answer_parsed}
|
||||
elif len(fields) > 1:
|
||||
field_group_answer_dict = string_utils.json_parsing_search(
|
||||
field_group_answer_raw, [field.field_name for field in fields]
|
||||
)
|
||||
# Validate field names coming out of Claude:
|
||||
expected_fields = {field.field_name for field in fields}
|
||||
received_fields = set(field_group_answer_dict.keys())
|
||||
if expected_fields != received_fields:
|
||||
missing = expected_fields - received_fields
|
||||
extra = received_fields - expected_fields
|
||||
raise ValueError(f"Field mismatch - Missing: {missing}, Extra: {extra}")
|
||||
|
||||
# Load up answers dict
|
||||
for field in fields:
|
||||
field_answer = field_group_answer_dict[field.field_name]
|
||||
answers_dict[field.field_name] = field_answer
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing field {field}: {type(e)}, {str(e)}")
|
||||
for field in fields:
|
||||
answers_dict[field.field_name] = f"Error: {str(e)}"
|
||||
|
||||
return answers_dict
|
||||
|
||||
|
||||
def prompt_date_fix(date: str) -> str:
|
||||
"""Processes dates using `prompt_templates.date_fix_prompt`.
|
||||
|
||||
@@ -662,107 +532,6 @@ def prompt_derived_term_date(
|
||||
return string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
|
||||
|
||||
def prompt_global_lesser_of_breakout(
|
||||
global_lesser_of_stmt: str, constants: Constants, filename: str
|
||||
) -> dict[str, str]:
|
||||
"""Parse global lesser-of statement into structured methodology fields using LLM breakout.
|
||||
|
||||
This function reuses the same METHODOLOGY_BREAKOUT prompt template used for individual
|
||||
service-level reimbursement terms to extract structured fields from contract-wide lesser-of
|
||||
constraints. The extracted fields follow the same structure as methodology_breakout fields
|
||||
defined in the field JSON.
|
||||
|
||||
The function works by:
|
||||
1. Creating a mock service term "Global Lesser Of (all services)" to fit the breakout template
|
||||
2. Loading methodology_breakout fields from the field JSON configuration
|
||||
3. Invoking Claude with the METHODOLOGY_BREAKOUT prompt to parse the statement
|
||||
4. Adding "[GLOBAL LESSER OF]" prefix to REIMB_TERM for tracking/identification
|
||||
5. Returning the parsed fields or a minimal dict with just REIMB_TERM if parsing fails
|
||||
|
||||
Args:
|
||||
global_lesser_of_stmt (str): Global lesser-of statement extracted from contract.
|
||||
Example: "At no time will reimbursement exceed 100% of billed charges"
|
||||
constants (Constants): Configuration object containing valid codes and business rules
|
||||
used during methodology breakout parsing
|
||||
filename (str): Source filename (e.g., "contract_001.txt"). Used for:
|
||||
- Logging context
|
||||
- LLM request attribution
|
||||
- Error tracking
|
||||
|
||||
Returns:
|
||||
dict[str, str]: Dictionary of methodology breakout fields extracted from the statement.
|
||||
Always includes REIMB_TERM with "[GLOBAL LESSER OF]" prefix. Other fields depend on
|
||||
what's defined as field_type="methodology_breakout" in the field JSON (e.g.,
|
||||
REIMB_PCT_RATE, REIMB_FEE_RATE, UNIT_OF_MEASURE, etc.)
|
||||
|
||||
If parsing fails, returns: {"REIMB_TERM": "[GLOBAL LESSER OF] {original_statement}"}
|
||||
|
||||
Side Effects:
|
||||
- Makes LLM API call via llm_utils.invoke_claude()
|
||||
- Logs debug information about prompt and response (via logging_utils context)
|
||||
- Logs error if parsing fails but continues execution
|
||||
|
||||
Notes:
|
||||
- Uses the same METHODOLOGY_BREAKOUT template as one-to-n service-level terms
|
||||
- The mock service term "Global Lesser Of (all services)" allows the global statement
|
||||
to fit the [service, term] tuple expected by METHODOLOGY_BREAKOUT
|
||||
- Parsing failures are non-fatal - returns minimal dict to allow processing to continue
|
||||
- The "[GLOBAL LESSER OF]" prefix in REIMB_TERM distinguishes these from service-level terms
|
||||
- This function is typically called from row_funcs.apply_global_lesser_of() which uses
|
||||
the results to create duplicate rows via create_global_lesser_of_row()
|
||||
- The specific methodology fields returned depend on the field_type="methodology_breakout"
|
||||
definitions in the field JSON configuration
|
||||
|
||||
Example:
|
||||
>>> stmt = "At no time will reimbursement exceed 100% of billed charges"
|
||||
>>> constants = Constants(...)
|
||||
>>> result = run_global_lesser_of_breakout(stmt, constants, "contract.txt")
|
||||
>>> result["REIMB_TERM"]
|
||||
'[GLOBAL LESSER OF] At no time will reimbursement exceed 100% of billed charges'
|
||||
>>> result["AARETE_DERIVED_REIMB_METHOD"]
|
||||
'billed charges'
|
||||
>>> result["REIMB_PCT_RATE"]
|
||||
'100'
|
||||
>>> # Other fields depend on methodology_breakout field definitions in JSON
|
||||
"""
|
||||
|
||||
# Create a mock service term for the breakout
|
||||
mock_service = "Global Lesser Of (all services)"
|
||||
|
||||
prompt = prompt_templates.METHODOLOGY_BREAKOUT(
|
||||
[mock_service, global_lesser_of_stmt],
|
||||
FieldSet(
|
||||
file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout"
|
||||
).print_prompt_dict(constants),
|
||||
)
|
||||
logging.debug(
|
||||
f"Running global lesser of breakout for {filename} with prompt: {prompt}"
|
||||
)
|
||||
|
||||
llm_response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.METHODOLOGY_BREAKOUT_INSTRUCTION(),
|
||||
usage_label="GLOBAL_LESSER_OF_BREAKOUT",
|
||||
)
|
||||
|
||||
try:
|
||||
breakout_results = string_utils.universal_json_load(llm_response)
|
||||
result = breakout_results[0] if breakout_results else {}
|
||||
|
||||
# Add the global lesser of statement as the REIMB_TERM
|
||||
result["REIMB_TERM"] = f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"
|
||||
|
||||
return result
|
||||
except:
|
||||
logging.error(
|
||||
f"Failed to parse global lesser of breakout response for {filename}. Response: {llm_response}"
|
||||
)
|
||||
return {"REIMB_TERM": f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"}
|
||||
|
||||
|
||||
def prompt_dynamic_assignment(
|
||||
service_term: str,
|
||||
reimb_term: str,
|
||||
@@ -912,7 +681,7 @@ def prompt_lesser_of_check(
|
||||
- exhibit_reference: str | None (exhibit name if target is OTHER)
|
||||
"""
|
||||
# DEBUG: Log what we're sending
|
||||
logging.info(
|
||||
logging.debug(
|
||||
f"LESSER_OF_CHECK input: service='{service_term[:50]}...', reimb_term='{reimb_term[:100]}...'"
|
||||
)
|
||||
|
||||
|
||||
@@ -78,34 +78,6 @@ def prompt_exhibit_level_breakout(
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_exhibit_lesser(
|
||||
exhibit_text: str,
|
||||
exhibit_level_answers: dict,
|
||||
filename: str,
|
||||
):
|
||||
logging.debug(f"Running Exhibit-Level Lesser for {filename}:")
|
||||
|
||||
prompt = prompt_templates.EXHIBIT_LEVEL_LESSER_OF(exhibit_text)
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"legacy_sonnet",
|
||||
filename,
|
||||
max_tokens=8192,
|
||||
cache=True,
|
||||
instruction=prompt_templates.EXHIBIT_LEVEL_LESSER_OF_INSTRUCTION(),
|
||||
usage_label="EXHIBIT_LEVEL_LESSER_OF",
|
||||
)
|
||||
logging.debug(f"LLM raw output for {filename}: {llm_answer_raw}")
|
||||
try:
|
||||
exhibit_lesser_of_answer = string_utils.extract_text_from_delimiters(
|
||||
llm_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
except:
|
||||
exhibit_lesser_of_answer = "N/A"
|
||||
exhibit_level_answers["EXHIBIT_LESSER_OF_STATEMENT"] = exhibit_lesser_of_answer
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_dynamic_primary(
|
||||
exhibit_text: str, field: Field, constants: Constants, filename: str, TEMPLATE
|
||||
):
|
||||
@@ -439,43 +411,6 @@ def prompt_full_context(
|
||||
return full_context_answers_dict
|
||||
|
||||
|
||||
def is_exhibit_lesser_of_redundant(
|
||||
reimb_term: str, exhibit_lesser_of: str, filename: str
|
||||
) -> bool:
|
||||
"""Use LLM to determine if exhibit-level lesser-of statement is redundant with the reimbursement term.
|
||||
|
||||
This is a second step of filtering, after checking for exact substring matches.
|
||||
Small variations in punctuation may make it past the first filter, so in those cases
|
||||
where it's determined NOT to contain the substring, we use LLM to check if the
|
||||
semantic meaning is the same.
|
||||
|
||||
Args:
|
||||
reimb_term (str): The original reimbursement term.
|
||||
exhibit_lesser_of (str): The exhibit-level lesser-of statement.
|
||||
filename (str): The name of the file being processed, for logging.
|
||||
|
||||
Returns:
|
||||
bool: True if the exhibit-level lesser-of statement is redundant, False otherwise.
|
||||
"""
|
||||
|
||||
prompt = prompt_templates.CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of)
|
||||
|
||||
response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"haiku_latest",
|
||||
filename=filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.CHECK_REDUNDANT_LESSER_INSTRUCTION(),
|
||||
)
|
||||
logging.debug(f"LLM response for redundancy check in {filename}: {response}")
|
||||
return (
|
||||
string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
.strip()
|
||||
.upper()
|
||||
== "YES"
|
||||
)
|
||||
|
||||
|
||||
def validate_reimbursements_for_llm(answer_dict: dict[str, str], filename: str) -> bool:
|
||||
"""Use LLM to determine if a service has actual reimbursement methodology.
|
||||
|
||||
@@ -606,89 +541,6 @@ def prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename):
|
||||
return llm_answer_extracted
|
||||
|
||||
|
||||
def prompt_smart_chunked(
|
||||
answers_dict, new_field_groups, field_group, one_to_one_chunks, filename
|
||||
):
|
||||
"""Extract field values from chunked contract text using appropriate prompt templates.
|
||||
|
||||
Selects single-field or multi-field prompt templates based on the number of fields
|
||||
in the group and processes the corresponding text chunk to extract structured answers.
|
||||
|
||||
Args:
|
||||
answers_dict (dict): Existing answers dictionary, mutated in place with new field values.
|
||||
new_field_groups (dict): Field group definitions containing fields and their prompts.
|
||||
field_group (str): Name of the field group to process.
|
||||
one_to_one_chunks (dict): Text chunks mapped to field groups.
|
||||
filename (str): Source filename for logging and LLM attribution.
|
||||
|
||||
Returns:
|
||||
dict: The mutated answers_dict with extracted field values or error messages.
|
||||
"""
|
||||
fields = new_field_groups[field_group].get("fields", [])
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field.field_name] = field.prompt
|
||||
|
||||
context = one_to_one_chunks[field_group]
|
||||
context = context[0 : min(config.MAX_CONTEXT_LENGTH, len(context)) - 1]
|
||||
|
||||
# prepare prompt
|
||||
if len(fields) == 1:
|
||||
question = questions[
|
||||
fields[0].field_name
|
||||
] # Take the one question for the single field included
|
||||
prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
elif len(fields) > 1:
|
||||
prompt = prompt_templates.ONE_TO_ONE_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
else:
|
||||
raise ValueError("Field group has no defined fields")
|
||||
|
||||
try:
|
||||
# Use appropriate instruction based on single vs multi-field
|
||||
instruction = (
|
||||
prompt_templates.ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION()
|
||||
if len(fields) == 1
|
||||
else prompt_templates.ONE_TO_ONE_MULTI_FIELD_INSTRUCTION()
|
||||
)
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
8192,
|
||||
cache=True,
|
||||
instruction=instruction,
|
||||
usage_label="ONE_TO_ONE_CHUNKED",
|
||||
)
|
||||
if len(fields) == 1:
|
||||
field_group_answer_parsed = string_utils.extract_text_from_delimiters(
|
||||
field_group_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
field_group_answer_dict = {fields[0].field_name: field_group_answer_parsed}
|
||||
elif len(fields) > 1:
|
||||
field_group_answer_dict = string_utils.json_parsing_search(
|
||||
field_group_answer_raw, [field.field_name for field in fields]
|
||||
)
|
||||
# Validate field names coming out of Claude:
|
||||
expected_fields = {field.field_name for field in fields}
|
||||
received_fields = set(field_group_answer_dict.keys())
|
||||
if expected_fields != received_fields:
|
||||
missing = expected_fields - received_fields
|
||||
extra = received_fields - expected_fields
|
||||
raise ValueError(f"Field mismatch - Missing: {missing}, Extra: {extra}")
|
||||
|
||||
# Load up answers dict
|
||||
for field in fields:
|
||||
field_answer = field_group_answer_dict[field.field_name]
|
||||
answers_dict[field.field_name] = field_answer
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing field {field}: {type(e)}, {str(e)}")
|
||||
for field in fields:
|
||||
answers_dict[field.field_name] = f"Error: {str(e)}"
|
||||
|
||||
return answers_dict
|
||||
|
||||
|
||||
def prompt_date_fix(date: str) -> str:
|
||||
"""Processes dates using `prompt_templates.date_fix_prompt`.
|
||||
|
||||
@@ -741,107 +593,6 @@ def prompt_derived_term_date(
|
||||
return string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
|
||||
|
||||
def prompt_global_lesser_of_breakout(
|
||||
global_lesser_of_stmt: str, constants: Constants, filename: str
|
||||
) -> dict[str, str]:
|
||||
"""Parse global lesser-of statement into structured methodology fields using LLM breakout.
|
||||
|
||||
This function reuses the same METHODOLOGY_BREAKOUT prompt template used for individual
|
||||
service-level reimbursement terms to extract structured fields from contract-wide lesser-of
|
||||
constraints. The extracted fields follow the same structure as methodology_breakout fields
|
||||
defined in the field JSON.
|
||||
|
||||
The function works by:
|
||||
1. Creating a mock service term "Global Lesser Of (all services)" to fit the breakout template
|
||||
2. Loading methodology_breakout fields from the field JSON configuration
|
||||
3. Invoking Claude with the METHODOLOGY_BREAKOUT prompt to parse the statement
|
||||
4. Adding "[GLOBAL LESSER OF]" prefix to REIMB_TERM for tracking/identification
|
||||
5. Returning the parsed fields or a minimal dict with just REIMB_TERM if parsing fails
|
||||
|
||||
Args:
|
||||
global_lesser_of_stmt (str): Global lesser-of statement extracted from contract.
|
||||
Example: "At no time will reimbursement exceed 100% of billed charges"
|
||||
constants (Constants): Configuration object containing valid codes and business rules
|
||||
used during methodology breakout parsing
|
||||
filename (str): Source filename (e.g., "contract_001.txt"). Used for:
|
||||
- Logging context
|
||||
- LLM request attribution
|
||||
- Error tracking
|
||||
|
||||
Returns:
|
||||
dict[str, str]: Dictionary of methodology breakout fields extracted from the statement.
|
||||
Always includes REIMB_TERM with "[GLOBAL LESSER OF]" prefix. Other fields depend on
|
||||
what's defined as field_type="methodology_breakout" in the field JSON (e.g.,
|
||||
REIMB_PCT_RATE, REIMB_FEE_RATE, UNIT_OF_MEASURE, etc.)
|
||||
|
||||
If parsing fails, returns: {"REIMB_TERM": "[GLOBAL LESSER OF] {original_statement}"}
|
||||
|
||||
Side Effects:
|
||||
- Makes LLM API call via llm_utils.invoke_claude()
|
||||
- Logs debug information about prompt and response (via logging_utils context)
|
||||
- Logs error if parsing fails but continues execution
|
||||
|
||||
Notes:
|
||||
- Uses the same METHODOLOGY_BREAKOUT template as one-to-n service-level terms
|
||||
- The mock service term "Global Lesser Of (all services)" allows the global statement
|
||||
to fit the [service, term] tuple expected by METHODOLOGY_BREAKOUT
|
||||
- Parsing failures are non-fatal - returns minimal dict to allow processing to continue
|
||||
- The "[GLOBAL LESSER OF]" prefix in REIMB_TERM distinguishes these from service-level terms
|
||||
- This function is typically called from row_funcs.apply_global_lesser_of() which uses
|
||||
the results to create duplicate rows via create_global_lesser_of_row()
|
||||
- The specific methodology fields returned depend on the field_type="methodology_breakout"
|
||||
definitions in the field JSON configuration
|
||||
|
||||
Example:
|
||||
>>> stmt = "At no time will reimbursement exceed 100% of billed charges"
|
||||
>>> constants = Constants(...)
|
||||
>>> result = run_global_lesser_of_breakout(stmt, constants, "contract.txt")
|
||||
>>> result["REIMB_TERM"]
|
||||
'[GLOBAL LESSER OF] At no time will reimbursement exceed 100% of billed charges'
|
||||
>>> result["AARETE_DERIVED_REIMB_METHOD"]
|
||||
'billed charges'
|
||||
>>> result["REIMB_PCT_RATE"]
|
||||
'100'
|
||||
>>> # Other fields depend on methodology_breakout field definitions in JSON
|
||||
"""
|
||||
|
||||
# Create a mock service term for the breakout
|
||||
mock_service = "Global Lesser Of (all services)"
|
||||
|
||||
prompt = prompt_templates.METHODOLOGY_BREAKOUT(
|
||||
[mock_service, global_lesser_of_stmt],
|
||||
FieldSet(
|
||||
file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout"
|
||||
).print_prompt_dict(constants),
|
||||
)
|
||||
logging.debug(
|
||||
f"Running global lesser of breakout for {filename} with prompt: {prompt}"
|
||||
)
|
||||
|
||||
llm_response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.METHODOLOGY_BREAKOUT_INSTRUCTION(),
|
||||
usage_label="GLOBAL_LESSER_OF_BREAKOUT",
|
||||
)
|
||||
|
||||
try:
|
||||
breakout_results = string_utils.universal_json_load(llm_response)
|
||||
result = breakout_results[0] if breakout_results else {}
|
||||
|
||||
# Add the global lesser of statement as the REIMB_TERM
|
||||
result["REIMB_TERM"] = f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"
|
||||
|
||||
return result
|
||||
except:
|
||||
logging.error(
|
||||
f"Failed to parse global lesser of breakout response for {filename}. Response: {llm_response}"
|
||||
)
|
||||
return {"REIMB_TERM": f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"}
|
||||
|
||||
|
||||
def prompt_dynamic_assignment(
|
||||
service_term: str,
|
||||
reimb_term: str,
|
||||
@@ -991,7 +742,7 @@ def prompt_lesser_of_check(
|
||||
- exhibit_reference: str | None (exhibit name if target is OTHER)
|
||||
"""
|
||||
# DEBUG: Log what we're sending
|
||||
logging.info(
|
||||
logging.debug(
|
||||
f"LESSER_OF_CHECK input: service='{service_term[:50]}...', reimb_term='{reimb_term[:100]}...'"
|
||||
)
|
||||
|
||||
|
||||
@@ -71,41 +71,12 @@ def prompt_exhibit_level_breakout(
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt=prompt, model_id="sonnet_latest", filename=filename
|
||||
)
|
||||
print("Facility Adjustment: ", llm_answer_raw)
|
||||
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
|
||||
exhibit_level_answers.update(llm_answer_final)
|
||||
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_exhibit_lesser(
|
||||
exhibit_text: str,
|
||||
exhibit_level_answers: dict,
|
||||
filename: str,
|
||||
):
|
||||
logging.debug(f"Running Exhibit-Level Lesser for {filename}:")
|
||||
|
||||
prompt = prompt_templates.EXHIBIT_LEVEL_LESSER_OF(exhibit_text)
|
||||
llm_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"legacy_sonnet",
|
||||
filename,
|
||||
max_tokens=8192,
|
||||
cache=True,
|
||||
instruction=prompt_templates.EXHIBIT_LEVEL_LESSER_OF_INSTRUCTION(),
|
||||
usage_label="EXHIBIT_LEVEL_LESSER_OF",
|
||||
)
|
||||
logging.debug(f"LLM raw output for {filename}: {llm_answer_raw}")
|
||||
try:
|
||||
exhibit_lesser_of_answer = string_utils.extract_text_from_delimiters(
|
||||
llm_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
except:
|
||||
exhibit_lesser_of_answer = "N/A"
|
||||
exhibit_level_answers["EXHIBIT_LESSER_OF_STATEMENT"] = exhibit_lesser_of_answer
|
||||
return exhibit_level_answers
|
||||
|
||||
|
||||
def prompt_dynamic_primary(
|
||||
exhibit_text: str, field: Field, constants: Constants, filename: str, TEMPLATE
|
||||
):
|
||||
@@ -420,43 +391,6 @@ def prompt_full_context(
|
||||
return full_context_answers_dict
|
||||
|
||||
|
||||
def is_exhibit_lesser_of_redundant(
|
||||
reimb_term: str, exhibit_lesser_of: str, filename: str
|
||||
) -> bool:
|
||||
"""Use LLM to determine if exhibit-level lesser-of statement is redundant with the reimbursement term.
|
||||
|
||||
This is a second step of filtering, after checking for exact substring matches.
|
||||
Small variations in punctuation may make it past the first filter, so in those cases
|
||||
where it's determined NOT to contain the substring, we use LLM to check if the
|
||||
semantic meaning is the same.
|
||||
|
||||
Args:
|
||||
reimb_term (str): The original reimbursement term.
|
||||
exhibit_lesser_of (str): The exhibit-level lesser-of statement.
|
||||
filename (str): The name of the file being processed, for logging.
|
||||
|
||||
Returns:
|
||||
bool: True if the exhibit-level lesser-of statement is redundant, False otherwise.
|
||||
"""
|
||||
|
||||
prompt = prompt_templates.CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of)
|
||||
|
||||
response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"haiku_latest",
|
||||
filename=filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.CHECK_REDUNDANT_LESSER_INSTRUCTION(),
|
||||
)
|
||||
logging.debug(f"LLM response for redundancy check in {filename}: {response}")
|
||||
return (
|
||||
string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
.strip()
|
||||
.upper()
|
||||
== "YES"
|
||||
)
|
||||
|
||||
|
||||
def validate_reimbursements_for_llm(answer_dict: dict[str, str], filename: str) -> bool:
|
||||
"""Use LLM to determine if a service has actual reimbursement methodology.
|
||||
|
||||
@@ -587,89 +521,6 @@ def prompt_exhibit_header(page_content, EXHIBIT_HEADER_MARKERS, filename):
|
||||
return llm_answer_extracted
|
||||
|
||||
|
||||
def prompt_smart_chunked(
|
||||
answers_dict, new_field_groups, field_group, one_to_one_chunks, filename
|
||||
):
|
||||
"""Extract field values from chunked contract text using appropriate prompt templates.
|
||||
|
||||
Selects single-field or multi-field prompt templates based on the number of fields
|
||||
in the group and processes the corresponding text chunk to extract structured answers.
|
||||
|
||||
Args:
|
||||
answers_dict (dict): Existing answers dictionary, mutated in place with new field values.
|
||||
new_field_groups (dict): Field group definitions containing fields and their prompts.
|
||||
field_group (str): Name of the field group to process.
|
||||
one_to_one_chunks (dict): Text chunks mapped to field groups.
|
||||
filename (str): Source filename for logging and LLM attribution.
|
||||
|
||||
Returns:
|
||||
dict: The mutated answers_dict with extracted field values or error messages.
|
||||
"""
|
||||
fields = new_field_groups[field_group].get("fields", [])
|
||||
questions = {}
|
||||
for field in fields:
|
||||
questions[field.field_name] = field.prompt
|
||||
|
||||
context = one_to_one_chunks[field_group]
|
||||
context = context[0 : min(config.MAX_CONTEXT_LENGTH, len(context)) - 1]
|
||||
|
||||
# prepare prompt
|
||||
if len(fields) == 1:
|
||||
question = questions[
|
||||
fields[0].field_name
|
||||
] # Take the one question for the single field included
|
||||
prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(context, question)
|
||||
elif len(fields) > 1:
|
||||
prompt = prompt_templates.ONE_TO_ONE_MULTI_FIELD_TEMPLATE(context, questions)
|
||||
else:
|
||||
raise ValueError("Field group has no defined fields")
|
||||
|
||||
try:
|
||||
# Use appropriate instruction based on single vs multi-field
|
||||
instruction = (
|
||||
prompt_templates.ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION()
|
||||
if len(fields) == 1
|
||||
else prompt_templates.ONE_TO_ONE_MULTI_FIELD_INSTRUCTION()
|
||||
)
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
8192,
|
||||
cache=True,
|
||||
instruction=instruction,
|
||||
usage_label="ONE_TO_ONE_CHUNKED",
|
||||
)
|
||||
if len(fields) == 1:
|
||||
field_group_answer_parsed = string_utils.extract_text_from_delimiters(
|
||||
field_group_answer_raw, Delimiter.PIPE
|
||||
)
|
||||
field_group_answer_dict = {fields[0].field_name: field_group_answer_parsed}
|
||||
elif len(fields) > 1:
|
||||
field_group_answer_dict = string_utils.json_parsing_search(
|
||||
field_group_answer_raw, [field.field_name for field in fields]
|
||||
)
|
||||
# Validate field names coming out of Claude:
|
||||
expected_fields = {field.field_name for field in fields}
|
||||
received_fields = set(field_group_answer_dict.keys())
|
||||
if expected_fields != received_fields:
|
||||
missing = expected_fields - received_fields
|
||||
extra = received_fields - expected_fields
|
||||
raise ValueError(f"Field mismatch - Missing: {missing}, Extra: {extra}")
|
||||
|
||||
# Load up answers dict
|
||||
for field in fields:
|
||||
field_answer = field_group_answer_dict[field.field_name]
|
||||
answers_dict[field.field_name] = field_answer
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing field {field}: {type(e)}, {str(e)}")
|
||||
for field in fields:
|
||||
answers_dict[field.field_name] = f"Error: {str(e)}"
|
||||
|
||||
return answers_dict
|
||||
|
||||
|
||||
def prompt_date_fix(date: str) -> str:
|
||||
"""Processes dates using `prompt_templates.date_fix_prompt`.
|
||||
|
||||
@@ -722,107 +573,6 @@ def prompt_derived_term_date(
|
||||
return string_utils.extract_text_from_delimiters(response, Delimiter.PIPE)
|
||||
|
||||
|
||||
def prompt_global_lesser_of_breakout(
|
||||
global_lesser_of_stmt: str, constants: Constants, filename: str
|
||||
) -> dict[str, str]:
|
||||
"""Parse global lesser-of statement into structured methodology fields using LLM breakout.
|
||||
|
||||
This function reuses the same METHODOLOGY_BREAKOUT prompt template used for individual
|
||||
service-level reimbursement terms to extract structured fields from contract-wide lesser-of
|
||||
constraints. The extracted fields follow the same structure as methodology_breakout fields
|
||||
defined in the field JSON.
|
||||
|
||||
The function works by:
|
||||
1. Creating a mock service term "Global Lesser Of (all services)" to fit the breakout template
|
||||
2. Loading methodology_breakout fields from the field JSON configuration
|
||||
3. Invoking Claude with the METHODOLOGY_BREAKOUT prompt to parse the statement
|
||||
4. Adding "[GLOBAL LESSER OF]" prefix to REIMB_TERM for tracking/identification
|
||||
5. Returning the parsed fields or a minimal dict with just REIMB_TERM if parsing fails
|
||||
|
||||
Args:
|
||||
global_lesser_of_stmt (str): Global lesser-of statement extracted from contract.
|
||||
Example: "At no time will reimbursement exceed 100% of billed charges"
|
||||
constants (Constants): Configuration object containing valid codes and business rules
|
||||
used during methodology breakout parsing
|
||||
filename (str): Source filename (e.g., "contract_001.txt"). Used for:
|
||||
- Logging context
|
||||
- LLM request attribution
|
||||
- Error tracking
|
||||
|
||||
Returns:
|
||||
dict[str, str]: Dictionary of methodology breakout fields extracted from the statement.
|
||||
Always includes REIMB_TERM with "[GLOBAL LESSER OF]" prefix. Other fields depend on
|
||||
what's defined as field_type="methodology_breakout" in the field JSON (e.g.,
|
||||
REIMB_PCT_RATE, REIMB_FEE_RATE, UNIT_OF_MEASURE, etc.)
|
||||
|
||||
If parsing fails, returns: {"REIMB_TERM": "[GLOBAL LESSER OF] {original_statement}"}
|
||||
|
||||
Side Effects:
|
||||
- Makes LLM API call via llm_utils.invoke_claude()
|
||||
- Logs debug information about prompt and response (via logging_utils context)
|
||||
- Logs error if parsing fails but continues execution
|
||||
|
||||
Notes:
|
||||
- Uses the same METHODOLOGY_BREAKOUT template as one-to-n service-level terms
|
||||
- The mock service term "Global Lesser Of (all services)" allows the global statement
|
||||
to fit the [service, term] tuple expected by METHODOLOGY_BREAKOUT
|
||||
- Parsing failures are non-fatal - returns minimal dict to allow processing to continue
|
||||
- The "[GLOBAL LESSER OF]" prefix in REIMB_TERM distinguishes these from service-level terms
|
||||
- This function is typically called from row_funcs.apply_global_lesser_of() which uses
|
||||
the results to create duplicate rows via create_global_lesser_of_row()
|
||||
- The specific methodology fields returned depend on the field_type="methodology_breakout"
|
||||
definitions in the field JSON configuration
|
||||
|
||||
Example:
|
||||
>>> stmt = "At no time will reimbursement exceed 100% of billed charges"
|
||||
>>> constants = Constants(...)
|
||||
>>> result = run_global_lesser_of_breakout(stmt, constants, "contract.txt")
|
||||
>>> result["REIMB_TERM"]
|
||||
'[GLOBAL LESSER OF] At no time will reimbursement exceed 100% of billed charges'
|
||||
>>> result["AARETE_DERIVED_REIMB_METHOD"]
|
||||
'billed charges'
|
||||
>>> result["REIMB_PCT_RATE"]
|
||||
'100'
|
||||
>>> # Other fields depend on methodology_breakout field definitions in JSON
|
||||
"""
|
||||
|
||||
# Create a mock service term for the breakout
|
||||
mock_service = "Global Lesser Of (all services)"
|
||||
|
||||
prompt = prompt_templates.METHODOLOGY_BREAKOUT(
|
||||
[mock_service, global_lesser_of_stmt],
|
||||
FieldSet(
|
||||
file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout"
|
||||
).print_prompt_dict(constants),
|
||||
)
|
||||
logging.debug(
|
||||
f"Running global lesser of breakout for {filename} with prompt: {prompt}"
|
||||
)
|
||||
|
||||
llm_response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.METHODOLOGY_BREAKOUT_INSTRUCTION(),
|
||||
usage_label="GLOBAL_LESSER_OF_BREAKOUT",
|
||||
)
|
||||
|
||||
try:
|
||||
breakout_results = string_utils.universal_json_load(llm_response)
|
||||
result = breakout_results[0] if breakout_results else {}
|
||||
|
||||
# Add the global lesser of statement as the REIMB_TERM
|
||||
result["REIMB_TERM"] = f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"
|
||||
|
||||
return result
|
||||
except:
|
||||
logging.error(
|
||||
f"Failed to parse global lesser of breakout response for {filename}. Response: {llm_response}"
|
||||
)
|
||||
return {"REIMB_TERM": f"[GLOBAL LESSER OF] {global_lesser_of_stmt}"}
|
||||
|
||||
|
||||
def prompt_dynamic_assignment(
|
||||
service_term: str,
|
||||
reimb_term: str,
|
||||
@@ -972,7 +722,7 @@ def prompt_lesser_of_check(
|
||||
- exhibit_reference: str | None (exhibit name if target is OTHER)
|
||||
"""
|
||||
# DEBUG: Log what we're sending
|
||||
logging.info(
|
||||
logging.debug(
|
||||
f"LESSER_OF_CHECK input: service='{service_term[:50]}...', reimb_term='{reimb_term[:100]}...'"
|
||||
)
|
||||
|
||||
|
||||
@@ -17,59 +17,6 @@ from src.pipelines.shared.extraction.vision_funcs import get_image_array_based_a
|
||||
from src.prompts.fieldset import Field, FieldSet
|
||||
|
||||
|
||||
def extract_global_lesser_of(
|
||||
text_dict: str, first_reimbursement_page: str, filename: str
|
||||
) -> dict:
|
||||
"""Extract global "lesser of" statements that apply contract-wide.
|
||||
|
||||
Args:
|
||||
contract_text (str): The full text of the contract to process.
|
||||
filename (str): The name of the file being processed.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the extracted "lesser of" statement.
|
||||
"""
|
||||
|
||||
global_lesser_of_field = Field.load_from_file(
|
||||
file_path=config.FIELD_JSON_PATH, field_name="GLOBAL_LESSER_OF_STATEMENT"
|
||||
)
|
||||
|
||||
context = "\n".join(
|
||||
[
|
||||
page_text
|
||||
for page_num, page_text in text_dict.items()
|
||||
if int(page_num) < int(first_reimbursement_page)
|
||||
]
|
||||
)
|
||||
|
||||
# Process using existing full context logic
|
||||
prompt = prompt_templates.ONE_TO_ONE_SINGLE_FIELD_TEMPLATE(
|
||||
context,
|
||||
field_prompt=global_lesser_of_field.prompt,
|
||||
)
|
||||
|
||||
logging.debug(
|
||||
f"Extracting global lesser of statement for {filename} with prompt: {prompt}"
|
||||
)
|
||||
|
||||
response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
"sonnet_latest",
|
||||
filename,
|
||||
cache=True,
|
||||
instruction=prompt_templates.ONE_TO_ONE_SINGLE_FIELD_INSTRUCTION(),
|
||||
usage_label="ONE_TO_ONE_SINGLE_FIELD",
|
||||
)
|
||||
|
||||
logging.debug(f"Response for global lesser of statement extraction: {response}")
|
||||
return {
|
||||
"GLOBAL_LESSER_OF_STATEMENT": string_utils.extract_text_from_delimiters(
|
||||
response, Delimiter.PIPE
|
||||
)
|
||||
or "N/A"
|
||||
}
|
||||
|
||||
|
||||
def pass_smart_chunked_to_full_context(answers_dict, one_to_one_fields):
|
||||
# even after running the vision api, if the effective date is still N/A, we need to run the prompt again with full context
|
||||
if "EFFECTIVE_DT" in answers_dict and string_utils.is_empty(
|
||||
@@ -208,8 +155,6 @@ def run_full_context_fields(
|
||||
(e.g., "California" → "CA", "New York" → "NY")
|
||||
5. **Special Case Breakout**: Processes any "special_case_primary" fields that require additional
|
||||
parsing (e.g., complex reimbursement terms)
|
||||
6. **Separate Field Handling**: Processes "full_context_separate" fields individually
|
||||
(e.g., GLOBAL_LESSER_OF_STATEMENT extraction)
|
||||
|
||||
Args:
|
||||
one_to_one_fields (FieldSet): Complete set of one-to-one fields to process. The function
|
||||
@@ -224,13 +169,10 @@ def run_full_context_fields(
|
||||
dict[str, str]: Dictionary mapping field names to extracted values. Common fields include:
|
||||
- PAYER_STATE: Two-letter state abbreviation or value from config.STATE[0] if single-state batch
|
||||
- PROVIDER_STATE: Two-letter state abbreviation
|
||||
- GLOBAL_LESSER_OF_STATEMENT: Global lesser-of constraint text or "N/A"
|
||||
- Any special_case_primary fields with their breakout results appended
|
||||
|
||||
Side Effects:
|
||||
- Makes LLM API call via prompt_calls.prompt_full_context()
|
||||
- May make additional LLM calls for special case breakouts via one_to_one_breakout()
|
||||
- May make additional LLM call for GLOBAL_LESSER_OF_STATEMENT via extract_global_lesser_of()
|
||||
- Logs extraction progress and results via logging_utils context
|
||||
|
||||
Notes:
|
||||
@@ -238,7 +180,6 @@ def run_full_context_fields(
|
||||
- State normalization uses string_utils.normalize_state_to_abbreviation() which handles
|
||||
full names, abbreviations, and common misspellings
|
||||
- Special case breakout is only triggered for fields with field_type="special_case_primary"
|
||||
- GLOBAL_LESSER_OF_STATEMENT is always processed separately due to its unique extraction logic
|
||||
- Full context fields are processed together in a single LLM call for efficiency
|
||||
|
||||
Example:
|
||||
@@ -300,14 +241,6 @@ def run_full_context_fields(
|
||||
# Handle separate one-to-one fields that require individual processing
|
||||
separate_fields = one_to_one_fields.filter(field_type="full_context_separate")
|
||||
|
||||
for field in separate_fields.fields:
|
||||
if field.field_name == "GLOBAL_LESSER_OF_STATEMENT":
|
||||
# Add global lesser of extraction
|
||||
global_lesser_of_results = extract_global_lesser_of(
|
||||
text_dict, first_reimbursement_page, filename
|
||||
)
|
||||
full_context_answers_dict.update(global_lesser_of_results)
|
||||
|
||||
return full_context_answers_dict
|
||||
|
||||
|
||||
@@ -397,64 +330,6 @@ def one_to_one_breakout(
|
||||
return full_context_answers_dict
|
||||
|
||||
|
||||
def create_global_lesser_of_row(
|
||||
original_row: pd.Series, global_methodology_breakout: dict[str, str]
|
||||
) -> pd.Series:
|
||||
"""Create a duplicate reimbursement row with global lesser-of constraint applied.
|
||||
|
||||
This function supports the global lesser-of logic by creating an additional reimbursement
|
||||
row for services that didn't originally have lesser-of constraints. When a contract has
|
||||
a global lesser-of statement (e.g., "At no time will reimbursement exceed 100% of billed charges"),
|
||||
this function creates a new row that applies that constraint to existing reimbursement terms which
|
||||
lack their own lesser-of logic
|
||||
|
||||
The workflow:
|
||||
1. Copies all fields from the original row (preserving REIMB_ID, SERVICE_TERM, etc.)
|
||||
2. Overwrites methodology fields with values from global_methodology_breakout
|
||||
3. Sets LESSER_OF_IND="Y" to indicate lesser-of logic applies
|
||||
4. Adds GLOBAL_LESSER_OF_APPLIED="Y" flag for tracking
|
||||
|
||||
This allows the final output to have two rows for the same service:
|
||||
- Original row: Standard reimbursement without lesser-of constraint
|
||||
- New row: Same service with global lesser-of constraint applied
|
||||
|
||||
Args:
|
||||
original_row (pd.Series): Original reimbursement row with LESSER_OF_IND="N".
|
||||
Contains fields like REIMB_ID, SERVICE_TERM, REIMB_PCT_RATE, etc.
|
||||
global_methodology_breakout (dict[str, str]): Results from
|
||||
run_global_lesser_of_breakout() containing methodology fields to apply.
|
||||
Common fields include: REIMB_PCT_RATE, REIMB_FEE_RATE, UNIT_OF_MEASURE, etc.
|
||||
|
||||
Returns:
|
||||
pd.Series: New reimbursement row with:
|
||||
- All original fields copied (REIMB_ID, SERVICE_TERM, etc. unchanged)
|
||||
- Methodology fields updated with global_methodology_breakout values
|
||||
- LESSER_OF_IND="Y"
|
||||
- GLOBAL_LESSER_OF_APPLIED="Y"
|
||||
|
||||
Side Effects:
|
||||
None - creates a new Series without mutating the input
|
||||
|
||||
Notes:
|
||||
- The function does a shallow copy of the original row
|
||||
- Only fields present in global_methodology_breakout are overwritten
|
||||
- This function is typically called from row_funcs.apply_global_lesser_of()
|
||||
which handles the iteration over all eligible rows
|
||||
"""
|
||||
new_row = original_row.copy()
|
||||
|
||||
# Update with global lesser of methodology breakout results.
|
||||
for field, value in global_methodology_breakout.items():
|
||||
new_row[field] = value
|
||||
|
||||
# Ensure lesser of indicator is set
|
||||
new_row["LESSER_OF_IND"] = "Y"
|
||||
|
||||
# Add flag to track that this is a global lesser of row
|
||||
new_row["GLOBAL_LESSER_OF_APPLIED"] = "Y"
|
||||
return new_row
|
||||
|
||||
|
||||
def get_aarete_derived_termination_dt(one_to_one_results: dict):
|
||||
"""Derive standardized termination date from effective date and term duration.
|
||||
|
||||
|
||||
@@ -92,104 +92,4 @@ def merge_one_to_one_into_one_to_n(
|
||||
lambda row_val: str(v) if string_utils.is_empty(row_val) else row_val
|
||||
)
|
||||
|
||||
# Handle global lesser of injection
|
||||
global_lesser_of = one_to_one_results.get("GLOBAL_LESSER_OF_STATEMENT")
|
||||
if global_lesser_of and global_lesser_of not in ["N/A", "UNKNOWN"]:
|
||||
logging.debug(
|
||||
f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] Injecting global lesser of logic: {global_lesser_of}"
|
||||
)
|
||||
one_to_n_results = inject_global_lesser_of_rows(
|
||||
one_to_n_results,
|
||||
global_lesser_of,
|
||||
constants,
|
||||
one_to_one_results.get("FILE_NAME", "UNKNOWN"),
|
||||
)
|
||||
else:
|
||||
logging.debug(
|
||||
f"[{one_to_one_results.get('FILE_NAME', 'UNKNOWN')}] No global lesser of logic found or applicable."
|
||||
)
|
||||
# Initialize the tracking flag when no global lesser of logic is found
|
||||
one_to_n_results["GLOBAL_LESSER_OF_APPLIED"] = "N"
|
||||
return one_to_n_results
|
||||
|
||||
|
||||
def inject_global_lesser_of_rows(
|
||||
one_to_n_results: pd.DataFrame,
|
||||
global_lesser_of_stmt: str,
|
||||
constants: Constants,
|
||||
filename: str,
|
||||
) -> pd.DataFrame:
|
||||
"""Create new rows with global lesser of constraint for REIMB_IDs that lack lesser of logic.
|
||||
|
||||
For each row where LESSER_OF_IND="N", creates a duplicate row with the global lesser
|
||||
of statement applied via methodology breakout. The original row is preserved with
|
||||
LESSER_OF_IND updated to "Y" (since it's now subject to lesser of) and GLOBAL_LESSER_OF_APPLIED="N".
|
||||
The new row has LESSER_OF_IND="Y" and GLOBAL_LESSER_OF_APPLIED="Y". Both rows
|
||||
share the same REIMB_ID.
|
||||
|
||||
Args:
|
||||
one_to_n_results (pd.DataFrame): DataFrame containing one-to-n results with a 'LESSER_OF_IND' column.
|
||||
global_lesser_of_stmt (str): Global lesser of statement extracted from contract (e.g., "lesser of billed or allowable")
|
||||
filename (str): Name of the file being processed, used for logging and debugging.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Updated one_to_n_results with new rows injected for global lesser of logic.
|
||||
"""
|
||||
|
||||
# Sanity Check: Ensure LESSER_OF_IND is populated
|
||||
if "LESSER_OF_IND" in one_to_n_results.columns:
|
||||
# Find rows that need global lesser of injection
|
||||
rows_needing_injection = one_to_n_results[
|
||||
one_to_n_results["LESSER_OF_IND"] == "N"
|
||||
].copy()
|
||||
elif "LESSER_OF_IND" not in one_to_n_results.columns:
|
||||
if "SERVICE_TERM" not in one_to_n_results.columns:
|
||||
return one_to_n_results
|
||||
else:
|
||||
one_to_n_results["LESSER_OF_IND"] = "N"
|
||||
rows_needing_injection = one_to_n_results.copy()
|
||||
|
||||
if rows_needing_injection.empty:
|
||||
logging.debug(f"No rows needing global lesser of injection for {filename}.")
|
||||
# Still need to set tracking flag for all existing rows
|
||||
one_to_n_results["GLOBAL_LESSER_OF_APPLIED"] = "N"
|
||||
return one_to_n_results
|
||||
|
||||
# Update original rows that lacked lesser of: they're now subject to lesser of due to
|
||||
# global constraint
|
||||
one_to_n_results.loc[one_to_n_results["LESSER_OF_IND"] == "N", "LESSER_OF_IND"] = (
|
||||
"Y"
|
||||
)
|
||||
# Set tracking flag for all existing rows (both original injected rows and rows that already had lesser of logic) to "N"
|
||||
# New rows will get "Y" for this flag in create_global_lesser_of_row
|
||||
one_to_n_results["GLOBAL_LESSER_OF_APPLIED"] = "N"
|
||||
|
||||
# Run methodology breakout on global statement (once)
|
||||
global_methodology_breakout = prompt_calls.prompt_global_lesser_of_breakout(
|
||||
global_lesser_of_stmt, constants, filename
|
||||
)
|
||||
|
||||
# Create new rows for each REIMB_ID that was lacking lesser of logic
|
||||
new_rows = []
|
||||
seen_reimb_ids = set()
|
||||
for _, original_row in rows_needing_injection.iterrows():
|
||||
reimb_id = original_row.get("REIMB_ID")
|
||||
if reimb_id in seen_reimb_ids:
|
||||
continue # Skip if we've already processed this REIMB_ID
|
||||
seen_reimb_ids.add(reimb_id)
|
||||
new_row = one_to_one_funcs.create_global_lesser_of_row(
|
||||
original_row, global_methodology_breakout
|
||||
)
|
||||
new_rows.append(new_row)
|
||||
|
||||
# Append new rows to results
|
||||
if new_rows:
|
||||
new_rows_df = pd.DataFrame(new_rows)
|
||||
one_to_n_results = pd.concat([one_to_n_results, new_rows_df], ignore_index=True)
|
||||
logging.debug(f"Injected {len(new_rows)} global lesser of rows for {filename}.")
|
||||
else:
|
||||
logging.debug(
|
||||
f"No new rows created for global lesser of injection for {filename}."
|
||||
)
|
||||
|
||||
return one_to_n_results
|
||||
|
||||
@@ -55,12 +55,6 @@
|
||||
"field_type": "methodology_breakout",
|
||||
"prompt": "Write the full name of the Fee Schedule here. Include any secondary part of the fee schedule such as clinical lab, outpatient imaging, DMEPOS, encounter, surgeon, clinical diagnostic lab, parenteral and enteral nutrition (PEN), ASC, Part B Drug, Behavioral health, obstetrical, dental, transportation, etc. CHIP, STAR, and other Programs are not fee schedules. Allowed amounts or allowable charges that are not set by CMS, Medicaid, or Medicare do not constitute a fee schedule. Only use exact text from the Reimbursement Term."
|
||||
},
|
||||
{
|
||||
"field_name": "GLOBAL_LESSER_OF_STATEMENT",
|
||||
"relationship": "one_to_one",
|
||||
"field_type": "full_context_separate",
|
||||
"prompt": "Find 'Lesser of' language that applies to ALL Services in the contract. Be careful not to include 'Lesser of' phrasing that only applies to the specific Exhibit. If you find a true contract-wide global constraint, extract it exactly. Most contracts will return 'N/A' - only extract if genuinely global."
|
||||
},
|
||||
{
|
||||
"field_name": "GREATER_OF_IND",
|
||||
"relationship": "one_to_n",
|
||||
|
||||
@@ -55,9 +55,6 @@ def get_cacheable_instructions():
|
||||
if config.FIELDS in ["all", "one_to_n"]:
|
||||
try:
|
||||
cacheable_instructions["EXHIBIT_LEVEL"] = EXHIBIT_LEVEL_INSTRUCTION()
|
||||
cacheable_instructions["EXHIBIT_LEVEL_LESSER_OF"] = (
|
||||
EXHIBIT_LEVEL_LESSER_OF_INSTRUCTION()
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -67,9 +64,6 @@ def get_cacheable_instructions():
|
||||
cacheable_instructions["VALIDATE_REIMBURSEMENTS"] = (
|
||||
VALIDATE_REIMBURSEMENTS_INSTRUCTION()
|
||||
)
|
||||
cacheable_instructions["CHECK_REDUNDANT_LESSER"] = (
|
||||
CHECK_REDUNDANT_LESSER_INSTRUCTION()
|
||||
)
|
||||
cacheable_instructions["LOB_RELATIONSHIP"] = LOB_RELATIONSHIP_INSTRUCTION()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -169,53 +163,6 @@ def EXHIBIT_LEVEL_INSTRUCTION() -> str:
|
||||
)
|
||||
|
||||
|
||||
def EXHIBIT_LEVEL_LESSER_OF(context):
|
||||
return f"""Analyze a section of a contract.
|
||||
|
||||
Look for exhibit-wide payment constraints that apply universally to ALL services in this exhibit, regardless of service category or line of business.
|
||||
|
||||
Valid exhibit-wide constraints have language like:
|
||||
- 'All services under this exhibit are subject to the lesser of...'
|
||||
- 'Except as otherwise provided in this Exhibit, the Allowed Amount for Covered Services shall be the lesser of...'
|
||||
- 'Payment for any service in this exhibit shall not exceed...'
|
||||
- 'At no time shall Plan pay an amount that exceeds...'
|
||||
|
||||
DO NOT extract constraints that:
|
||||
- Apply only to specific lines of business or service categories
|
||||
- Are part of individual service methodologies
|
||||
|
||||
If you find a true exhibit-wide constraint, return the component that represents the actual payment limitation, excluding administrative processing language such as:
|
||||
- 'less any applicable co-payments, deductibles and coinsurance'
|
||||
- 'minus patient responsibility amounts'
|
||||
|
||||
**GUIDANCE FOR ABBREVIATED TERMS:**
|
||||
- 'billed' can be expanded to 'Provider's billed charges'
|
||||
- 'allowable' typically refers to amounts defined elsewhere in the contract and is not an acceptable exhibit-wide constraint due to its nonspecificity
|
||||
- Other generic terms like 'contracted rate' or 'negotiated amount' should be avoided unless they represent specific, quantifiable constraints
|
||||
- Specific percentages of external fee schedules (e.g., '105% of Medicare rates') are acceptable constraints
|
||||
|
||||
Examples:
|
||||
- 'all rates will be reimbursed at the lesser of billed or allowable' → return 'Provider's billed charges'
|
||||
- 'Lesser of 105% of Medicare rates or allowable charges' → return '105% of Medicare rates'
|
||||
- 'Lesser of Provider's charges or contracted rates' → return 'Provider's billed charges'
|
||||
- 'At no time shall payment exceed Provider's billed charges' → return 'Provider's billed charges'
|
||||
|
||||
If no exhibit-wide constraint exists, return 'N/A'.
|
||||
|
||||
Here is the Exhibit to analyze:
|
||||
{context}
|
||||
|
||||
Briefly explain your answer, then put your final answer in |pipes|
|
||||
"""
|
||||
|
||||
|
||||
def EXHIBIT_LEVEL_LESSER_OF_INSTRUCTION() -> str:
|
||||
return (
|
||||
"[OBJECTIVE]\n"
|
||||
"Identify presence of exhibit-level 'lesser of' or 'not to exceed' statements. Return answer in pipes."
|
||||
)
|
||||
|
||||
|
||||
def DYNAMIC_PRIMARY_HEADER(context, field_name, field_prompt):
|
||||
return f"""Extract attributes from the provided text, paying close attention to detail.
|
||||
|
||||
@@ -1166,40 +1113,6 @@ Contract text:
|
||||
"""
|
||||
|
||||
|
||||
def GROUP_TIN_NPI_TEMPLATE(key_sections: str, provider_list: str) -> str:
|
||||
return f"""
|
||||
Identify which provider is the MAIN CONTRACTING GROUP in this contract.
|
||||
|
||||
Below is a list of all providers found in the contract:
|
||||
{provider_list}
|
||||
|
||||
Based on the key sections of the contract (first and last parts where party definitions and signatures are usually located):
|
||||
|
||||
{key_sections.replace('"', "'")}
|
||||
|
||||
Look for these indicators to identify the main contracting group:
|
||||
1. Provider explicitly referred to as "Provider", "Group", or "Group Practice" in party definitions
|
||||
2. Provider appearing in signature blocks or with authorized signatories
|
||||
3. Provider referenced with phrases like "hereby agrees", "enters into agreement with", or "party of the first/second part"
|
||||
4. Entity described as representing multiple practitioners or locations
|
||||
5. Entity mentioned most frequently throughout the key sections
|
||||
|
||||
In case of multiple potential matches, prioritize:
|
||||
- Entities with both TIN and NPI over those with only one identifier
|
||||
- Entities described as medical groups over individual practitioners
|
||||
- Entities appearing in formal contract party definitions over those mentioned elsewhere
|
||||
|
||||
Determine which provider is the main contracting entity/group.
|
||||
Return the name, TIN, and NPI of the main group provider in JSON format like this:
|
||||
|{{"NAME": "Provider Name", "TIN": "123456789", "NPI": "1234567890"}}|
|
||||
|
||||
If you cannot confidently determine the main group, return null values:
|
||||
|{{"NAME": null, "TIN": null, "NPI": null}}|
|
||||
|
||||
Feel free to justify your answer, but enclose your final answer in |pipes|.
|
||||
"""
|
||||
|
||||
|
||||
def FULL_CONTEXT_CLAIM_TYPES_ADDITIONAL_INSTRUCTION():
|
||||
return "Additionally Check the contract text for the provider name from the preamble or signature section to help determine the claim type."
|
||||
|
||||
@@ -1500,41 +1413,6 @@ def DATE_FIX_INSTRUCTION() -> str:
|
||||
)
|
||||
|
||||
|
||||
def CHECK_REDUNDANT_LESSER(reimb_term, exhibit_lesser_of):
|
||||
return f"""You are analyzing whether an exhibit-level lesser-of constraint is semantically redundant with a reimbursement methodology.
|
||||
|
||||
REIMBURSEMENT METHODOLOGY: {reimb_term}
|
||||
|
||||
EXHIBIT CONSTRAINT: {exhibit_lesser_of}
|
||||
|
||||
Determine if the exhibit constraint is already covered by the reimbursement methodology.
|
||||
|
||||
CRITICAL: Pay careful attention to NUMERICAL DIFFERENCES. Different percentages (e.g., 90% vs 100%) are NOT redundant even if they both reference billed charges.
|
||||
|
||||
Examples of REDUNDANT language:
|
||||
- Reimb: "lesser of fee schedule or billed charges" + Exhibit: "not to exceed billed charges" → REDUNDANT
|
||||
- Reimb: "payment shall not exceed provider's charges" + Exhibit: "lesser of rate or charges" → REDUNDANT
|
||||
- Reimb: "100% of billed charges not to exceed usual and customary" + Exhibit: "not to exceed billed charges" → REDUNDANT
|
||||
|
||||
Examples of NON-REDUNDANT language:
|
||||
- Reimb: "100% of billed charges" + Exhibit: "90% of billed charges" → NOT REDUNDANT (90% is more restrictive than 100%)
|
||||
- Reimb: "billed charges not to exceed usual and customary" + Exhibit: "90% of billed charges" → NOT REDUNDANT (90% is a specific cap vs general UCR limit)
|
||||
- Reimb: "fee schedule rate" + Exhibit: "90% of billed charges" → NOT REDUNDANT (different constraint types)
|
||||
|
||||
IMPORTANT: If the exhibit constraint introduces ANY new numerical limit, percentage, or dollar amount not present in the reimbursement methodology, it is NOT redundant.
|
||||
|
||||
Answer YES only if the exhibit constraint adds NO additional limitation beyond what's already in the reimbursement methodology.
|
||||
|
||||
Briefly explain your reasoning, then return YES or NO in |pipes|."""
|
||||
|
||||
|
||||
def CHECK_REDUNDANT_LESSER_INSTRUCTION() -> str:
|
||||
return (
|
||||
"[OBJECTIVE]\n"
|
||||
"Decide if two texts are semantically redundant with respect to 'lesser of' meaning. Return YES/NO in pipes."
|
||||
)
|
||||
|
||||
|
||||
def CARVEOUT_CHECK(
|
||||
service_term: str, reimb_term: str, carveout_definitions, special_case_definitions
|
||||
):
|
||||
|
||||
@@ -1184,9 +1184,7 @@ def get_one_to_one_analysis(testbed, results):
|
||||
["PROV_GROUP_TIN", "PROV_GROUP_NPI", "PROV_GROUP_NAME_FULL"]
|
||||
)
|
||||
one_to_one_fields = [
|
||||
f
|
||||
for f in one_to_one_fields
|
||||
if f not in {"TIN", "NPI", "NAME", "CLIENT_NAME", "GLOBAL_LESSER_OF_STATEMENT"}
|
||||
f for f in one_to_one_fields if f not in {"TIN", "NPI", "NAME", "CLIENT_NAME"}
|
||||
]
|
||||
|
||||
# Create filtered dataframes with only one-to-one fields
|
||||
|
||||
@@ -391,7 +391,7 @@ def invoke_claude_with_image_array(
|
||||
base64_images,
|
||||
usage_label=usage_label,
|
||||
)
|
||||
logging.info(
|
||||
logging.debug(
|
||||
f"Effective Date Response from local Claude 3 for {filename}: {response}"
|
||||
)
|
||||
else:
|
||||
@@ -707,12 +707,12 @@ def ec2_claude_3_and_up(
|
||||
]:
|
||||
if attempt < max_retries - 1:
|
||||
delay = (2**attempt + random.uniform(0, 1)) * initial_delay
|
||||
logging.info(
|
||||
logging.warning(
|
||||
f"Attempt {attempt + 1} failed. Retrying in {delay:.2f} seconds..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
else:
|
||||
logging.info(
|
||||
logging.warning(
|
||||
f"Switching to next runtime due to {error_code}: {str(e)}."
|
||||
)
|
||||
break # Switch to the next runtime
|
||||
@@ -744,7 +744,7 @@ def ec2_claude_3_and_up(
|
||||
# Timeout errors should probably retry
|
||||
if attempt < max_retries - 1:
|
||||
delay = (2**attempt + random.uniform(0, 1)) * initial_delay
|
||||
logging.info(
|
||||
logging.warning(
|
||||
f"Attempt {attempt + 1} failed due to timeout. Retrying in {delay:.2f} seconds..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
@@ -755,7 +755,7 @@ def ec2_claude_3_and_up(
|
||||
if error_code in ["ThrottlingException", "TooManyRequestsException"]:
|
||||
if attempt < max_retries - 1:
|
||||
delay = (2**attempt + random.uniform(0, 1)) * initial_delay
|
||||
logging.info(
|
||||
logging.warning(
|
||||
f"Attempt {attempt + 1} failed due to rate limiting. Retrying in {delay:.2f} seconds..."
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
@@ -781,7 +781,6 @@ QC_YES_NO_COLUMNS = [
|
||||
"OUTLIER_FIRST_DOLLAR_IND",
|
||||
"STOP_LOSS_FIRST_DOLLAR_IND",
|
||||
"COMPOUND_SPLIT_IND",
|
||||
"GLOBAL_LESSER_OF_APPLIED",
|
||||
]
|
||||
|
||||
# Standard Line of Business values
|
||||
|
||||
Reference in New Issue
Block a user