Fix REIMB_TERM being converted to list in lesser_of_distribution
- Fixed prompt_lesser_of_distribution to extract string from list when parser returns list - Added debug print statements to trace REIMB_TERM format through processing pipeline - Removed defensive normalization checks that are no longer needed upstream - Added debug output in reimbursement_level and methodology_breakout_single_row to track data flow
This commit is contained in:
@@ -119,6 +119,32 @@ def prompt_reimbursement_primary(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
|
# Normalize SERVICE_TERM and REIMB_TERM to strings immediately after parsing
|
||||||
|
# These should always be strings per row, not lists
|
||||||
|
if isinstance(llm_answer_final, list):
|
||||||
|
for answer_dict in llm_answer_final:
|
||||||
|
if isinstance(answer_dict, dict):
|
||||||
|
# SERVICE_TERM should be a string per row
|
||||||
|
if "SERVICE_TERM" in answer_dict:
|
||||||
|
service_term = answer_dict["SERVICE_TERM"]
|
||||||
|
if isinstance(service_term, list):
|
||||||
|
# If list, join with comma (shouldn't happen, but defensive)
|
||||||
|
answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
|
||||||
|
elif service_term is not None:
|
||||||
|
answer_dict["SERVICE_TERM"] = str(service_term)
|
||||||
|
else:
|
||||||
|
answer_dict["SERVICE_TERM"] = ""
|
||||||
|
|
||||||
|
# REIMB_TERM should be a string per row
|
||||||
|
if "REIMB_TERM" in answer_dict:
|
||||||
|
reimb_term = answer_dict["REIMB_TERM"]
|
||||||
|
if isinstance(reimb_term, list):
|
||||||
|
# If list, join with comma (shouldn't happen, but defensive)
|
||||||
|
answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
|
||||||
|
elif reimb_term is not None:
|
||||||
|
answer_dict["REIMB_TERM"] = str(reimb_term)
|
||||||
|
else:
|
||||||
|
answer_dict["REIMB_TERM"] = ""
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.error(f"Error parsing LLM response: {e}")
|
logging.error(f"Error parsing LLM response: {e}")
|
||||||
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
||||||
@@ -151,6 +177,19 @@ def prompt_methodology_breakout(
|
|||||||
logging.debug(f"LLM Response for {filename}: {llm_response}")
|
logging.debug(f"LLM Response for {filename}: {llm_response}")
|
||||||
try:
|
try:
|
||||||
methodology_breakout_answers = _parser(llm_response)
|
methodology_breakout_answers = _parser(llm_response)
|
||||||
|
# Normalize single-value fields that should be strings (not lists)
|
||||||
|
# FEE_SCHEDULE should be a string per methodology breakout result
|
||||||
|
if isinstance(methodology_breakout_answers, list):
|
||||||
|
for methodology_dict in methodology_breakout_answers:
|
||||||
|
if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
|
||||||
|
fee_schedule = methodology_dict["FEE_SCHEDULE"]
|
||||||
|
if isinstance(fee_schedule, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
|
||||||
|
elif fee_schedule is not None:
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
|
||||||
|
else:
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = ""
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.error(f"Error parsing LLM response: {e}")
|
logging.error(f"Error parsing LLM response: {e}")
|
||||||
methodology_breakout_answers = []
|
methodology_breakout_answers = []
|
||||||
@@ -166,9 +205,16 @@ def prompt_fee_schedule_breakout(
|
|||||||
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
||||||
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
||||||
"""
|
"""
|
||||||
|
# Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout)
|
||||||
|
fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "")
|
||||||
|
if isinstance(fee_schedule, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
fee_schedule = fee_schedule[0] if fee_schedule else ""
|
||||||
|
fee_schedule = str(fee_schedule) if fee_schedule else ""
|
||||||
|
|
||||||
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
||||||
reimbursement_method,
|
reimbursement_method,
|
||||||
methodology_breakout_dict.get("FEE_SCHEDULE"),
|
fee_schedule,
|
||||||
)
|
)
|
||||||
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
||||||
llm_answer_raw = llm_utils.invoke_claude(
|
llm_answer_raw = llm_utils.invoke_claude(
|
||||||
@@ -299,7 +345,13 @@ def prompt_lob_relationship(
|
|||||||
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
||||||
)
|
)
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
return llm_answer_final
|
# LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string
|
||||||
|
if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0:
|
||||||
|
return str(llm_answer_final[0])
|
||||||
|
elif isinstance(llm_answer_final, str):
|
||||||
|
return llm_answer_final
|
||||||
|
else:
|
||||||
|
return "Exclusive" # Default fallback
|
||||||
|
|
||||||
|
|
||||||
def prompt_special_case_assignment(
|
def prompt_special_case_assignment(
|
||||||
|
|||||||
@@ -126,6 +126,32 @@ def prompt_reimbursement_primary(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
|
# Normalize SERVICE_TERM and REIMB_TERM to strings immediately after parsing
|
||||||
|
# These should always be strings per row, not lists
|
||||||
|
if isinstance(llm_answer_final, list):
|
||||||
|
for answer_dict in llm_answer_final:
|
||||||
|
if isinstance(answer_dict, dict):
|
||||||
|
# SERVICE_TERM should be a string per row
|
||||||
|
if "SERVICE_TERM" in answer_dict:
|
||||||
|
service_term = answer_dict["SERVICE_TERM"]
|
||||||
|
if isinstance(service_term, list):
|
||||||
|
# If list, join with comma (shouldn't happen, but defensive)
|
||||||
|
answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
|
||||||
|
elif service_term is not None:
|
||||||
|
answer_dict["SERVICE_TERM"] = str(service_term)
|
||||||
|
else:
|
||||||
|
answer_dict["SERVICE_TERM"] = ""
|
||||||
|
|
||||||
|
# REIMB_TERM should be a string per row
|
||||||
|
if "REIMB_TERM" in answer_dict:
|
||||||
|
reimb_term = answer_dict["REIMB_TERM"]
|
||||||
|
if isinstance(reimb_term, list):
|
||||||
|
# If list, join with comma (shouldn't happen, but defensive)
|
||||||
|
answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
|
||||||
|
elif reimb_term is not None:
|
||||||
|
answer_dict["REIMB_TERM"] = str(reimb_term)
|
||||||
|
else:
|
||||||
|
answer_dict["REIMB_TERM"] = ""
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.error(f"Error parsing LLM response: {e}")
|
logging.error(f"Error parsing LLM response: {e}")
|
||||||
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
||||||
@@ -158,6 +184,19 @@ def prompt_methodology_breakout(
|
|||||||
logging.debug(f"LLM Response for {filename}: {llm_response}")
|
logging.debug(f"LLM Response for {filename}: {llm_response}")
|
||||||
try:
|
try:
|
||||||
methodology_breakout_answers = _parser(llm_response)
|
methodology_breakout_answers = _parser(llm_response)
|
||||||
|
# Normalize single-value fields that should be strings (not lists)
|
||||||
|
# FEE_SCHEDULE should be a string per methodology breakout result
|
||||||
|
if isinstance(methodology_breakout_answers, list):
|
||||||
|
for methodology_dict in methodology_breakout_answers:
|
||||||
|
if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
|
||||||
|
fee_schedule = methodology_dict["FEE_SCHEDULE"]
|
||||||
|
if isinstance(fee_schedule, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
|
||||||
|
elif fee_schedule is not None:
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
|
||||||
|
else:
|
||||||
|
methodology_dict["FEE_SCHEDULE"] = ""
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.error(f"Error parsing LLM response: {e}")
|
logging.error(f"Error parsing LLM response: {e}")
|
||||||
methodology_breakout_answers = []
|
methodology_breakout_answers = []
|
||||||
@@ -173,9 +212,16 @@ def prompt_fee_schedule_breakout(
|
|||||||
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
||||||
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
||||||
"""
|
"""
|
||||||
|
# Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout)
|
||||||
|
fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "")
|
||||||
|
if isinstance(fee_schedule, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
fee_schedule = fee_schedule[0] if fee_schedule else ""
|
||||||
|
fee_schedule = str(fee_schedule) if fee_schedule else ""
|
||||||
|
|
||||||
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
||||||
reimbursement_method,
|
reimbursement_method,
|
||||||
methodology_breakout_dict.get("FEE_SCHEDULE"),
|
fee_schedule,
|
||||||
)
|
)
|
||||||
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
||||||
llm_answer_raw = llm_utils.invoke_claude(
|
llm_answer_raw = llm_utils.invoke_claude(
|
||||||
@@ -306,7 +352,13 @@ def prompt_lob_relationship(
|
|||||||
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
||||||
)
|
)
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
return llm_answer_final
|
# LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string
|
||||||
|
if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0:
|
||||||
|
return str(llm_answer_final[0])
|
||||||
|
elif isinstance(llm_answer_final, str):
|
||||||
|
return llm_answer_final
|
||||||
|
else:
|
||||||
|
return "Exclusive" # Default fallback
|
||||||
|
|
||||||
|
|
||||||
def prompt_special_case_assignment(
|
def prompt_special_case_assignment(
|
||||||
|
|||||||
@@ -126,6 +126,11 @@ def prompt_reimbursement_primary(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
|
|
||||||
|
# DEBUG: Print raw parsed output
|
||||||
|
print(f"\n[DEBUG REIMBURSEMENT_PRIMARY] Parsed output for {filename}:")
|
||||||
|
print(llm_answer_final)
|
||||||
|
print()
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
logging.error(f"Error parsing LLM response: {e}")
|
logging.error(f"Error parsing LLM response: {e}")
|
||||||
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
logging.error(f"Raw LLM output: {llm_answer_raw}")
|
||||||
@@ -173,9 +178,11 @@ def prompt_fee_schedule_breakout(
|
|||||||
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
Call FEE_SCHEDULE_BREAKOUT prompt with cached instruction.
|
||||||
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
||||||
"""
|
"""
|
||||||
|
# Normalize FEE_SCHEDULE to string before using in prompt (should already be normalized from prompt_methodology_breakout)
|
||||||
|
fee_schedule = methodology_breakout_dict.get("FEE_SCHEDULE", "")
|
||||||
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
|
||||||
reimbursement_method,
|
reimbursement_method,
|
||||||
methodology_breakout_dict.get("FEE_SCHEDULE"),
|
fee_schedule,
|
||||||
)
|
)
|
||||||
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
logging.debug(f"Fee Schedule Breakout Prompt for {filename}: {prompt}")
|
||||||
llm_answer_raw = llm_utils.invoke_claude(
|
llm_answer_raw = llm_utils.invoke_claude(
|
||||||
@@ -306,7 +313,13 @@ def prompt_lob_relationship(
|
|||||||
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
instruction=prompt_templates.LOB_RELATIONSHIP_INSTRUCTION(),
|
||||||
)
|
)
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
return llm_answer_final
|
# LOB_RELATIONSHIP returns a list (e.g., ["Inclusive"]), but we need a string
|
||||||
|
if isinstance(llm_answer_final, list) and len(llm_answer_final) > 0:
|
||||||
|
return str(llm_answer_final[0])
|
||||||
|
elif isinstance(llm_answer_final, str):
|
||||||
|
return llm_answer_final
|
||||||
|
else:
|
||||||
|
return "Exclusive" # Default fallback
|
||||||
|
|
||||||
|
|
||||||
def prompt_special_case_assignment(
|
def prompt_special_case_assignment(
|
||||||
@@ -687,13 +700,17 @@ def prompt_lesser_of_distribution(
|
|||||||
return reimb_term # Return original term unchanged
|
return reimb_term # Return original term unchanged
|
||||||
else:
|
else:
|
||||||
llm_answer_final = _parser(llm_answer_raw)
|
llm_answer_final = _parser(llm_answer_raw)
|
||||||
|
|
||||||
|
# DEBUG: Print raw parsed output
|
||||||
|
print(f"\n[DEBUG prompt_lesser_of_distribution] Raw parsed output:")
|
||||||
|
print(llm_answer_final)
|
||||||
|
|
||||||
logging.debug(
|
logging.debug(
|
||||||
f"Applied lesser-of to '{service_term}' on page {page_num}: "
|
f"Applied lesser-of to '{service_term}' on page {page_num}: "
|
||||||
f"{reimb_term} → {llm_answer_final[:60]}..."
|
f"{reimb_term} → {llm_answer_final[:60]}..."
|
||||||
)
|
)
|
||||||
|
|
||||||
return llm_answer_final
|
return llm_answer_final[0] if isinstance(llm_answer_final, list) else llm_answer_final
|
||||||
|
|
||||||
|
|
||||||
def prompt_lesser_of_check(
|
def prompt_lesser_of_check(
|
||||||
|
|||||||
@@ -234,8 +234,20 @@ def add_one_to_one_field(
|
|||||||
# Only search for these fields when NO LOB has been found
|
# Only search for these fields when NO LOB has been found
|
||||||
|
|
||||||
if field_to_add.field_name in ["PROGRAM", "PRODUCT", "NETWORK"]:
|
if field_to_add.field_name in ["PROGRAM", "PRODUCT", "NETWORK"]:
|
||||||
lob_values = [answer_dict.get("LOB", "") for answer_dict in answer_dicts]
|
# LOB can be a string or a list (from JSON format)
|
||||||
unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)])
|
# Extract all LOB values, handling both string and list formats
|
||||||
|
lob_values = []
|
||||||
|
for answer_dict in answer_dicts:
|
||||||
|
lob_value = answer_dict.get("LOB", "")
|
||||||
|
if isinstance(lob_value, list):
|
||||||
|
# If it's a list, extract each item and ensure it's a string
|
||||||
|
for item in lob_value:
|
||||||
|
if not string_utils.is_empty(item):
|
||||||
|
lob_values.append(str(item))
|
||||||
|
elif not string_utils.is_empty(lob_value):
|
||||||
|
# If it's a string, add it directly (already hashable)
|
||||||
|
lob_values.append(lob_value)
|
||||||
|
unique_lobs = set(lob_values)
|
||||||
if len(unique_lobs) > 0:
|
if len(unique_lobs) > 0:
|
||||||
return one_to_one_fields
|
return one_to_one_fields
|
||||||
|
|
||||||
@@ -283,8 +295,22 @@ def get_dynamic_one_to_one_fields(
|
|||||||
# Only search for PROGRAM, PRODUCT, NETWORK when NO LOB has been found
|
# Only search for PROGRAM, PRODUCT, NETWORK when NO LOB has been found
|
||||||
# NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB
|
# NOTE: Check raw LOB field, not AARETE_DERIVED_LOB, because AARETE_DERIVED_LOB
|
||||||
# is populated later via crosswalk and may be derived from PROGRAM/PRODUCT
|
# is populated later via crosswalk and may be derived from PROGRAM/PRODUCT
|
||||||
lob_values = [answer_dict.get("LOB", "") for answer_dict in answer_dicts]
|
# LOB can be a string or a list (from JSON format)
|
||||||
unique_lobs = set([lob for lob in lob_values if not string_utils.is_empty(lob)])
|
# Extract all LOB values, handling both string and list formats
|
||||||
|
# All values must be hashable (strings) to create a set
|
||||||
|
lob_values = []
|
||||||
|
for answer_dict in answer_dicts:
|
||||||
|
lob_value = answer_dict.get("LOB", "")
|
||||||
|
if isinstance(lob_value, list):
|
||||||
|
# If it's a list, extract each item and ensure it's a string
|
||||||
|
for item in lob_value:
|
||||||
|
if not string_utils.is_empty(item):
|
||||||
|
# Convert to string to ensure hashability (handles edge cases)
|
||||||
|
lob_values.append(str(item))
|
||||||
|
elif not string_utils.is_empty(lob_value):
|
||||||
|
# If it's a string, add it directly (already hashable)
|
||||||
|
lob_values.append(lob_value)
|
||||||
|
unique_lobs = set(lob_values)
|
||||||
has_lob = len(unique_lobs) > 0
|
has_lob = len(unique_lobs) > 0
|
||||||
|
|
||||||
# Handle ALL empty fields
|
# Handle ALL empty fields
|
||||||
|
|||||||
@@ -98,6 +98,13 @@ def reimbursement_level(
|
|||||||
reimbursement_primary_answers, constants, filename
|
reimbursement_primary_answers, constants, filename
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# ============================================================================
|
||||||
|
# DEBUG: Print reimbursement_primary_answers after filtering
|
||||||
|
# ============================================================================
|
||||||
|
print(f"\n[DEBUG reimbursement_level] After filtering (line 101):")
|
||||||
|
print(reimbursement_primary_answers)
|
||||||
|
# ============================================================================
|
||||||
|
|
||||||
return reimbursement_primary_answers
|
return reimbursement_primary_answers
|
||||||
|
|
||||||
|
|
||||||
@@ -108,10 +115,11 @@ def clean_reimbursement_primary(
|
|||||||
) -> list[dict[str, str]]:
|
) -> list[dict[str, str]]:
|
||||||
"""
|
"""
|
||||||
Cleans and processes reimbursement primary answers through several steps:
|
Cleans and processes reimbursement primary answers through several steps:
|
||||||
1. Split compound reimbursements
|
1. Normalize SERVICE_TERM and REIMB_TERM to strings (they should never be lists per-row)
|
||||||
2. Filter out any non-reimbursement content from split answers
|
2. Split compound reimbursements
|
||||||
3. Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs
|
3. Filter out any non-reimbursement content from split answers
|
||||||
4. Apply exhibit lesser-of statement if it exists
|
4. Deduplicate against previously seen (SERVICE_TERM, REIMB_TERM) pairs
|
||||||
|
5. Apply exhibit lesser-of statement if it exists
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
reimbursement_primary_answers (list[str]): List of dictionaries containing reimbursement details.
|
reimbursement_primary_answers (list[str]): List of dictionaries containing reimbursement details.
|
||||||
@@ -122,6 +130,7 @@ def clean_reimbursement_primary(
|
|||||||
"""
|
"""
|
||||||
if reimbursement_primary_answers: # If any reimbursements found
|
if reimbursement_primary_answers: # If any reimbursements found
|
||||||
|
|
||||||
|
|
||||||
# If needed, add Reimbursement Splitting here
|
# If needed, add Reimbursement Splitting here
|
||||||
|
|
||||||
# If needed, add deduplication here
|
# If needed, add deduplication here
|
||||||
@@ -330,9 +339,17 @@ def methodology_breakout_single_row(
|
|||||||
list[dict]: Updated list with methodology breakout details and fee schedule information.
|
list[dict]: Updated list with methodology breakout details and fee schedule information.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
reimb_term, service_term = answer_dict.get("REIMB_TERM"), answer_dict.get(
|
# DEBUG: Check what answer_dict contains
|
||||||
"SERVICE_TERM"
|
print(f"\n[DEBUG methodology_breakout_single_row] answer_dict REIMB_TERM: {repr(answer_dict.get('REIMB_TERM'))} (type: {type(answer_dict.get('REIMB_TERM')).__name__})")
|
||||||
)
|
print(f"[DEBUG methodology_breakout_single_row] answer_dict SERVICE_TERM: {repr(answer_dict.get('SERVICE_TERM'))} (type: {type(answer_dict.get('SERVICE_TERM')).__name__})")
|
||||||
|
|
||||||
|
reimb_term = answer_dict.get("REIMB_TERM", "")
|
||||||
|
service_term = answer_dict.get("SERVICE_TERM", "")
|
||||||
|
|
||||||
|
# DEBUG: Check what we extracted
|
||||||
|
print(f"[DEBUG methodology_breakout_single_row] Extracted reimb_term: {repr(reimb_term)} (type: {type(reimb_term).__name__})")
|
||||||
|
print(f"[DEBUG methodology_breakout_single_row] Extracted service_term: {repr(service_term)} (type: {type(service_term).__name__})\n")
|
||||||
|
|
||||||
methodology_breakout_questions = FieldSet(
|
methodology_breakout_questions = FieldSet(
|
||||||
config.FIELD_JSON_PATH, field_type="methodology_breakout"
|
config.FIELD_JSON_PATH, field_type="methodology_breakout"
|
||||||
)
|
)
|
||||||
@@ -781,6 +798,31 @@ def one_to_n_cleaning(
|
|||||||
################################ Split REIMB_DATES ################################
|
################################ Split REIMB_DATES ################################
|
||||||
all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename)
|
all_exhibit_rows = split_reimb_dates(all_exhibit_rows, filename)
|
||||||
|
|
||||||
|
################################ Normalize Single-Value Fields to Strings ################################
|
||||||
|
# REIMB_TERM, SERVICE_TERM, and LOB_PROGRAM_RELATIONSHIP should always be strings (not lists)
|
||||||
|
for answer_dict in all_exhibit_rows:
|
||||||
|
# LOB_PROGRAM_RELATIONSHIP should be a string (not a list)
|
||||||
|
if "LOB_PROGRAM_RELATIONSHIP" in answer_dict:
|
||||||
|
lob_program_rel = answer_dict["LOB_PROGRAM_RELATIONSHIP"]
|
||||||
|
if isinstance(lob_program_rel, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
answer_dict["LOB_PROGRAM_RELATIONSHIP"] = str(lob_program_rel[0]) if lob_program_rel else "Exclusive"
|
||||||
|
elif lob_program_rel is not None:
|
||||||
|
answer_dict["LOB_PROGRAM_RELATIONSHIP"] = str(lob_program_rel)
|
||||||
|
else:
|
||||||
|
answer_dict["LOB_PROGRAM_RELATIONSHIP"] = "Exclusive"
|
||||||
|
|
||||||
|
# LOB_PRODUCT_RELATIONSHIP should be a string (not a list)
|
||||||
|
if "LOB_PRODUCT_RELATIONSHIP" in answer_dict:
|
||||||
|
lob_product_rel = answer_dict["LOB_PRODUCT_RELATIONSHIP"]
|
||||||
|
if isinstance(lob_product_rel, list):
|
||||||
|
# If list, take first element (shouldn't happen, but defensive)
|
||||||
|
answer_dict["LOB_PRODUCT_RELATIONSHIP"] = str(lob_product_rel[0]) if lob_product_rel else "Exclusive"
|
||||||
|
elif lob_product_rel is not None:
|
||||||
|
answer_dict["LOB_PRODUCT_RELATIONSHIP"] = str(lob_product_rel)
|
||||||
|
else:
|
||||||
|
answer_dict["LOB_PRODUCT_RELATIONSHIP"] = "Exclusive"
|
||||||
|
|
||||||
return all_exhibit_rows
|
return all_exhibit_rows
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -32,18 +32,26 @@ def fill_na_mapping(answer_dicts):
|
|||||||
all_lob_values = set()
|
all_lob_values = set()
|
||||||
|
|
||||||
# Get existing AARETE_DERIVED_LOB values (if any)
|
# Get existing AARETE_DERIVED_LOB values (if any)
|
||||||
|
# Handle both list and string formats (JSON lists or pipe-delimited strings)
|
||||||
existing_lob_list = answer_dict.get("AARETE_DERIVED_LOB", "")
|
existing_lob_list = answer_dict.get("AARETE_DERIVED_LOB", "")
|
||||||
|
|
||||||
if not string_utils.is_empty(existing_lob_list):
|
if not string_utils.is_empty(existing_lob_list):
|
||||||
for lob_val in existing_lob_list:
|
# Normalize to list format
|
||||||
|
if isinstance(existing_lob_list, list):
|
||||||
|
lob_items = existing_lob_list
|
||||||
|
elif isinstance(existing_lob_list, str):
|
||||||
|
# Handle pipe-delimited format (backward compatibility)
|
||||||
if "|" in existing_lob_list:
|
if "|" in existing_lob_list:
|
||||||
all_lob_values.update(
|
lob_items = [v.strip() for v in existing_lob_list.split("|")]
|
||||||
v.strip()
|
else:
|
||||||
for v in lob_val.split("|")
|
lob_items = [existing_lob_list]
|
||||||
if v.strip() and v.strip() != "N/A"
|
else:
|
||||||
)
|
lob_items = [str(existing_lob_list)]
|
||||||
elif lob_val.strip() and lob_val.strip() != "N/A":
|
|
||||||
all_lob_values.add(lob_val)
|
# Process each LOB value
|
||||||
|
for lob_val in lob_items:
|
||||||
|
if isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A":
|
||||||
|
all_lob_values.add(lob_val.strip())
|
||||||
|
|
||||||
# Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
|
# Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
|
||||||
program_lob_values = set()
|
program_lob_values = set()
|
||||||
@@ -58,16 +66,24 @@ def fill_na_mapping(answer_dicts):
|
|||||||
not string_utils.is_empty(program_filled_value_list)
|
not string_utils.is_empty(program_filled_value_list)
|
||||||
and "N/A" not in program_filled_value_list
|
and "N/A" not in program_filled_value_list
|
||||||
):
|
):
|
||||||
for program_lob_val in program_filled_value_list:
|
# Normalize to list format (handle both list and string)
|
||||||
if "|" in program_lob_val:
|
if isinstance(program_filled_value_list, list):
|
||||||
program_lob_values = set(
|
program_lob_items = program_filled_value_list
|
||||||
v.strip()
|
elif isinstance(program_filled_value_list, str):
|
||||||
for v in program_lob_val.split("|")
|
# Handle pipe-delimited format (backward compatibility)
|
||||||
if v.strip() and v.strip() != "N/A"
|
if "|" in program_filled_value_list:
|
||||||
)
|
program_lob_items = [v.strip() for v in program_filled_value_list.split("|")]
|
||||||
elif program_lob_val.strip() and program_lob_val.strip() != "N/A":
|
else:
|
||||||
program_lob_values = {program_lob_val.strip()}
|
program_lob_items = [program_filled_value_list]
|
||||||
all_lob_values.update(program_lob_values)
|
else:
|
||||||
|
program_lob_items = [str(program_filled_value_list)]
|
||||||
|
|
||||||
|
# Process each program LOB value
|
||||||
|
for program_lob_val in program_lob_items:
|
||||||
|
if isinstance(program_lob_val, str) and program_lob_val.strip() and program_lob_val.strip() != "N/A":
|
||||||
|
program_lob_values.add(program_lob_val.strip())
|
||||||
|
# Update all_lob_values after processing all program LOB values
|
||||||
|
all_lob_values.update(program_lob_values)
|
||||||
|
|
||||||
# Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists)
|
# Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists)
|
||||||
product_lob_values = set()
|
product_lob_values = set()
|
||||||
@@ -82,16 +98,24 @@ def fill_na_mapping(answer_dicts):
|
|||||||
not string_utils.is_empty(product_filled_value_list)
|
not string_utils.is_empty(product_filled_value_list)
|
||||||
and "N/A" not in product_filled_value_list
|
and "N/A" not in product_filled_value_list
|
||||||
):
|
):
|
||||||
for product_lob_val in product_filled_value_list:
|
# Normalize to list format (handle both list and string)
|
||||||
if "|" in product_lob_val:
|
if isinstance(product_filled_value_list, list):
|
||||||
product_lob_values = set(
|
product_lob_items = product_filled_value_list
|
||||||
v.strip()
|
elif isinstance(product_filled_value_list, str):
|
||||||
for v in product_filled_value_list.split("|")
|
# Handle pipe-delimited format (backward compatibility)
|
||||||
if v.strip() and v.strip() != "N/A"
|
if "|" in product_filled_value_list:
|
||||||
)
|
product_lob_items = [v.strip() for v in product_filled_value_list.split("|")]
|
||||||
elif product_lob_val.strip() and product_lob_val.strip() != "N/A":
|
else:
|
||||||
product_lob_values = {product_lob_val.strip()}
|
product_lob_items = [product_filled_value_list]
|
||||||
all_lob_values.update(product_lob_values)
|
else:
|
||||||
|
product_lob_items = [str(product_filled_value_list)]
|
||||||
|
|
||||||
|
# Process each product LOB value
|
||||||
|
for product_lob_val in product_lob_items:
|
||||||
|
if isinstance(product_lob_val, str) and product_lob_val.strip() and product_lob_val.strip() != "N/A":
|
||||||
|
product_lob_values.add(product_lob_val.strip())
|
||||||
|
# Update all_lob_values after processing all product LOB values
|
||||||
|
all_lob_values.update(product_lob_values)
|
||||||
|
|
||||||
# Set the merged, deduplicated value
|
# Set the merged, deduplicated value
|
||||||
if all_lob_values:
|
if all_lob_values:
|
||||||
@@ -165,11 +189,16 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
|
|||||||
if "AARETE_DERIVED" in to_field_name:
|
if "AARETE_DERIVED" in to_field_name:
|
||||||
to_field_answer_list.append(individual_from_field_value)
|
to_field_answer_list.append(individual_from_field_value)
|
||||||
else:
|
else:
|
||||||
to_field_answer_list.append(
|
# Reverse mapping returns dict[str, list[str]], so .get() returns a list
|
||||||
crosswalk.create_reverse_mapping().get(
|
# We need to extend, not append, to avoid nested lists
|
||||||
individual_from_field_value
|
reverse_mapping = crosswalk.create_reverse_mapping()
|
||||||
)
|
reverse_result = reverse_mapping.get(individual_from_field_value)
|
||||||
)
|
if reverse_result is not None:
|
||||||
|
# reverse_result is a list, extend it
|
||||||
|
to_field_answer_list.extend(reverse_result)
|
||||||
|
else:
|
||||||
|
# If not found in reverse mapping, keep original value
|
||||||
|
to_field_answer_list.append(individual_from_field_value)
|
||||||
answer_dict[to_field_name] = to_field_answer_list
|
answer_dict[to_field_name] = to_field_answer_list
|
||||||
# update from field value to list if not already
|
# update from field value to list if not already
|
||||||
if not isinstance(from_field_value, list):
|
if not isinstance(from_field_value, list):
|
||||||
|
|||||||
@@ -26,7 +26,8 @@ JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS = """Return your final answer as a valid
|
|||||||
- Each item in the list should be a dictionary with the specified field names as keys.
|
- Each item in the list should be a dictionary with the specified field names as keys.
|
||||||
- If multiple items are found, include all of them as separate dictionaries in the list.
|
- If multiple items are found, include all of them as separate dictionaries in the list.
|
||||||
- Use "N/A" for fields within each dictionary where the information doesn't apply.
|
- Use "N/A" for fields within each dictionary where the information doesn't apply.
|
||||||
- Ensure the JSON is properly formatted with square brackets, curly braces, and double quotes."""
|
- Ensure the JSON is properly formatted with square brackets, curly braces, and double quotes.
|
||||||
|
- The Keys and Values must be strings, and not lists."""
|
||||||
|
|
||||||
|
|
||||||
# Parser helper functions for prompt templates
|
# Parser helper functions for prompt templates
|
||||||
@@ -486,7 +487,7 @@ def REIMBURSEMENT_PRIMARY(context) -> Tuple[str, Callable[[str], list]]:
|
|||||||
Call REIMBURSEMENT_PRIMARY_INSTRUCTION() separately for the cached instruction.
|
Call REIMBURSEMENT_PRIMARY_INSTRUCTION() separately for the cached instruction.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON list output. Should return a list of dicts of strings
|
||||||
"""
|
"""
|
||||||
prompt = f"""
|
prompt = f"""
|
||||||
[START CONTEXT]
|
[START CONTEXT]
|
||||||
@@ -500,7 +501,7 @@ Briefly talk through your reasoning. Then return a properly formatted JSON list
|
|||||||
|
|
||||||
|
|
||||||
def REIMBURSEMENT_PRIMARY_INSTRUCTION() -> str:
|
def REIMBURSEMENT_PRIMARY_INSTRUCTION() -> str:
|
||||||
return """[OBJECTIVE]
|
return f"""[OBJECTIVE]
|
||||||
Extract Reimbursement Terms from a section of a payer-provider contract
|
Extract Reimbursement Terms from a section of a payer-provider contract
|
||||||
The output from this prompt will be used to populate downstream fields in the system.
|
The output from this prompt will be used to populate downstream fields in the system.
|
||||||
|
|
||||||
@@ -536,6 +537,17 @@ Some types of Reimbursements are special and require additional thought.
|
|||||||
- Stop Loss and Outliers - These reimbursements usually consist of multiple rates and/or thresholds. When a Stop-Loss or Outlier is found, include the entire statement (which may be multiple sentences) as a single REIMB_TERM.
|
- Stop Loss and Outliers - These reimbursements usually consist of multiple rates and/or thresholds. When a Stop-Loss or Outlier is found, include the entire statement (which may be multiple sentences) as a single REIMB_TERM.
|
||||||
- Discounts, Escalators and Premiums - These will also include multiple rates. Capture all relevant numbers
|
- Discounts, Escalators and Premiums - These will also include multiple rates. Capture all relevant numbers
|
||||||
|
|
||||||
|
[OUTPUT FORMAT]
|
||||||
|
You MUST return your answer as a JSON list of dictionaries. Each dictionary must contain exactly two keys: "SERVICE_TERM" and "REIMB_TERM". Both values must be strings (not lists).
|
||||||
|
|
||||||
|
Example output format:
|
||||||
|
[
|
||||||
|
{{"SERVICE_TERM": "Service 1", "REIMB_TERM": "Reimb 1"}},
|
||||||
|
{{"SERVICE_TERM": "Service 2", "REIMB_TERM": "Reimb 2"}}
|
||||||
|
]
|
||||||
|
|
||||||
|
{JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS}
|
||||||
|
|
||||||
[ANALYSIS CONTEXT]"""
|
[ANALYSIS CONTEXT]"""
|
||||||
|
|
||||||
|
|
||||||
@@ -852,7 +864,7 @@ Populate a list of JSON dictionaries with the following fields:
|
|||||||
|
|
||||||
[OUTPUT FORMAT]
|
[OUTPUT FORMAT]
|
||||||
The output should always be a JSON list/array of dictionaries, even when there is only one methodology.
|
The output should always be a JSON list/array of dictionaries, even when there is only one methodology.
|
||||||
For any fields that don't apply to a particular methodology, use "N/A" as the value.
|
{JSON_LIST_OF_DICTS_FORMAT_INSTRUCTIONS}
|
||||||
|
|
||||||
[CONTEXT]"""
|
[CONTEXT]"""
|
||||||
|
|
||||||
@@ -874,16 +886,20 @@ Populate a JSON dictionary with the following fields:
|
|||||||
{fields_text}
|
{fields_text}
|
||||||
|
|
||||||
[OUTPUT FORMAT]
|
[OUTPUT FORMAT]
|
||||||
Briefly explain your reasoning, then write your JSON dictionary."""
|
{JSON_DICT_FORMAT_INSTRUCTIONS}"""
|
||||||
|
|
||||||
|
|
||||||
def FEE_SCHEDULE_BREAKOUT(
|
def FEE_SCHEDULE_BREAKOUT(
|
||||||
methodology, fee_schedule
|
methodology:str, fee_schedule:str
|
||||||
) -> Tuple[str, Callable[[str], dict]]:
|
) -> Tuple[str, Callable[[str], dict]]:
|
||||||
"""Returns ONLY dynamic content for fee schedule breakout.
|
"""Returns ONLY dynamic content for fee schedule breakout.
|
||||||
Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction.
|
Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction.
|
||||||
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
Note: Field definitions are now included in FEE_SCHEDULE_BREAKOUT_INSTRUCTION() for caching.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
methodology: The reimbursement methodology string (already normalized to string in prompt_calls)
|
||||||
|
fee_schedule: The fee schedule string (already normalized to string in prompt_calls)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON dict output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON dict output.
|
||||||
"""
|
"""
|
||||||
@@ -899,7 +915,6 @@ def GROUPER_BREAKOUT_INSTRUCTION() -> str:
|
|||||||
Contains objective, extraction rules, field definitions, and output format.
|
Contains objective, extraction rules, field definitions, and output format.
|
||||||
Field definitions are loaded from investment_prompts.json.
|
Field definitions are loaded from investment_prompts.json.
|
||||||
"""
|
"""
|
||||||
# Load fields from investment_prompts.json with resolved valid_values
|
|
||||||
fields_text = _get_fields_text("grouper_breakout")
|
fields_text = _get_fields_text("grouper_breakout")
|
||||||
|
|
||||||
return f"""[OBJECTIVE]
|
return f"""[OBJECTIVE]
|
||||||
@@ -1151,6 +1166,10 @@ def CODE_EXPLICIT(service, methodology) -> Tuple[str, Callable[[str], dict]]:
|
|||||||
Call CODE_EXPLICIT_INSTRUCTION() separately for the cached instruction.
|
Call CODE_EXPLICIT_INSTRUCTION() separately for the cached instruction.
|
||||||
Note: Field definitions are now included in CODE_EXPLICIT_INSTRUCTION() for caching.
|
Note: Field definitions are now included in CODE_EXPLICIT_INSTRUCTION() for caching.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
service: The service term (already normalized to string upstream)
|
||||||
|
methodology: The reimbursement methodology (already normalized to string upstream)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON dict output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON dict output.
|
||||||
"""
|
"""
|
||||||
@@ -1182,6 +1201,10 @@ Explain your answer in 1-2 sentences. Write your final answer in a properly-form
|
|||||||
def CODE_CATEGORY(service, choices) -> Tuple[str, Callable[[str], list]]:
|
def CODE_CATEGORY(service, choices) -> Tuple[str, Callable[[str], list]]:
|
||||||
"""Call CODE_CATEGORY_INSTRUCTION() separately for the cached instruction.
|
"""Call CODE_CATEGORY_INSTRUCTION() separately for the cached instruction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
service: The service term (already normalized to string upstream)
|
||||||
|
choices: The valid descriptions to choose from
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
||||||
"""
|
"""
|
||||||
@@ -1255,6 +1278,10 @@ Explain your answer, ensuring each point in the instructions is addressed. Then
|
|||||||
def CODE_IMPLICIT(service, choices) -> Tuple[str, Callable[[str], list]]:
|
def CODE_IMPLICIT(service, choices) -> Tuple[str, Callable[[str], list]]:
|
||||||
"""Call CODE_IMPLICIT_INSTRUCTION() separately for the cached instruction.
|
"""Call CODE_IMPLICIT_INSTRUCTION() separately for the cached instruction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
service: The service term (already normalized to string upstream)
|
||||||
|
choices: The valid descriptions to choose from
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
||||||
"""
|
"""
|
||||||
@@ -1325,6 +1352,12 @@ def FILL_BILL_TYPE(
|
|||||||
) -> Tuple[str, Callable[[str], list]]:
|
) -> Tuple[str, Callable[[str], list]]:
|
||||||
"""Call FILL_BILL_TYPE_INSTRUCTION() separately for the cached instruction.
|
"""Call FILL_BILL_TYPE_INSTRUCTION() separately for the cached instruction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
service: The service term (already normalized to string upstream)
|
||||||
|
choices: The valid bill type descriptions to choose from
|
||||||
|
reimb_term: The reimbursement term (already normalized to string upstream, optional)
|
||||||
|
exhibit_text: Additional exhibit context (optional)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
Tuple of (prompt_string, parser_function) where parser expects JSON list output.
|
||||||
"""
|
"""
|
||||||
@@ -1346,6 +1379,7 @@ If the Service Term and Reimbursement Term together do not provide sufficient in
|
|||||||
{exhibit_text.replace('"', "'")}
|
{exhibit_text.replace('"', "'")}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# service is already normalized to string upstream
|
||||||
prompt = f"""[VALID DESCRIPTIONS]
|
prompt = f"""[VALID DESCRIPTIONS]
|
||||||
Here are the descriptions to choose from:
|
Here are the descriptions to choose from:
|
||||||
{choices}
|
{choices}
|
||||||
|
|||||||
@@ -4,16 +4,16 @@ import logging
|
|||||||
import src.utils.string_utils as string_utils
|
import src.utils.string_utils as string_utils
|
||||||
|
|
||||||
|
|
||||||
def apply_crosswalk(val: str, mapping: dict[str, str], default: str = "N/A") -> str:
|
def apply_crosswalk(val: str | list, mapping: dict[str, str], default: str = "N/A") -> list:
|
||||||
"""Apply a crosswalk to a value
|
"""Apply a crosswalk to a value
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
val (str): Input value
|
val (str | list): Input value (can be string, list, or string representation of list)
|
||||||
mapping (dict[str, str]): Mapping dictionary
|
mapping (dict[str, str]): Mapping dictionary
|
||||||
default (str | None, optional): Default value; if None and val is not found in the mapping, val is returned. Defaults to "".
|
default (str | None, optional): Default value; if None and val is not found in the mapping, val is returned. Defaults to "".
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str: Crosswalked value
|
list: Crosswalked value as a list
|
||||||
"""
|
"""
|
||||||
# Handle None or blank values defensively
|
# Handle None or blank values defensively
|
||||||
if val is None:
|
if val is None:
|
||||||
|
|||||||
@@ -253,4 +253,4 @@ def parse_json_dict_or_list(raw_llm_output: str) -> dict[str, Any] | list[Any]:
|
|||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"No valid JSON object found in LLM output. "
|
f"No valid JSON object found in LLM output. "
|
||||||
f"Output preview: {raw_llm_output[:200]}..."
|
f"Output preview: {raw_llm_output[:200]}..."
|
||||||
)
|
)
|
||||||
+29
-10
@@ -1092,22 +1092,41 @@ def safe_token_to_str(tok: Any) -> str:
|
|||||||
|
|
||||||
def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
|
def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
|
||||||
"""
|
"""
|
||||||
Format TINs to 9 digits, preserving pipe-delimited multiple values.
|
Format TINs to 9 digits, handling JSON format (direct values or list of values).
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
val: TIN value (may be pipe-delimited)
|
val: TIN value (may be a string, list of strings, or None/empty)
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple of (formatted_tins, has_invalid)
|
Tuple of (formatted_tins, has_invalid)
|
||||||
"""
|
"""
|
||||||
if pd.isna(val) or val is None or str(val).strip() == "":
|
# Handle list/array values (e.g., from PROV_GROUP_TIN, PROV_OTHER_TIN)
|
||||||
return ("", True)
|
if isinstance(val, (list, tuple)) or (hasattr(val, '__iter__') and not isinstance(val, str)):
|
||||||
|
# Convert list to list of strings, filtering out empty values
|
||||||
parts = [
|
if len(val) == 0:
|
||||||
part.strip()
|
return ("", True)
|
||||||
for part in str(val).split("|")
|
parts = [
|
||||||
if part is not None and part.strip() != ""
|
str(item).strip()
|
||||||
]
|
for item in val
|
||||||
|
if item is not None and str(item).strip() != ""
|
||||||
|
]
|
||||||
|
if not parts:
|
||||||
|
return ("", True)
|
||||||
|
else:
|
||||||
|
# Handle single value (string or None)
|
||||||
|
# Check for NaN/None/empty
|
||||||
|
try:
|
||||||
|
if pd.isna(val):
|
||||||
|
return ("", True)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
# pd.isna() can't handle arrays, but we've already handled lists above
|
||||||
|
pass
|
||||||
|
|
||||||
|
if val is None or str(val).strip() == "":
|
||||||
|
return ("", True)
|
||||||
|
|
||||||
|
# Single value - convert to list for consistent processing
|
||||||
|
parts = [str(val).strip()]
|
||||||
out, any_invalid = [], False
|
out, any_invalid = [], False
|
||||||
|
|
||||||
for part in parts:
|
for part in parts:
|
||||||
|
|||||||
Reference in New Issue
Block a user