fix: Phase 4 - update model_evaluation_utils.py for JSON parsing

- Fixed DYNAMIC_PRIMARY_TEXT: now unpacks (prompt, parser) tuple and uses parser
- Fixed REIMBURSEMENT_PRIMARY: corrected function signature (only takes context)
- Replaced extract_text_from_delimiters with JSON parser
- Replaced universal_json_load with parser from tuple
- Added proper instruction caching for both prompts

Other testbed/QC files verified: they process internal data formats (out of scope per PRD)
This commit is contained in:
ppanchigar
2026-02-03 11:07:18 -06:00
parent 8339dea499
commit 13f3cbeac6
+10 -11
View File
@@ -21,15 +21,17 @@ def prompt_dynamic_primary(
for exhibit_page, exhibit_text in exhibit_dict.items(): for exhibit_page, exhibit_text in exhibit_dict.items():
answer_dict = {} answer_dict = {}
for field in dynamic_primary_fields.fields: for field in dynamic_primary_fields.fields:
prompt = prompt_templates.DYNAMIC_PRIMARY_TEXT( prompt, parser = prompt_templates.DYNAMIC_PRIMARY_TEXT(
exhibit_text, field.field_name, field.get_prompt(constants) exhibit_text, field.field_name, field.get_prompt(constants)
) )
llm_answer_raw = llm_utils.invoke_claude( llm_answer_raw = llm_utils.invoke_claude(
prompt=prompt, model_id=model_id, filename=filename prompt=prompt,
) model_id=model_id,
llm_answer_final = string_utils.extract_text_from_delimiters( filename=filename,
llm_answer_raw, string_utils.Delimiter.PIPE cache=True,
instruction=prompt_templates.DYNAMIC_PRIMARY_TEXT_INSTRUCTION(),
) )
llm_answer_final = parser(llm_answer_raw)
answer_dict[field.field_name] = llm_answer_final answer_dict[field.field_name] = llm_answer_final
answer_dicts.append(answer_dict) answer_dicts.append(answer_dict)
@@ -171,17 +173,14 @@ def evaluate_reimbursement_primary(
for model_name, model_id in model_id_dict.items(): for model_name, model_id in model_id_dict.items():
seen_pairs = set() seen_pairs = set()
for exhibit_page, exhibit_text in exhibit_dict.items(): for exhibit_page, exhibit_text in exhibit_dict.items():
field_prompts = reimbursement_primary_fields.print_prompt_dict(constants) prompt, parser = prompt_templates.REIMBURSEMENT_PRIMARY(exhibit_text)
instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY(
exhibit_text, field_prompts
)
llm_answer_raw = llm_utils.invoke_claude( llm_answer_raw = llm_utils.invoke_claude(
prompt, prompt,
model_id, model_id,
filename, filename,
max_tokens=25000, max_tokens=25000,
cache=True, cache=True,
instruction=instruction, instruction=prompt_templates.REIMBURSEMENT_PRIMARY_INSTRUCTION(),
usage_label="REIMBURSEMENT_PRIMARY", usage_label="REIMBURSEMENT_PRIMARY",
) )
# Check for the special "no results" case # Check for the special "no results" case
@@ -189,7 +188,7 @@ def evaluate_reimbursement_primary(
continue continue
try: try:
llm_answer_final = string_utils.universal_json_load(llm_answer_raw) llm_answer_final = parser(llm_answer_raw)
reimbursement_primary_answers = ( reimbursement_primary_answers = (
one_to_n_funcs.clean_reimbursement_primary( one_to_n_funcs.clean_reimbursement_primary(