fix: Phase 4 - update model_evaluation_utils.py for JSON parsing

- Fixed DYNAMIC_PRIMARY_TEXT: now unpacks (prompt, parser) tuple and uses parser
- Fixed REIMBURSEMENT_PRIMARY: corrected function signature (only takes context)
- Replaced extract_text_from_delimiters with JSON parser
- Replaced universal_json_load with parser from tuple
- Added proper instruction caching for both prompts

Other testbed/QC files verified: they process internal data formats (out of scope per PRD)
This commit is contained in:
ppanchigar
2026-02-03 11:07:18 -06:00
parent 8339dea499
commit 13f3cbeac6
+10 -11
View File
@@ -21,15 +21,17 @@ def prompt_dynamic_primary(
for exhibit_page, exhibit_text in exhibit_dict.items():
answer_dict = {}
for field in dynamic_primary_fields.fields:
prompt = prompt_templates.DYNAMIC_PRIMARY_TEXT(
prompt, parser = prompt_templates.DYNAMIC_PRIMARY_TEXT(
exhibit_text, field.field_name, field.get_prompt(constants)
)
llm_answer_raw = llm_utils.invoke_claude(
prompt=prompt, model_id=model_id, filename=filename
)
llm_answer_final = string_utils.extract_text_from_delimiters(
llm_answer_raw, string_utils.Delimiter.PIPE
prompt=prompt,
model_id=model_id,
filename=filename,
cache=True,
instruction=prompt_templates.DYNAMIC_PRIMARY_TEXT_INSTRUCTION(),
)
llm_answer_final = parser(llm_answer_raw)
answer_dict[field.field_name] = llm_answer_final
answer_dicts.append(answer_dict)
@@ -171,17 +173,14 @@ def evaluate_reimbursement_primary(
for model_name, model_id in model_id_dict.items():
seen_pairs = set()
for exhibit_page, exhibit_text in exhibit_dict.items():
field_prompts = reimbursement_primary_fields.print_prompt_dict(constants)
instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY(
exhibit_text, field_prompts
)
prompt, parser = prompt_templates.REIMBURSEMENT_PRIMARY(exhibit_text)
llm_answer_raw = llm_utils.invoke_claude(
prompt,
model_id,
filename,
max_tokens=25000,
cache=True,
instruction=instruction,
instruction=prompt_templates.REIMBURSEMENT_PRIMARY_INSTRUCTION(),
usage_label="REIMBURSEMENT_PRIMARY",
)
# Check for the special "no results" case
@@ -189,7 +188,7 @@ def evaluate_reimbursement_primary(
continue
try:
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
llm_answer_final = parser(llm_answer_raw)
reimbursement_primary_answers = (
one_to_n_funcs.clean_reimbursement_primary(