From 13f3cbeac6d0087ecbb032b860fcaef55d3c817b Mon Sep 17 00:00:00 2001 From: ppanchigar Date: Tue, 3 Feb 2026 11:07:18 -0600 Subject: [PATCH] fix: Phase 4 - update model_evaluation_utils.py for JSON parsing - Fixed DYNAMIC_PRIMARY_TEXT: now unpacks (prompt, parser) tuple and uses parser - Fixed REIMBURSEMENT_PRIMARY: corrected function signature (only takes context) - Replaced extract_text_from_delimiters with JSON parser - Replaced universal_json_load with parser from tuple - Added proper instruction caching for both prompts Other testbed/QC files verified: they process internal data formats (out of scope per PRD) --- src/testbed/model_evaluation_utils.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/testbed/model_evaluation_utils.py b/src/testbed/model_evaluation_utils.py index ae6ac8e..ec1e639 100644 --- a/src/testbed/model_evaluation_utils.py +++ b/src/testbed/model_evaluation_utils.py @@ -21,15 +21,17 @@ def prompt_dynamic_primary( for exhibit_page, exhibit_text in exhibit_dict.items(): answer_dict = {} for field in dynamic_primary_fields.fields: - prompt = prompt_templates.DYNAMIC_PRIMARY_TEXT( + prompt, parser = prompt_templates.DYNAMIC_PRIMARY_TEXT( exhibit_text, field.field_name, field.get_prompt(constants) ) llm_answer_raw = llm_utils.invoke_claude( - prompt=prompt, model_id=model_id, filename=filename - ) - llm_answer_final = string_utils.extract_text_from_delimiters( - llm_answer_raw, string_utils.Delimiter.PIPE + prompt=prompt, + model_id=model_id, + filename=filename, + cache=True, + instruction=prompt_templates.DYNAMIC_PRIMARY_TEXT_INSTRUCTION(), ) + llm_answer_final = parser(llm_answer_raw) answer_dict[field.field_name] = llm_answer_final answer_dicts.append(answer_dict) @@ -171,17 +173,14 @@ def evaluate_reimbursement_primary( for model_name, model_id in model_id_dict.items(): seen_pairs = set() for exhibit_page, exhibit_text in exhibit_dict.items(): - field_prompts = reimbursement_primary_fields.print_prompt_dict(constants) - instruction, prompt = prompt_templates.REIMBURSEMENT_PRIMARY( - exhibit_text, field_prompts - ) + prompt, parser = prompt_templates.REIMBURSEMENT_PRIMARY(exhibit_text) llm_answer_raw = llm_utils.invoke_claude( prompt, model_id, filename, max_tokens=25000, cache=True, - instruction=instruction, + instruction=prompt_templates.REIMBURSEMENT_PRIMARY_INSTRUCTION(), usage_label="REIMBURSEMENT_PRIMARY", ) # Check for the special "no results" case @@ -189,7 +188,7 @@ def evaluate_reimbursement_primary( continue try: - llm_answer_final = string_utils.universal_json_load(llm_answer_raw) + llm_answer_final = parser(llm_answer_raw) reimbursement_primary_answers = ( one_to_n_funcs.clean_reimbursement_primary(