From 0939399f0ef1bf854d1150fa01fc97f5954b2277 Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Fri, 25 Apr 2025 21:34:51 +0000 Subject: [PATCH] Merged in bugfix/up-tokens-and-json-parse-logging (pull request #493) Bugfix/up tokens and json parse logging * Increase max_tokens limit for get_reimbursement_primary * print entire problematic input in universal_json_load Approved-by: Katon Minhas --- fieldExtraction/src/investment/one_to_n_funcs.py | 2 +- fieldExtraction/src/utils/string_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 4c1a2e8..010077d 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -60,7 +60,7 @@ def get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename field_prompts = reimbursement_level_fields.get_prompt_dict() prompt = investment_prompts.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts) llm_answer_raw = llm_utils.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=8192 + prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=25000 ) llm_answer_final = string_utils.universal_json_load(llm_answer_raw) return llm_answer_final diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index 1b71904..2bcc275 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -271,8 +271,8 @@ def universal_json_load(string_dict: str): return json.loads(matched_str) except json.JSONDecodeError as e: print(f"Failed to parse JSON: {e}") - print(f"Problematic JSON string: {matched_str}") - print(f"Original string: {string_dict[:200]}{'...' if len(string_dict) > 200 else ''}") # Print first 200 chars + # print(f"Problematic JSON string: {matched_str}") + print(f"Original string: {string_dict}") print(f"Traceback: {traceback.format_exc()}") raise ValueError(f"JSON parsing failed: {e.msg} at position {e.pos}") from e else: