diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index 57f99d4..6b94ac7 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -351,9 +351,11 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): ) ) + prompt, _parser = prompt_templates.CODE_IMPLICIT(service, level_dict["match_list"]) + # Run Prompt - claude_answer_raw = llm_utils.invoke_claude( - prompt_templates.CODE_IMPLICIT(service, level_dict["match_list"]), + llm_answer_raw = llm_utils.invoke_claude( + prompt, "sonnet_latest", filename, cache=True, @@ -361,18 +363,18 @@ def code_implicit_rag(service, implicit_run_dict, filename, constants): ) try: - claude_answer_final = string_utils.universal_json_load(claude_answer_raw) + llm_answer_final = _parser(llm_answer_raw) except Exception as e: return {"CODE_METHODOLOGY": e} - if not claude_answer_final: + if not llm_answer_final: continue # Populate answers, if any code_answer_dict = {} proc_codes, rev_codes = [], [] proc_descs, rev_descs = [], [] - for description in claude_answer_final: + for description in llm_answer_final: if description == "INVALID_SERVICE": code_answer_dict["CODE_METHODOLOGY"] = "Generic - Prompt" continue diff --git a/src/constants/delimiters.py b/src/constants/delimiters.py index 98cc389..dc34783 100644 --- a/src/constants/delimiters.py +++ b/src/constants/delimiters.py @@ -2,6 +2,5 @@ from enum import Enum class Delimiter(Enum): - PIPE = "|" BACKTICK = "`" TRIPLE_BACKTICK = "```" diff --git a/src/crosswalk/crosswalk_builder.py b/src/crosswalk/crosswalk_builder.py index e5a928c..b16b62b 100644 --- a/src/crosswalk/crosswalk_builder.py +++ b/src/crosswalk/crosswalk_builder.py @@ -147,7 +147,7 @@ class CrosswalkBuilder: df = pd.DataFrame(self.mapping.items(), columns=[from_col, to_col]) df.to_csv(output_path, index=False, quoting=csv.QUOTE_ALL, encoding="utf-8-sig") - def create_reverse_mapping(self) -> dict[str, str]: + def create_reverse_mapping(self) -> dict[str, list[str]]: """Create a reverse mapping where values that map to the same key are concatenated Example: @@ -166,7 +166,7 @@ class CrosswalkBuilder: reverse_dict[target] = [str(source)] # Then join the lists with commas - return {target: "|".join(sources) for target, sources in reverse_dict.items()} + return {target: sources for target, sources in reverse_dict.items()} def map_value(self, key_value): return self.mapping.get(key_value) diff --git a/src/pipelines/shared/extraction/one_to_one_funcs.py b/src/pipelines/shared/extraction/one_to_one_funcs.py index ff20fbe..cc423ab 100644 --- a/src/pipelines/shared/extraction/one_to_one_funcs.py +++ b/src/pipelines/shared/extraction/one_to_one_funcs.py @@ -6,15 +6,12 @@ import json import pandas as pd import src.pipelines.shared.postprocessing.postprocessing_funcs as postprocessing_funcs import src.pipelines.saas.prompts.prompt_calls as prompt_calls -import src.prompts.prompt_templates as prompt_templates -import src.utils.llm_utils as llm_utils import src.utils.string_utils as string_utils from src.constants.constants import Constants -from src.constants.delimiters import Delimiter from src import config from src.pipelines.shared.postprocessing import postprocessing_funcs from src.pipelines.shared.extraction.vision_funcs import get_image_array_based_answer -from src.prompts.fieldset import Field, FieldSet +from src.prompts.fieldset import FieldSet def pass_smart_chunked_to_full_context(answers_dict, one_to_one_fields): diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index d6f7bef..b7eb859 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -2,16 +2,12 @@ import json import logging import re -from typing import Optional - import src.constants.regex_patterns as regex_patterns import src.config as config import src.prompts.prompt_templates as prompt_templates from src.utils import llm_utils, string_utils from src.pipelines.saas.prompts import prompt_calls -from src.constants.delimiters import Delimiter from src.prompts.fieldset import Field, FieldSet -from difflib import SequenceMatcher def get_all_matches(text: str, pattern: str) -> list[str]: diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 6e7cb78..3c004fa 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -1,12 +1,7 @@ -import logging -import os -from collections import defaultdict -import pandas as pd import src.config as config import src.utils.string_utils as string_utils from src.constants.constants import Constants -from src.constants.delimiters import Delimiter from src.crosswalk.crosswalk_builder import CrosswalkBuilder from src.prompts.fieldset import FieldSet from src.utils.crosswalk_utils import apply_crosswalk diff --git a/src/pipelines/shared/postprocessing/postprocessing_funcs.py b/src/pipelines/shared/postprocessing/postprocessing_funcs.py index 2d32437..fa2c279 100644 --- a/src/pipelines/shared/postprocessing/postprocessing_funcs.py +++ b/src/pipelines/shared/postprocessing/postprocessing_funcs.py @@ -4,11 +4,7 @@ import logging import os import re from datetime import datetime -import json import pandas as pd -import src.prompts.prompt_templates as prompt_templates -import src.utils.llm_utils as llm_utils -from src.constants.delimiters import Delimiter from src.utils import string_utils # Determine the base directory for the project