Merged in refactor/one-time-io (pull request #648)

Refactor/one time io

* Delete client work

* Streamline imports

* Fix list class

* Update tests

* Clear code funcs test

* Fix unit tests

* Single read code funcs

* Single-read model

* Single-load embeddings

* Successful E2E test

* update unit tests

* Update importsg

* Reload poetry.lock

* remove old exhibit header function

* Remove aarete_derived generic function

* remove align and format tables

* Remove strings to dict

* references

* Clear code

* Move preprocessing_funcs

* remove keywords

* refactor postprocessing_funcs

* Pass unit test - remove qa_qc directory

* Black and isort

* remove print


Approved-by: Alex Galarce

fieldExtraction/src/utils/string_utils.py

@@ -7,12 +7,12 @@ from datetime import datetime
 import numpy as np
 import pandas as pd
 
-import src.utils.llm_utils as llm_utils
-from src import config
-from src.enums.delimiters import Delimiter
-from src.prompts import preprocessing_prompts
-from src.regex.regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
-                                      TRIPLE_BACKTICK_PATTERN)
+from constants.delimiters import Delimiter
+from constants.regex_patterns import (
+    BACKTICK_PATTERN,
+    PIPE_PATTERN,
+    TRIPLE_BACKTICK_PATTERN,
+)
 
 
 def extract_text_from_delimiters(
@@ -184,78 +184,6 @@ def json_parsing_search(response_text: str, field_list: list[str]) -> dict[str,
     return dict(zip(field_l, answer_l))
 
 
-def secondary_string_to_dict(dict_string: str, filename: str) -> dict:
-    """
-    Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues.
-
-    This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing.
-    It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common
-    parsing errors by replacing problematic single quotes with double quotes before re-parsing.
-
-    Parameters:
-        dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters.
-        filename (str): The filename associated with these entries, used to tag each resulting dictionary.
-
-    Returns:
-        dict: The dictionary obtained from parsing the cleaned and corrected string.
-    """
-    start_index = dict_string.find("{")
-    end_index = dict_string.rfind("}") + 1
-    dict_substring = dict_string[start_index:end_index]
-    try:
-        result_dict = json.loads(dict_substring)
-    except:
-        try:
-            dict_substring = dict_substring.replace("'", '"')
-            result_dict = json.loads(dict_substring)
-        except:
-            try:
-                prompt = preprocessing_prompts.FIX_JSON(dict_substring)
-                dict_substring = llm_utils.invoke_claude(
-                    prompt, "haiku_latest", filename
-                )
-                result_dict = json.loads(dict_substring)
-            except Exception as e:
-                print(e)
-                result_dict = {}
-    return result_dict
-
-
-def primary_string_to_dict(string_dict, filename):
-    """
-    Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries,
-    augmenting each with metadata such as page number and filename.
-
-    This function iterates over each page number's string data, extracting and converting string representations of
-    dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries.
-    All dictionaries are then augmented with their respective page number and the filename before being compiled into a list.
-
-    Parameters:
-        string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries.
-        filename (str): The filename associated with these entries, used to tag each resulting dictionary.
-
-    Returns:
-        list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with
-                      their page number and filename.
-    """
-    data = []
-    pattern = r"\{.*?\}"
-    for page_num in string_dict.keys():
-        primary_list = string_dict[page_num]
-        dicts = primary_list.split("[")[1]  # Strip front
-        dicts = dicts.split("]")[0]  # Strip back
-        dicts = dicts.replace("\n", "")  # Remove new lines
-        dicts = dicts.replace("<<<", "")
-        dicts = dicts.replace(">>>", "")
-        dicts = re.sub(r"(?<!\\)'([^']*?)'(?<!\\):", r'"\1":', dicts)
-        dict_list = re.findall(pattern, dicts)
-        dict_list = [secondary_string_to_dict(d, filename) for d in dict_list]
-        for dict_ in dict_list:
-            dict_["page_num"] = page_num
-            data.append(dict_)
-    return data
-
-
 def universal_json_load(string_dict: str):
     """Matches json dicts and list of dicts - NOT simple lists
     Args: