Merged in refactor/daip-2-9-code-refactor (pull request #339)
Refactor/daip2-9 code refactor * add missing import * refactor ac_smart_chunking.py * update tests * removed old and unused imports * removed outdated import * forgot to import re * refactor bottom up funcs * remove unused imports from file_processing.py * refactor dict_operations.py * remove unused import from conditional_funcs.py * replace top_down_funcs with one_to_n_funcs and remove top_down_funcs.py * remove duplicate import from one_to_n_funcs.py * refactor all utils.py imports * refactor error handling by removing last code in utils and integrating InvalidDateException into postprocessing_funcs * refactor import in claude_funcs.py to use string_funcs directly and (hopefully) resolve circular import * refactor regex_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor hotfix_helper_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor postprocess.py to remove unused imports * add numpy import to string_funcs * fix tests after utils refactor * move adhoc from tests to scripts * remove unused imports * remove scripts from mypy checking * isort * Merged main into refactor/daip-2-9-code-refactor Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
committed by
Katon Minhas
Katon Minhas
parent
bfbd8d6b8b
commit
3740588efa
@@ -0,0 +1,237 @@
|
||||
import json
|
||||
import re
|
||||
import warnings
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import claude_funcs
|
||||
import config
|
||||
import prompts
|
||||
from enums.delimiters import Delimiter
|
||||
from regex_patterns import (BACKTICK_PATTERN, PIPE_PATTERN,
|
||||
TRIPLE_BACKTICK_PATTERN)
|
||||
|
||||
|
||||
def extract_text_from_delimiters(
|
||||
raw_output: str, delimiter: Delimiter, match_index: int = -1
|
||||
) -> str:
|
||||
"""
|
||||
Extracts a match enclosed in specified delimiters from the raw output.
|
||||
This function searches for text enclosed by a specified delimiter in the given raw output string.
|
||||
It returns the match specified by the `which_match` index. If no match is found, it returns "N/A".
|
||||
|
||||
- raw_output (str): The raw output string to search within.
|
||||
- which_match (int, optional): The index of the match to return. Defaults to -1, which returns the last match. If the index is out of range, the last match is returned.
|
||||
- str: The extracted text or "N/A" if no match is found.
|
||||
|
||||
Raises:
|
||||
- TypeError: If `raw_output` is not a string or `delimiter` is not a Delimiter enum value.
|
||||
- ValueError: If an unsupported delimiter is provided.
|
||||
|
||||
Returns:
|
||||
- str: The extracted answer or "N/A" if no match is found.
|
||||
"""
|
||||
if not isinstance(raw_output, str):
|
||||
raise TypeError(
|
||||
"Expected a string for raw_output, got {0}.".format(
|
||||
type(raw_output).__name__
|
||||
)
|
||||
)
|
||||
|
||||
if not isinstance(delimiter, Delimiter):
|
||||
raise TypeError(
|
||||
"Expected a Delimiter enum value, got {0}.".format(type(delimiter).__name__)
|
||||
)
|
||||
|
||||
# Define a pattern based on the delimiter type
|
||||
if delimiter == Delimiter.PIPE:
|
||||
pattern = PIPE_PATTERN
|
||||
elif delimiter == Delimiter.BACKTICK:
|
||||
pattern = BACKTICK_PATTERN
|
||||
elif delimiter == Delimiter.TRIPLE_BACKTICK:
|
||||
pattern = TRIPLE_BACKTICK_PATTERN
|
||||
else:
|
||||
raise ValueError("Unsupported delimiter. Use one of the Delimiter enum values.")
|
||||
|
||||
# Find all matches based on the pattern
|
||||
matches = re.findall(pattern=pattern, string=raw_output)
|
||||
|
||||
if len(matches) == 0:
|
||||
return "N/A"
|
||||
|
||||
# Adjust for negative indices
|
||||
if match_index < 0:
|
||||
match_index += len(matches)
|
||||
|
||||
if not 0 <= match_index < len(matches):
|
||||
warnings.warn("Index out of range. Returning the last match by default.")
|
||||
match_index = -1
|
||||
|
||||
return matches[match_index]
|
||||
|
||||
|
||||
def json_parsing_search(response_text, field_list):
|
||||
try:
|
||||
if response_text.split("{", 1)[1].strip()[0] == '"':
|
||||
response_text = "{" + response_text.split("{", 1)[1]
|
||||
else:
|
||||
response_text = "{" + response_text
|
||||
except:
|
||||
response_text = "{" + response_text
|
||||
|
||||
if len(response_text.split("}", 1)) > 1:
|
||||
if response_text.rsplit("}", 1)[0].strip()[-1] == '"':
|
||||
response_text = response_text.rsplit("}", 1)[0] + "}"
|
||||
elif response_text.rsplit("}", 1)[0].strip()[-1] == "}":
|
||||
response_text = response_text.rsplit("}", 1)[0]
|
||||
else:
|
||||
response_text = response_text.rstrip(",") + "}"
|
||||
field_l = []
|
||||
answer_l = []
|
||||
position_dict = {}
|
||||
for f in field_list:
|
||||
location = response_text.find('"' + f + '"')
|
||||
if location != -1:
|
||||
position_dict[location] = f
|
||||
field_list = list(dict(sorted(position_dict.items())).values())
|
||||
for f in field_list:
|
||||
if f in response_text:
|
||||
field_l.append(f)
|
||||
value = response_text.split('"' + f + '"')[0]
|
||||
response_text = response_text.split('"' + f + '"')[1]
|
||||
if f != field_list[0] and f != field_list[-1]:
|
||||
answer_l.append(
|
||||
value.strip("\n")
|
||||
.strip('"')
|
||||
.strip(":")
|
||||
.strip(" ")
|
||||
.strip("\n")
|
||||
.strip(" ")
|
||||
.strip(",")
|
||||
.strip('"')
|
||||
)
|
||||
elif f == field_list[-1]:
|
||||
answer_l.append(
|
||||
value.strip("\n")
|
||||
.strip('"')
|
||||
.strip(":")
|
||||
.strip(" ")
|
||||
.strip("\n")
|
||||
.strip(" ")
|
||||
.strip(",")
|
||||
.strip('"')
|
||||
)
|
||||
answer_l.append(
|
||||
response_text.strip("\n")
|
||||
.strip('"')
|
||||
.strip(":")
|
||||
.strip(" ")
|
||||
.strip("}")
|
||||
.strip("\n")
|
||||
.strip(" ")
|
||||
.strip('"')
|
||||
)
|
||||
|
||||
return dict(zip(field_l, answer_l))
|
||||
|
||||
|
||||
def secondary_string_to_dict(dict_string, filename):
|
||||
"""
|
||||
Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues.
|
||||
|
||||
This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing.
|
||||
It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common
|
||||
parsing errors by replacing problematic single quotes with double quotes before re-parsing.
|
||||
|
||||
Parameters:
|
||||
dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters.
|
||||
|
||||
Returns:
|
||||
dict: The dictionary obtained from parsing the cleaned and corrected string.
|
||||
"""
|
||||
start_index = dict_string.find("{")
|
||||
end_index = dict_string.rfind("}") + 1
|
||||
dict_substring = dict_string[start_index:end_index]
|
||||
try:
|
||||
result_dict = json.loads(dict_substring)
|
||||
except:
|
||||
try:
|
||||
dict_substring = dict_substring.replace("'", '"')
|
||||
result_dict = json.loads(dict_substring)
|
||||
except:
|
||||
try:
|
||||
prompt = prompts.FIX_JSON(dict_substring)
|
||||
dict_substring = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename
|
||||
)
|
||||
result_dict = json.loads(dict_substring)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
result_dict = {}
|
||||
return result_dict
|
||||
|
||||
|
||||
def primary_string_to_dict(string_dict, filename):
|
||||
"""
|
||||
Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries,
|
||||
augmenting each with metadata such as page number and filename.
|
||||
|
||||
This function iterates over each page number's string data, extracting and converting string representations of
|
||||
dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries.
|
||||
All dictionaries are then augmented with their respective page number and the filename before being compiled into a list.
|
||||
|
||||
Parameters:
|
||||
string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries.
|
||||
filename (str): The filename associated with these entries, used to tag each resulting dictionary.
|
||||
|
||||
Returns:
|
||||
list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with
|
||||
their page number and filename.
|
||||
"""
|
||||
data = []
|
||||
pattern = r"\{.*?\}"
|
||||
for page_num in string_dict.keys():
|
||||
primary_list = string_dict[page_num]
|
||||
dicts = primary_list.split("[")[1] # Strip front
|
||||
dicts = dicts.split("]")[0] # Strip back
|
||||
dicts = dicts.replace("\n", "") # Remove new lines
|
||||
dicts = dicts.replace("<<<", "")
|
||||
dicts = dicts.replace(">>>", "")
|
||||
dicts = re.sub(r"(?<!\\)'([^']*?)'(?<!\\):", r'"\1":', dicts)
|
||||
dict_list = re.findall(pattern, dicts)
|
||||
dict_list = [secondary_string_to_dict(d, filename) for d in dict_list]
|
||||
for dict_ in dict_list:
|
||||
dict_["page_num"] = page_num
|
||||
dict_["Filename"] = filename
|
||||
data.append(dict_)
|
||||
return data
|
||||
|
||||
|
||||
def contains_reimbursement(text, page="1"): # string_funcs.py
|
||||
if isinstance(text, dict):
|
||||
return page.isdigit() and (
|
||||
"%" in text[page]
|
||||
or "$" in text[page]
|
||||
or "percent " in text[page]
|
||||
or "compensation schedule" in text[page].lower()
|
||||
or "reimbursement schedule" in text[page].lower()
|
||||
)
|
||||
elif isinstance(text, str):
|
||||
return (
|
||||
"%" in text
|
||||
or "$" in text
|
||||
or "percent " in text
|
||||
or "compensation schedule" in text.lower()
|
||||
or "reimbursement schedule" in text.lower()
|
||||
)
|
||||
else:
|
||||
print("contains_reimbursement - Invalid data type")
|
||||
|
||||
|
||||
def is_empty(value): # string_funcs.py
|
||||
if pd.isna(value):
|
||||
return True
|
||||
else:
|
||||
empty_values = [None, "", "N/A", "NA", "null", "none", "NaN", np.nan, "nan"]
|
||||
return value in empty_values
|
||||
Reference in New Issue
Block a user