Michael McGuinness
107 lines
5.1 KiB
Python
107 lines
5.1 KiB
Python
|
|
import re
|
|
import json
|
|
from collections import defaultdict
|
|
|
|
def secondary_string_to_dict(dict_string):
|
|
"""
|
|
Converts a string representation of a dictionary into an actual dictionary object, handling potential formatting issues.
|
|
|
|
This function cleans the string by removing specific unwanted characters and markers that may interfere with JSON parsing.
|
|
It then locates the substring that correctly forms a dictionary format, attempts to parse it as JSON, and handles common
|
|
parsing errors by replacing problematic single quotes with double quotes before re-parsing.
|
|
|
|
Parameters:
|
|
dict_string (str): The string containing the dictionary-like content, potentially surrounded by extra text or characters.
|
|
|
|
Returns:
|
|
dict: The dictionary obtained from parsing the cleaned and corrected string.
|
|
"""
|
|
dict_string = dict_string.replace('<<<', '')
|
|
dict_string = dict_string.replace('>>>', '')
|
|
dict_string = dict_string.replace('\n', '') # Remove new line
|
|
|
|
start_index = dict_string.find('{')
|
|
end_index = dict_string.rfind('}') + 1
|
|
dict_substring = dict_string[start_index:end_index]
|
|
try:
|
|
result_dict = json.loads(dict_substring)
|
|
except:
|
|
dict_substring = dict_substring.replace("'", '"')
|
|
result_dict = json.loads(dict_substring)
|
|
return result_dict
|
|
|
|
def primary_string_to_dict(string_dict, filename):
|
|
"""
|
|
Converts a dictionary of strings, where each string represents multiple dictionary entries, into a list of dictionaries,
|
|
augmenting each with metadata such as page number and filename.
|
|
|
|
This function iterates over each page number's string data, extracting and converting string representations of
|
|
dictionaries into actual dictionary objects. It handles and cleans the string format to properly parse it into dictionaries.
|
|
All dictionaries are then augmented with their respective page number and the filename before being compiled into a list.
|
|
|
|
Parameters:
|
|
string_dict (dict): A dictionary where keys are page numbers and values are strings containing multiple dictionary entries.
|
|
filename (str): The filename associated with these entries, used to tag each resulting dictionary.
|
|
|
|
Returns:
|
|
list of dict: A list of dictionaries, each representing data extracted and converted from the input string, tagged with
|
|
their page number and filename.
|
|
"""
|
|
data = []
|
|
pattern = r'\{.*?\}'
|
|
for page_num in string_dict.keys():
|
|
primary_list = string_dict[page_num]
|
|
dicts = primary_list.split('[')[1] # Strip front
|
|
dicts = dicts.split(']')[0] # Strip back
|
|
dicts = dicts.replace('\n', '') # Remove new lines
|
|
dicts = dicts.replace('<<<', '')
|
|
dicts = dicts.replace('>>>', '')
|
|
dicts = re.sub(r"(?<!\\)'([^']*?)'(?<!\\):", r'"\1":', dicts)
|
|
dict_list = re.findall(pattern, dicts)
|
|
dict_list = [secondary_string_to_dict(d) for d in dict_list]
|
|
for dict_ in dict_list:
|
|
dict_['page_num'] = page_num
|
|
dict_['Filename'] = filename
|
|
data.append(dict_)
|
|
return data
|
|
|
|
def append_secondary(result_dicts):
|
|
"""
|
|
Aggregates and restructures a list of dictionaries by grouping based on specific fields and appending secondary and tertiary
|
|
entries for specific keys.
|
|
|
|
This function organizes dictionaries by creating a unique key from non-'REIMBURSEMENT' fields and groups them accordingly.
|
|
It then processes each group to combine entries, designating the first three as primary, secondary, and tertiary respectively
|
|
for any 'REIMBURSEMENT' related fields. The combined results are stored in new dictionaries reflecting this hierarchical structure.
|
|
|
|
Parameters:
|
|
result_dicts (list of dict): A list of dictionaries containing data with fields that may include several related to 'REIMBURSEMENT'.
|
|
|
|
Returns:
|
|
list of dict: A list of new dictionaries where each dictionary represents combined information from the original list,
|
|
with 'REIMBURSEMENT' fields clearly distinguished as primary, secondary, or tertiary.
|
|
"""
|
|
# Helper function to create a key from dict excluding certain keys
|
|
def create_key(d):
|
|
return tuple((k, d[k]) for k in d if not k.startswith('REIMBURSEMENT'))
|
|
|
|
# Group dictionaries by non-REIMBURSEMENT fields
|
|
grouped = defaultdict(list)
|
|
for d in result_dicts:
|
|
grouped[create_key(d)].append(d)
|
|
|
|
# Process groups to create combined entries
|
|
combined_results = []
|
|
for key, group in grouped.items():
|
|
# Create a new dictionary based on non-REIMBURSEMENT fields
|
|
new_dict = {k: v for k, v in key}
|
|
# Iterate over REIMBURSEMENT fields and assign them as primary, secondary, tertiary
|
|
for i, entry in enumerate(group[:3]): # Process only the first three entries
|
|
for k, v in entry.items():
|
|
if k.startswith('REIMBURSEMENT'):
|
|
new_key = f'{["PRIMARY", "SECONDARY", "TERTIARY"][i]}_{k}'
|
|
new_dict[new_key] = v
|
|
combined_results.append(new_dict)
|
|
return combined_results
|