Merged in bugfix/ut-errors (pull request #756)
Bugfix/ut errors * Update universal_json_load and unit tests * Update tests * Fix unit tests * remove test * remove imports * Update function * update pd Series is_empty * Update reimbursement check * added some test cases * Merged main into bugfix/ut-errors * Merged main into bugfix/ut-errors * remove import * Merged main into bugfix/ut-errors Approved-by: Karan Desai
This commit is contained in:
Katon Minhas
@@ -182,76 +182,47 @@ def json_parsing_search(response_text: str, field_list: list[str]) -> dict[str,
|
||||
return dict(zip(field_l, answer_l))
|
||||
|
||||
|
||||
def universal_json_load(string_dict: str):
|
||||
"""Matches json dicts and list of dicts - NOT simple lists
|
||||
Args:
|
||||
string_dict (str): The string containing the JSON-like content.
|
||||
Returns:
|
||||
list[dict] | dict: The dictionary or list of dictionaries obtained from parsing the cleaned and corrected string.
|
||||
Raises:
|
||||
ValueError: If no valid JSON object is found in the input string. In addition,
|
||||
the traceback as well as the offending string are printed.
|
||||
|
||||
Note: if more than one json-like (or list-like) is found in the input string, this function will
|
||||
take the last one positionally.
|
||||
def universal_json_load(s: str):
|
||||
"""
|
||||
# Try to match dictionary or list of dictionaries first
|
||||
# It will find any substring starting with { or [ and ending with } or ].
|
||||
# dict_matches = re.findall(r"\{.*?\}|\[\s*?\{.*?\}\s*?\]", string_dict, re.DOTALL)
|
||||
dict_matches = re.findall(r'(\[.*\]|\{.*?\})', string_dict, re.DOTALL)
|
||||
if dict_matches:
|
||||
matched_str = dict_matches[-1] # Take the last match in the string
|
||||
try:
|
||||
return json.loads(matched_str)
|
||||
except json.JSONDecodeError:
|
||||
# If it fails, it might be a malformed list (e.g., "[{...} text {...}]").
|
||||
# As a fallback, find the LAST dictionary within that malformed string.
|
||||
fallback_matches = re.findall(r'\{.*?\}', matched_str, re.DOTALL)
|
||||
if fallback_matches:
|
||||
last_dict_in_string = fallback_matches[-1]
|
||||
try:
|
||||
return json.loads(last_dict_in_string)
|
||||
except json.JSONDecodeError as e:
|
||||
logging.error(f"Failed to parse JSON in fallback: {e}")
|
||||
logging.error(f"Original string: {string_dict}")
|
||||
logging.error(f"Traceback: {traceback.format_exc()}")
|
||||
# Use placeholders to preserve JSON structure characters for CSV compatibility
|
||||
csv_safe_string = (
|
||||
string_dict.replace(",", "[COMMA]")
|
||||
.replace("\n", "[NEWLINE]")
|
||||
.replace('"', "[QUOTE]")
|
||||
)
|
||||
raise ValueError(
|
||||
f"JSON parsing failed in fallback: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
|
||||
) from e
|
||||
|
||||
# If no dict pattern found, try to match list of strings
|
||||
list_matches = re.findall(r"\[(.*?)\]", string_dict, re.DOTALL)
|
||||
if list_matches:
|
||||
matched_str = f"[{list_matches[-1]}]" # Take the last match in the string
|
||||
try:
|
||||
# Clean up the string list before parsing
|
||||
cleaned_str = re.sub(r'(?<!\\)"', '"', matched_str) # Handle escaped quotes
|
||||
cleaned_str = re.sub(
|
||||
r"'", '"', cleaned_str
|
||||
) # Replace single quotes with double quotes
|
||||
|
||||
return json.loads(cleaned_str)
|
||||
except json.JSONDecodeError as e:
|
||||
logging.error(f"Failed to parse list: {e}")
|
||||
logging.error(f"Original string: {string_dict}")
|
||||
logging.error(f"Traceback: {traceback.format_exc()}")
|
||||
# Use placeholders to preserve JSON structure characters for CSV compatibility
|
||||
csv_safe_string = (
|
||||
string_dict.replace(",", "[COMMA]")
|
||||
.replace("\n", "[NEWLINE]")
|
||||
.replace('"', "[QUOTE]")
|
||||
)
|
||||
raise ValueError(
|
||||
f"JSON parsing failed: {e.msg} at position {e.pos}; Original string:\n{csv_safe_string}"
|
||||
) from e
|
||||
|
||||
return {}
|
||||
Extract and parse all highest-level valid JSON objects from a string.
|
||||
Returns the last valid top-level JSON object found (dict, list, etc).
|
||||
Raises ValueError if no valid JSON objects are found. Does not clean up syntax.
|
||||
"""
|
||||
if not isinstance(s, str):
|
||||
raise TypeError(f"Expected string input, got {type(s).__name__}")
|
||||
found = []
|
||||
i = 0
|
||||
while i < len(s):
|
||||
if s[i] in '{[':
|
||||
start = i
|
||||
stack = [s[i]]
|
||||
for j in range(i+1, len(s)):
|
||||
if s[j] in '{[':
|
||||
stack.append(s[j])
|
||||
elif s[j] in '}]':
|
||||
if not stack:
|
||||
break
|
||||
open_bracket = stack.pop()
|
||||
if (open_bracket == '{' and s[j] != '}') or (open_bracket == '[' and s[j] != ']'):
|
||||
break
|
||||
if not stack:
|
||||
candidate = s[start:j+1]
|
||||
# Only consider top-level objects (not nested)
|
||||
# Check if start is not inside another object
|
||||
# This is guaranteed by stack being empty only at top-level
|
||||
try:
|
||||
obj = json.loads(candidate)
|
||||
found.append(obj)
|
||||
except Exception:
|
||||
pass
|
||||
i = j # Move past this top-level object
|
||||
break
|
||||
else:
|
||||
pass
|
||||
i += 1
|
||||
if found:
|
||||
return found[-1]
|
||||
raise ValueError(f"No valid JSON objects found in string: {s[:100]}...")
|
||||
|
||||
|
||||
reimbursement_strings = [ # These are for `method='keyword'`
|
||||
@@ -259,6 +230,9 @@ reimbursement_strings = [ # These are for `method='keyword'`
|
||||
"$",
|
||||
"percent",
|
||||
"billed charges",
|
||||
"compensation",
|
||||
"reimbursement",
|
||||
"fee schedule"
|
||||
]
|
||||
reimb_regex = r"(?<![$%])(?:\$\d+|\d+[$%])(?![$%])"
|
||||
|
||||
@@ -361,7 +335,11 @@ def is_empty(value: str | list | pd.Series, pd_mask: bool = True) -> bool | pd.S
|
||||
)
|
||||
)
|
||||
else:
|
||||
return value.isna().all()
|
||||
# If the Series is empty, it's considered empty
|
||||
if value.empty:
|
||||
return True
|
||||
# Use the string-version check for each element in the Series
|
||||
return all(is_empty(str(v)) for v in value)
|
||||
|
||||
if pd.isna(value):
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user