Merged in bugfix/prov_info_json_fixes (pull request #899)
Bugfix/prov info json fixes * fix: robust PROV_INFO_JSON sanitization and TIN backfill logic json_utils: - Add sanitize_prov_info_json with layered parsing (JSON, literal_eval, empty-value-after-colon fix, best-effort dict extraction). - Add _normalize_prov_entries and _prov_value_to_str for uniform str-valued output; flatten list values, strip TIN hyphens. - format_prov_info_json now delegates to sanitize_prov_info_json. postprocessing_funcs: - Add fill_prov_info_tin_from_filename_tin for TIN backfill. - Add validate_and_reformat_date (pipe-wrapped, datetime strings). - Add format_as_json_list (pipe-delimited, comma-separated, quote stripping). postprocess: - Integrate new postprocessing helpers into pipeline flow. postprocess_existing_output: - Support CSV and Excel input, configurable paths, fillna for CSV. tests: - Add test_json_parsers.py for PROV_INFO_JSON parsing coverage. - Add test_postprocess.py for date/list formatting and default_ind. * Merge branch 'DAIP2-1947-tin-and-prov-info-json-issues' into bugfix/prov_info_json_fixes * Merged DEV into bugfix/prov_info_json_fixes * Strip out unused functionality * Update filename_tin functionality * Add docstring * Update filename_tin cleaning in PROV_INFO_JSON * test prep * black format * missing function added * Merge branch 'DEV' into bugfix/prov_info_json_fixes * Black format * Strip unused functions * Strip unused code * update unit tests * Merge branch 'DEV' into bugfix/prov_info_json_fixes * Update test * Simplify process * handle list of group names * Resolve run_provider_info_field call * Merge branch 'DEV' into bugfix/prov_info_json_fixes * Correct type hints * Fix unit tests * Fix unit test * Remove redundant postprocessing_funcs Approved-by: Katon Minhas
This commit is contained in:
Mayank Aamseek
committed by
Katon Minhas
Katon Minhas
parent
6744c57f95
commit
3331da2a8c
@@ -1,12 +1,15 @@
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from src.utils import string_utils
|
||||
import json
|
||||
from src.utils import json_utils
|
||||
|
||||
# Determine the base directory for the project
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
@@ -221,24 +224,6 @@ def normalize_currency(value: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def remove_hyphens(value):
|
||||
"""
|
||||
Remove hyphens from the input value.
|
||||
|
||||
Args:
|
||||
value : The input might be list or string
|
||||
Returns:
|
||||
str: The cleaned value with hyphens removed.
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.replace("-", "")
|
||||
elif isinstance(value, list):
|
||||
return [v.replace("-", "") if isinstance(v, str) else v for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def flatten_singleton_string_list(list_str: str) -> str:
|
||||
"""Take in a list within a string, and if it has a single element, return that element.
|
||||
If it has multiple elements, return the list as a string.
|
||||
@@ -270,6 +255,9 @@ def validate_and_reformat_date(date_str: str) -> str:
|
||||
"""
|
||||
Validate if a string is in YYYY/MM/DD format or reformat it to YYYY/MM/DD if possible.
|
||||
|
||||
Strips leading/trailing pipes (e.g. |01/01/2022|) before parsing so date-only
|
||||
content is validated and reformatted.
|
||||
|
||||
Args:
|
||||
date_str (str): The input date string.
|
||||
|
||||
@@ -279,13 +267,25 @@ def validate_and_reformat_date(date_str: str) -> str:
|
||||
if not isinstance(date_str, str):
|
||||
return date_str
|
||||
|
||||
# Strip pipes and whitespace so |01/01/2022| becomes 01/01/2022 before parsing.
|
||||
date_str = date_str.strip().strip("|").strip()
|
||||
if not date_str:
|
||||
return ""
|
||||
|
||||
try:
|
||||
# Check if the date is already in YYYY/MM/DD format
|
||||
datetime.strptime(date_str, "%Y/%m/%d")
|
||||
return date_str
|
||||
except ValueError:
|
||||
# Attempt to reformat the date from known formats
|
||||
known_formats = ["%Y-%m-%d", "%m/%d/%Y", "%d-%b-%Y", "%d/%m/%Y"]
|
||||
# Attempt to reformat the date from known formats (date-only and datetime).
|
||||
known_formats = [
|
||||
"%Y-%m-%d",
|
||||
"%m/%d/%Y",
|
||||
"%d-%b-%Y",
|
||||
"%d/%m/%Y",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S.%f",
|
||||
]
|
||||
for fmt in known_formats:
|
||||
try:
|
||||
return datetime.strptime(date_str, fmt).strftime("%Y/%m/%d")
|
||||
@@ -1211,6 +1211,14 @@ def _is_empty_list_placeholder(item) -> bool:
|
||||
return s in ("[]", "['[]']", '["[]"]') or s == ""
|
||||
|
||||
|
||||
def _strip_wrapping_single_quotes(s: str) -> str:
|
||||
"""Remove surrounding single quotes so \"'NV'\" becomes \"NV\"."""
|
||||
s = s.strip()
|
||||
if len(s) >= 2 and s[0] == "'" and s[-1] == "'":
|
||||
return s[1:-1]
|
||||
return s
|
||||
|
||||
|
||||
def format_as_json_list(val):
|
||||
"""
|
||||
Format a value as a JSON list string. Returns blank for empty lists
|
||||
@@ -1225,9 +1233,15 @@ def format_as_json_list(val):
|
||||
continue
|
||||
# If item is a list itself, extend (flatten)
|
||||
if isinstance(i, list):
|
||||
flattened.extend([str(x).strip() for x in i if x is not None])
|
||||
flattened.extend(
|
||||
[
|
||||
_strip_wrapping_single_quotes(str(x).strip())
|
||||
for x in i
|
||||
if x is not None and not _is_empty_list_placeholder(x)
|
||||
]
|
||||
)
|
||||
else:
|
||||
flattened.append(str(i).strip())
|
||||
flattened.append(_strip_wrapping_single_quotes(str(i).strip()))
|
||||
|
||||
return json.dumps(flattened) if flattened else ""
|
||||
if pd.isna(val) or val == "" or val == "[]":
|
||||
@@ -1242,7 +1256,7 @@ def format_as_json_list(val):
|
||||
parsed = json.loads(text)
|
||||
if isinstance(parsed, list):
|
||||
filtered = [
|
||||
str(i).strip()
|
||||
_strip_wrapping_single_quotes(str(i).strip())
|
||||
for i in parsed
|
||||
if i is not None and not _is_empty_list_placeholder(i)
|
||||
]
|
||||
@@ -1250,9 +1264,36 @@ def format_as_json_list(val):
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
text = text.strip("[]") # Strip brackets if it's a "fake" list like ['A']
|
||||
|
||||
# 3. Handle comma-separated strings & Clean special characters
|
||||
# This regex keeps letters, numbers, commas, and spaces
|
||||
clean_text = re.sub(r"[^a-zA-Z0-9, ]+", "", text)
|
||||
# 3. Pipe-delimited string (e.g. CHIP|MMC from crosswalk/derived fields)
|
||||
if "|" in text:
|
||||
items = [
|
||||
_strip_wrapping_single_quotes(i.strip())
|
||||
for i in text.split("|")
|
||||
if i.strip()
|
||||
]
|
||||
return json.dumps(items) if items else ""
|
||||
|
||||
# 4. Handle comma-separated or single string; keep only letters, digits, comma,
|
||||
# space, apostrophe, hyphen (e.g. "Children's / Medicaid-Medicare (MM)" ->
|
||||
# "Children's Medicaid-Medicare MM")
|
||||
clean_text = re.sub(r"[^a-zA-Z0-9, ' -]+", "", text)
|
||||
items = [i.strip() for i in clean_text.split(",") if i.strip()]
|
||||
items = [
|
||||
_strip_wrapping_single_quotes(re.sub(r"\s+", " ", i).strip())
|
||||
for i in items
|
||||
if i
|
||||
]
|
||||
|
||||
return json.dumps(items) if items else ""
|
||||
|
||||
|
||||
def format_prov_info_json(val) -> str:
|
||||
"""
|
||||
Format PROV_INFO_JSON cell as a valid JSON string (list of objects).
|
||||
|
||||
Delegates to json_utils.format_prov_info_json after normalizing pandas NaN.
|
||||
Returns "[]" for empty; otherwise valid JSON string.
|
||||
"""
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
val = None
|
||||
return json_utils.format_prov_info_json(val)
|
||||
|
||||
Reference in New Issue
Block a user