jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Update fill_na_mapping

Browse Source
This commit is contained in:
Katon Minhas
2026-02-02 23:44:27 -05:00
parent 62605981c1
commit cdce35739b
3 changed files with 66 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipelines/shared/postprocessing/aarete_derived.py
+46 -40
View File
@@ -27,73 +27,79 @@ def fill_na_mapping(answer_dicts):
Returns:
list[dict]: List of answer dictionaries with filled/merged values
"""
print("Answer dicts for fill_na_mapping:", answer_dicts)
for answer_dict in answer_dicts:
# Collect all AARETE_DERIVED_LOB values from different sources
all_lob_values = set()
# Get existing AARETE_DERIVED_LOB values (if any)
existing_lob = answer_dict.get("AARETE_DERIVED_LOB", "")
if not string_utils.is_empty(existing_lob):
if "|" in existing_lob:
all_lob_values.update(
v.strip()
for v in existing_lob.split("|")
if v.strip() and v.strip() != "N/A"
)
elif existing_lob.strip() and existing_lob.strip() != "N/A":
all_lob_values.add(existing_lob.strip())
existing_lob_list = answer_dict.get("AARETE_DERIVED_LOB", "")
if not string_utils.is_empty(existing_lob_list):
for lob_val in existing_lob_list:
if "|" in existing_lob_list:
all_lob_values.update(
v.strip()
for v in lob_val.split("|")
if v.strip() and v.strip() != "N/A"
)
elif lob_val.strip() and lob_val.strip() != "N/A":
all_lob_values.add(lob_val)
# Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
program_lob_values = set()
if not string_utils.is_empty(answer_dict.get("AARETE_DERIVED_PROGRAM")):
program_filled_value = fill_na_from_field(
program_filled_value_list = fill_na_from_field(
answer_dict,
"AARETE_DERIVED_LOB",
"AARETE_DERIVED_PROGRAM",
"src/constants/mappings/crosswalk_program_lob.json",
)
print("program_filled_value: ", program_filled_value_list)
if (
not string_utils.is_empty(program_filled_value)
and program_filled_value != "N/A"
not string_utils.is_empty(program_filled_value_list) and "N/A" not in program_filled_value_list
):
if "|" in program_filled_value:
program_lob_values = set(
v.strip()
for v in program_filled_value.split("|")
if v.strip() and v.strip() != "N/A"
)
elif (
program_filled_value.strip()
and program_filled_value.strip() != "N/A"
):
program_lob_values = {program_filled_value.strip()}
all_lob_values.update(program_lob_values)
for program_lob_val in program_filled_value_list:
if "|" in program_lob_val:
program_lob_values = set(
v.strip()
for v in program_lob_val.split("|")
if v.strip() and v.strip() != "N/A"
)
elif (
program_lob_val.strip()
and program_lob_val.strip() != "N/A"
):
program_lob_values = {program_lob_val.strip()}
all_lob_values.update(program_lob_values)
# Get AARETE_DERIVED_LOB from PRODUCT crosswalk (always check if PRODUCT exists)
product_lob_values = set()
if not string_utils.is_empty(answer_dict.get("PRODUCT")):
product_filled_value = fill_na_from_field(
product_filled_value_list = fill_na_from_field(
answer_dict,
"AARETE_DERIVED_LOB",
"PRODUCT",
"src/constants/mappings/crosswalk_product_lob.json",
)
if (
not string_utils.is_empty(product_filled_value)
and product_filled_value != "N/A"
not string_utils.is_empty(product_filled_value_list)
and "N/A" not in product_filled_value_list
):
if "|" in product_filled_value:
product_lob_values = set(
v.strip()
for v in product_filled_value.split("|")
if v.strip() and v.strip() != "N/A"
)
elif (
product_filled_value.strip()
and product_filled_value.strip() != "N/A"
):
product_lob_values = {product_filled_value.strip()}
all_lob_values.update(product_lob_values)
for product_lob_val in product_filled_value_list:
if "|" in product_lob_val:
product_lob_values = set(
v.strip()
for v in product_filled_value_list.split("|")
if v.strip() and v.strip() != "N/A"
)
elif (
product_lob_val.strip()
and product_lob_val.strip() != "N/A"
):
product_lob_values = {product_lob_val.strip()}
all_lob_values.update(product_lob_values)
# Set the merged, deduplicated value
if all_lob_values: