Update AARETE_DERIVED_PRODUCT processing

src/pipelines/saas/test.py

@@ -0,0 +1,13 @@
+
+from src.pipelines.shared.postprocessing import aarete_derived
+from src.constants.constants import Constants
+
+constants = Constants()
+
+all_exhibit_rows = [
+    {"PRODUCT" : ['HEALTHfirst', 'KIDSfirst']}
+]
+
+all_exhibit_rows = aarete_derived.get_crosswalk_fields(all_exhibit_rows, constants)
+
+print("all_exhibit_rows: ", all_exhibit_rows)

src/pipelines/shared/postprocessing/aarete_derived.py

@@ -27,8 +27,6 @@ def fill_na_mapping(answer_dicts):
     Returns:
         list[dict]: List of answer dictionaries with filled/merged values
     """
-    print("Answer dicts for fill_na_mapping:", answer_dicts)
-
     for answer_dict in answer_dicts:
         # Collect all AARETE_DERIVED_LOB values from different sources
         all_lob_values = set()
@@ -56,7 +54,6 @@ def fill_na_mapping(answer_dicts):
                 "AARETE_DERIVED_PROGRAM",
                 "src/constants/mappings/crosswalk_program_lob.json",
             )
-            print("program_filled_value: ", program_filled_value_list)
             if (
                 not string_utils.is_empty(program_filled_value_list) and "N/A" not in program_filled_value_list
             ):
@@ -137,20 +134,27 @@ def fill_na_mapping(answer_dicts):
 
 
 def get_crosswalk_fields(answer_dicts: list, constants: Constants):
+
     crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
+    print("crosswalk_fields: ", crosswalk_fields)
+
     for to_field in crosswalk_fields.fields:
+        print("to_field: ", to_field)
         to_field_name, from_field_name = to_field.field_name, to_field.base_field
         # Find crosswalk
         crosswalk = constants.get_constant(to_field.crosswalk)
         if crosswalk:
             for answer_dict in answer_dicts:
-                to_field_value, from_field_value = answer_dict.get(
+                to_field_value = answer_dict.get(to_field_name)
-                    to_field_name
+                from_field_value = answer_dict.get(from_field_name)
-                ), answer_dict.get(from_field_name)
+
+                print("to_field_value: ", to_field_value)
+                print("from_field_value: ", from_field_value)
+
                 # If from_field_value is populated and to_field_value is not populated, perform mapping
                 if not string_utils.is_empty(
                     from_field_value
-                ) and string_utils.is_empty(to_field_value):
+                ) and (string_utils.is_empty(to_field_value) or "N/A" in to_field_value):
                     if isinstance(from_field_value, list):
                         from_field_value_list = from_field_value
                     else:

src/pipelines/shared/postprocessing/postprocessing_funcs.py

@@ -518,7 +518,7 @@ def add_aarete_derived_product(df):
     The new column should be Title Case of the PRODUCT column value.
     """
     if "PRODUCT" in df.columns:
-        df["AARETE_DERIVED_PRODUCT"] = df["PRODUCT"].str.title()
+        df["AARETE_DERIVED_PRODUCT"] = df["PRODUCT"]
     return df