refactor: complete JSON format standardization updates

- Updated all prompt_calls.py files (saas, bcbs_promise, clover) to use new parser pattern
- Updated prompt_templates.py with JSON format instructions
- Updated crosswalk_utils.py to handle JSON lists
- Updated json_utils.py with improved parsing
- Updated aarete_derived.py for JSON compatibility
- Updated tin_npi_funcs.py and qa_qc_utils.py for JSON parsing
- All changes align with Phase 4 completion of JSON standardization

src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py

@@ -129,18 +129,22 @@ def prompt_reimbursement_primary(
                         service_term = answer_dict["SERVICE_TERM"]
                         if isinstance(service_term, list):
                             # If list, join with comma (shouldn't happen, but defensive)
-                            answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
+                            answer_dict["SERVICE_TERM"] = ", ".join(
+                                str(item) for item in service_term if item
+                            )
                         elif service_term is not None:
                             answer_dict["SERVICE_TERM"] = str(service_term)
                         else:
                             answer_dict["SERVICE_TERM"] = ""
-                    
+
                     # REIMB_TERM should be a string per row
                     if "REIMB_TERM" in answer_dict:
                         reimb_term = answer_dict["REIMB_TERM"]
                         if isinstance(reimb_term, list):
                             # If list, join with comma (shouldn't happen, but defensive)
-                            answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
+                            answer_dict["REIMB_TERM"] = ", ".join(
+                                str(item) for item in reimb_term if item
+                            )
                         elif reimb_term is not None:
                             answer_dict["REIMB_TERM"] = str(reimb_term)
                         else:
@@ -181,11 +185,16 @@ def prompt_methodology_breakout(
         # FEE_SCHEDULE should be a string per methodology breakout result
         if isinstance(methodology_breakout_answers, list):
             for methodology_dict in methodology_breakout_answers:
-                if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
+                if (
+                    isinstance(methodology_dict, dict)
+                    and "FEE_SCHEDULE" in methodology_dict
+                ):
                     fee_schedule = methodology_dict["FEE_SCHEDULE"]
                     if isinstance(fee_schedule, list):
                         # If list, take first element (shouldn't happen, but defensive)
-                        methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
+                        methodology_dict["FEE_SCHEDULE"] = (
+                            fee_schedule[0] if fee_schedule else ""
+                        )
                     elif fee_schedule is not None:
                         methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
                     else:
@@ -211,7 +220,7 @@ def prompt_fee_schedule_breakout(
         # If list, take first element (shouldn't happen, but defensive)
         fee_schedule = fee_schedule[0] if fee_schedule else ""
     fee_schedule = str(fee_schedule) if fee_schedule else ""
-    
+
     prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
         reimbursement_method,
         fee_schedule,

src/pipelines/clients/clover/prompts/prompt_calls.py

@@ -136,18 +136,22 @@ def prompt_reimbursement_primary(
                         service_term = answer_dict["SERVICE_TERM"]
                         if isinstance(service_term, list):
                             # If list, join with comma (shouldn't happen, but defensive)
-                            answer_dict["SERVICE_TERM"] = ", ".join(str(item) for item in service_term if item)
+                            answer_dict["SERVICE_TERM"] = ", ".join(
+                                str(item) for item in service_term if item
+                            )
                         elif service_term is not None:
                             answer_dict["SERVICE_TERM"] = str(service_term)
                         else:
                             answer_dict["SERVICE_TERM"] = ""
-                    
+
                     # REIMB_TERM should be a string per row
                     if "REIMB_TERM" in answer_dict:
                         reimb_term = answer_dict["REIMB_TERM"]
                         if isinstance(reimb_term, list):
                             # If list, join with comma (shouldn't happen, but defensive)
-                            answer_dict["REIMB_TERM"] = ", ".join(str(item) for item in reimb_term if item)
+                            answer_dict["REIMB_TERM"] = ", ".join(
+                                str(item) for item in reimb_term if item
+                            )
                         elif reimb_term is not None:
                             answer_dict["REIMB_TERM"] = str(reimb_term)
                         else:
@@ -188,11 +192,16 @@ def prompt_methodology_breakout(
         # FEE_SCHEDULE should be a string per methodology breakout result
         if isinstance(methodology_breakout_answers, list):
             for methodology_dict in methodology_breakout_answers:
-                if isinstance(methodology_dict, dict) and "FEE_SCHEDULE" in methodology_dict:
+                if (
+                    isinstance(methodology_dict, dict)
+                    and "FEE_SCHEDULE" in methodology_dict
+                ):
                     fee_schedule = methodology_dict["FEE_SCHEDULE"]
                     if isinstance(fee_schedule, list):
                         # If list, take first element (shouldn't happen, but defensive)
-                        methodology_dict["FEE_SCHEDULE"] = fee_schedule[0] if fee_schedule else ""
+                        methodology_dict["FEE_SCHEDULE"] = (
+                            fee_schedule[0] if fee_schedule else ""
+                        )
                     elif fee_schedule is not None:
                         methodology_dict["FEE_SCHEDULE"] = str(fee_schedule)
                     else:
@@ -218,7 +227,7 @@ def prompt_fee_schedule_breakout(
         # If list, take first element (shouldn't happen, but defensive)
         fee_schedule = fee_schedule[0] if fee_schedule else ""
     fee_schedule = str(fee_schedule) if fee_schedule else ""
-    
+
     prompt, _parser = prompt_templates.FEE_SCHEDULE_BREAKOUT(
         reimbursement_method,
         fee_schedule,

src/pipelines/shared/extraction/tin_npi_funcs.py

@@ -294,7 +294,9 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
 
     # Filter out records with NO_IDENTIFIERS_FOUND
     prov_info_json_list = [
-        p for p in prov_info_json_list if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or [])
+        p
+        for p in prov_info_json_list
+        if "NO_IDENTIFIERS_FOUND" not in (p.get("NAME") or [])
     ]
 
     # Deduplicate provider_list by NAME, keeping records with actual TIN/NPI values
@@ -313,7 +315,7 @@ def merge_provider_info_with_hybrid_smart_chunking(one_to_one_results, filename)
     # Merge other provider information into one_to_one_results
     one_to_one_results["PROV_OTHER_TIN"] = other_tins if other_tins else []
     one_to_one_results["PROV_OTHER_NPI"] = other_npis if other_npis else []
-    one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else [] 
+    one_to_one_results["PROV_OTHER_NAME_FULL"] = other_names if other_names else []
     return one_to_one_results
 
 
@@ -426,7 +428,7 @@ def deduplicate_providers(
             tuple(npis),
             tuple(names),
         )
-    
+
         if key not in seen:
             seen.add(key)
             valid_providers.append(provider)

src/pipelines/shared/postprocessing/aarete_derived.py

@@ -50,7 +50,11 @@ def fill_na_mapping(answer_dicts):
 
             # Process each LOB value
             for lob_val in lob_items:
-                if isinstance(lob_val, str) and lob_val.strip() and lob_val.strip() != "N/A":
+                if (
+                    isinstance(lob_val, str)
+                    and lob_val.strip()
+                    and lob_val.strip() != "N/A"
+                ):
                     all_lob_values.add(lob_val.strip())
 
         # Get AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM crosswalk (always check if PROGRAM exists)
@@ -72,7 +76,9 @@ def fill_na_mapping(answer_dicts):
                 elif isinstance(program_filled_value_list, str):
                     # Handle pipe-delimited format (backward compatibility)
                     if "|" in program_filled_value_list:
-                        program_lob_items = [v.strip() for v in program_filled_value_list.split("|")]
+                        program_lob_items = [
+                            v.strip() for v in program_filled_value_list.split("|")
+                        ]
                     else:
                         program_lob_items = [program_filled_value_list]
                 else:
@@ -80,7 +86,11 @@ def fill_na_mapping(answer_dicts):
 
                 # Process each program LOB value
                 for program_lob_val in program_lob_items:
-                    if isinstance(program_lob_val, str) and program_lob_val.strip() and program_lob_val.strip() != "N/A":
+                    if (
+                        isinstance(program_lob_val, str)
+                        and program_lob_val.strip()
+                        and program_lob_val.strip() != "N/A"
+                    ):
                         program_lob_values.add(program_lob_val.strip())
                 # Update all_lob_values after processing all program LOB values
                 all_lob_values.update(program_lob_values)
@@ -104,7 +114,9 @@ def fill_na_mapping(answer_dicts):
                 elif isinstance(product_filled_value_list, str):
                     # Handle pipe-delimited format (backward compatibility)
                     if "|" in product_filled_value_list:
-                        product_lob_items = [v.strip() for v in product_filled_value_list.split("|")]
+                        product_lob_items = [
+                            v.strip() for v in product_filled_value_list.split("|")
+                        ]
                     else:
                         product_lob_items = [product_filled_value_list]
                 else:
@@ -112,7 +124,11 @@ def fill_na_mapping(answer_dicts):
 
                 # Process each product LOB value
                 for product_lob_val in product_lob_items:
-                    if isinstance(product_lob_val, str) and product_lob_val.strip() and product_lob_val.strip() != "N/A":
+                    if (
+                        isinstance(product_lob_val, str)
+                        and product_lob_val.strip()
+                        and product_lob_val.strip() != "N/A"
+                    ):
                         product_lob_values.add(product_lob_val.strip())
                 # Update all_lob_values after processing all product LOB values
                 all_lob_values.update(product_lob_values)
@@ -192,13 +208,17 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
                                 # Reverse mapping returns dict[str, list[str]], so .get() returns a list
                                 # We need to extend, not append, to avoid nested lists
                                 reverse_mapping = crosswalk.create_reverse_mapping()
-                                reverse_result = reverse_mapping.get(individual_from_field_value)
+                                reverse_result = reverse_mapping.get(
+                                    individual_from_field_value
+                                )
                                 if reverse_result is not None:
                                     # reverse_result is a list, extend it
                                     to_field_answer_list.extend(reverse_result)
                                 else:
                                     # If not found in reverse mapping, keep original value
-                                    to_field_answer_list.append(individual_from_field_value)
+                                    to_field_answer_list.append(
+                                        individual_from_field_value
+                                    )
                     answer_dict[to_field_name] = to_field_answer_list
                     # update from field value to list if not already
                     if not isinstance(from_field_value, list):

src/prompts/prompt_templates.py

@@ -890,7 +890,7 @@ Populate a JSON dictionary with the following fields:
 
 
 def FEE_SCHEDULE_BREAKOUT(
-    methodology:str, fee_schedule:str
+    methodology: str, fee_schedule: str
 ) -> Tuple[str, Callable[[str], dict]]:
     """Returns ONLY dynamic content for fee schedule breakout.
     Call FEE_SCHEDULE_BREAKOUT_INSTRUCTION() separately for the cached instruction.

src/utils/crosswalk_utils.py

@@ -4,7 +4,9 @@ import logging
 import src.utils.string_utils as string_utils
 
 
-def apply_crosswalk(val: str | list, mapping: dict[str, str], default: str = "N/A") -> list:
+def apply_crosswalk(
+    val: str | list, mapping: dict[str, str], default: str = "N/A"
+) -> list:
     """Apply a crosswalk to a value
 
     Args:

src/utils/json_utils.py

@@ -253,4 +253,4 @@ def parse_json_dict_or_list(raw_llm_output: str) -> dict[str, Any] | list[Any]:
     raise ValueError(
         f"No valid JSON object found in LLM output. "
         f"Output preview: {raw_llm_output[:200]}..."
-    )
+    )

src/utils/qa_qc_utils.py

@@ -1101,7 +1101,9 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
         Tuple of (formatted_tins, has_invalid)
     """
     # Handle list/array values (e.g., from PROV_GROUP_TIN, PROV_OTHER_TIN)
-    if isinstance(val, (list, tuple)) or (hasattr(val, '__iter__') and not isinstance(val, str)):
+    if isinstance(val, (list, tuple)) or (
+        hasattr(val, "__iter__") and not isinstance(val, str)
+    ):
         # Convert list to list of strings, filtering out empty values
         if len(val) == 0:
             return ("", True)
@@ -1121,10 +1123,10 @@ def format_or_preserve_tins(val: Any) -> Tuple[str, bool]:
         except (ValueError, TypeError):
             # pd.isna() can't handle arrays, but we've already handled lists above
             pass
-        
+
         if val is None or str(val).strip() == "":
             return ("", True)
-        
+
         # Single value - convert to list for consistent processing
         parts = [str(val).strip()]
     out, any_invalid = [], False