From 62605981c15396d6794529f0dbc5da80a0ceba13 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Mon, 2 Feb 2026 22:23:23 -0500 Subject: [PATCH] resolve hsc issue --- .../shared/extraction/tin_npi_funcs.py | 14 ++++++------- .../shared/postprocessing/aarete_derived.py | 11 +++------- .../hybrid_smart_chunking_funcs.py | 21 +++++-------------- 3 files changed, 15 insertions(+), 31 deletions(-) diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index b7eb859..e991b80 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -461,10 +461,10 @@ def get_provider_info( provider_fields = FieldSet( file_path=config.FIELD_JSON_PATH, field_type="provider_info" ) - prompt = prompt_templates.TIN_NPI_TEMPLATE( + prompt, _parser = prompt_templates.TIN_NPI_TEMPLATE( chunk, provider_fields.print_prompt_dict(), payer_name ) - claude_answer_raw = llm_utils.invoke_claude( + llm_answer_raw = llm_utils.invoke_claude( prompt, "sonnet_latest", filename, @@ -474,12 +474,12 @@ def get_provider_info( ) # Sometimes rosters can have 50+ provider logging.debug( - f"Raw LLM response for provider info on page {page_num}: {claude_answer_raw}" + f"Raw LLM response for provider info on page {page_num}: {llm_answer_raw}" ) # Defensive programming for any json parsing errors try: - providers = string_utils.universal_json_load(claude_answer_raw) + providers = _parser(llm_answer_raw) logging.debug(f"Parsed provider info on page {page_num}: {providers}") # Ensure we have a list of providers (not a dict or other type) if isinstance(providers, dict): @@ -521,13 +521,13 @@ def get_provider_info( for provider in providers: if provider.get("TIN") and provider["TIN"] != "UNKNOWN": # Normalize for comparison (remove hyphens/dots) - raw_tins = [tin.strip() for tin in provider["TIN"].split("|")] + raw_tins = [tin.strip() for tin in provider["TIN"]] normalized_tins = [ tin.replace("-", "").replace(".", "") for tin in raw_tins ] llm_found_tins.extend(normalized_tins) if provider.get("NPI") and provider["NPI"] != "UNKNOWN": - raw_npis = [npi.strip() for npi in provider["NPI"].split("|")] + raw_npis = [npi.strip() for npi in provider["NPI"]] normalized_npis = [ npi.replace("-", "").replace(".", "") for npi in raw_npis ] @@ -546,7 +546,7 @@ def get_provider_info( except ValueError as e: # Handle JSON parsing error logging.error(f"Error parsing JSON response from LLM: {str(e)}") - logging.error(f"Raw response: {claude_answer_raw}") + logging.error(f"Raw response: {llm_answer_raw}") # Return a default value return [{"TIN": "UNKNOWN", "NPI": "UNKNOWN", "NAME": "PARSING_ERROR"}] diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 3c004fa..6bccc66 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -145,11 +145,8 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): if not string_utils.is_empty( from_field_value ) and string_utils.is_empty(to_field_value): - # Get list of values to map - if "|" in from_field_value: - from_field_value_list = from_field_value.split("|") - elif "," in from_field_value: - from_field_value_list = from_field_value.split(",") + if isinstance(from_field_value, list): + from_field_value_list = from_field_value else: from_field_value_list = [from_field_value] @@ -159,7 +156,6 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): individual_from_field_value = ( individual_from_field_value.strip() ) - if individual_from_field_value in crosswalk.mapping.keys(): # Value is a key in the mapping - map it to the target value to_field_answer_list.append( @@ -176,6 +172,5 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): individual_from_field_value ) ) - answer_dict[to_field_name] = "|".join(to_field_answer_list) - + answer_dict[to_field_name] = to_field_answer_list return answer_dicts diff --git a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py index 594eebf..5d3e9a7 100644 --- a/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py +++ b/src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py @@ -279,14 +279,6 @@ def run_hybrid_smart_chunked_fields( answers_dict, field_name="PROVIDER_STATE" ) - # Normalize PAYER_STATE to standardized two-letter abbreviation - for state_field in ["PAYER_STATE"]: - if state_field in answers_dict: - answers_dict[state_field] = ( - string_utils.normalize_state_to_abbreviation( - answers_dict[state_field] - ) - ) # Normalize PAYER_STATE answers_dict = string_utils.normalize_state_field( answers_dict, field_name="PAYER_STATE" @@ -313,7 +305,7 @@ def extract_amendment_num_from_filename(answer_dict: dict, filename: str) -> dic or answer_dict.get("CONTRACT_AMENDMENT_NUM") == "UNKNOWN" ): - prompt = prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename) + prompt, _parser = prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename) llm_answer_raw = llm_utils.invoke_claude( prompt, "sonnet_latest", @@ -324,7 +316,7 @@ def extract_amendment_num_from_filename(answer_dict: dict, filename: str) -> dic ) try: - llm_answer_final = string_utils.universal_json_load(llm_answer_raw) + llm_answer_final = _parser(llm_answer_raw) answer_dict["FILENAME_AMENDMENT_NUM"] = llm_answer_final.get( "amendment_number", "N/A" ) @@ -389,13 +381,10 @@ def prompt_hsc_single_field( try: llm_answer_final = _parser(llm_answer_raw) # returns dict - val = llm_answer_final.get(field.field_name) - if isinstance(val, list): - llm_answer_final[field.field_name] = val - elif isinstance(val, str): - llm_answer_final[field.field_name] = [val] + field_value = llm_answer_final.get(field.field_name) + if not isinstance(field_value, list): + field_value = [field_value] - field_value = llm_answer_final[field.field_name] if field_value[0] == "N/A": field.field_type = "full_context" if field.field_name == "PAYER_NAME":