From 5b619b2f5f943334c5dd1939ccbcc0b9129a1e47 Mon Sep 17 00:00:00 2001 From: ppanchigar Date: Wed, 4 Feb 2026 10:57:40 -0600 Subject: [PATCH] fixed code implicit special and ran black for CI --- src/codes/code_funcs.py | 9 ++- src/config.py | 2 +- .../bcbs_promise/prompts/prompt_calls.py | 6 +- .../clients/clover/prompts/prompt_calls.py | 6 +- src/pipelines/runner.py | 12 ++-- src/pipelines/saas/file_processing.py | 1 - src/pipelines/saas/main.py | 10 +++- src/pipelines/saas/prompts/prompt_calls.py | 6 +- .../shared/extraction/tin_npi_funcs.py | 56 ++++++++++++------- .../shared/postprocessing/aarete_derived.py | 9 ++- src/tests/test_tin_npi_funcs.py | 1 - src/utils/string_utils.py | 12 ++-- 12 files changed, 81 insertions(+), 49 deletions(-) diff --git a/src/codes/code_funcs.py b/src/codes/code_funcs.py index 10aee0e..c99e273 100644 --- a/src/codes/code_funcs.py +++ b/src/codes/code_funcs.py @@ -229,8 +229,13 @@ def code_implicit_special(service, filename): "ST": ["ST Codes TBD"], } code_answer_dict = {} - if llm_answer_final[0] in special_case_mapping: - code_answer_dict["PROCEDURE_CD"] = special_case_mapping[llm_answer_final] + # Check if llm_answer_final is a list and has at least one element, and if the first element is in the special_case_mapping + if ( + isinstance(llm_answer_final, list) + and len(llm_answer_final) > 0 + and llm_answer_final[0] in special_case_mapping + ): + code_answer_dict["PROCEDURE_CD"] = special_case_mapping[llm_answer_final[0]] code_answer_dict["PROCEDURE_CD_DESC"] = llm_answer_final return code_answer_dict diff --git a/src/config.py b/src/config.py index 631159f..180807d 100644 --- a/src/config.py +++ b/src/config.py @@ -93,7 +93,7 @@ def get_arg_value(arg_name, default): # Run config args RUN_MODE = get_arg_value("run_mode", "ec2") # Valid: local or ec2 READ_MODE = get_arg_value("read_mode", "s3") # Valid: local or s3 -BATCH_ID = get_arg_value("batch_id", None) # BATCH_ID is mandatory +BATCH_ID = get_arg_value("batch_id", None) # BATCH_ID is mandatory # Per-file logging settings (for debugging) ENABLE_PER_FILE_LOGGING = get_arg_value("enable_per_file_logging", "False") == "True" diff --git a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py index 35c1dde..f87d6f0 100644 --- a/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py +++ b/src/pipelines/clients/bcbs_promise/prompts/prompt_calls.py @@ -34,8 +34,10 @@ def prompt_exhibit_level( # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings # This field should be a single value even though it's a 1:N field if "CLAIM_TYPE_CD" in llm_answer_final: - llm_answer_final["CLAIM_TYPE_CD"] = string_utils.normalize_one_to_one_field_value( - "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + llm_answer_final["CLAIM_TYPE_CD"] = ( + string_utils.normalize_one_to_one_field_value( + "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + ) ) return llm_answer_final diff --git a/src/pipelines/clients/clover/prompts/prompt_calls.py b/src/pipelines/clients/clover/prompts/prompt_calls.py index 80c45a2..f78aeda 100644 --- a/src/pipelines/clients/clover/prompts/prompt_calls.py +++ b/src/pipelines/clients/clover/prompts/prompt_calls.py @@ -40,8 +40,10 @@ def prompt_exhibit_level( # Normalize CLAIM_TYPE_CD if present: convert single-element lists to strings # This field should be a single value even though it's a 1:N field if "CLAIM_TYPE_CD" in llm_answer_final: - llm_answer_final["CLAIM_TYPE_CD"] = string_utils.normalize_one_to_one_field_value( - "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + llm_answer_final["CLAIM_TYPE_CD"] = ( + string_utils.normalize_one_to_one_field_value( + "CLAIM_TYPE_CD", llm_answer_final["CLAIM_TYPE_CD"] + ) ) return llm_answer_final diff --git a/src/pipelines/runner.py b/src/pipelines/runner.py index 08629f3..f328a5c 100644 --- a/src/pipelines/runner.py +++ b/src/pipelines/runner.py @@ -108,12 +108,16 @@ def main(client: str = "saas", testing=False, test_params={}): """ # Check if batch_id is specified if not config.BATCH_ID: - raise ValueError("batch_id is required. Please specify batch_id in your command") - + raise ValueError( + "batch_id is required. Please specify batch_id in your command" + ) + # Check if max_workers is not negative if config.MAX_WORKERS < 0: - raise ValueError(f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})") - + raise ValueError( + f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})" + ) + # Validate client registry = get_registry() if client != "saas" and not registry.has_client(client): diff --git a/src/pipelines/saas/file_processing.py b/src/pipelines/saas/file_processing.py index 550f416..9628810 100644 --- a/src/pipelines/saas/file_processing.py +++ b/src/pipelines/saas/file_processing.py @@ -228,7 +228,6 @@ def run_one_to_one_prompts( # Add HSC's N/A if field doesn't exist yet one_to_one_results[key] = value - one_to_one_results = tin_npi_funcs.merge_provider_info_with_one_to_one( one_to_one_results, filename ) diff --git a/src/pipelines/saas/main.py b/src/pipelines/saas/main.py index 243bf64..0a74473 100644 --- a/src/pipelines/saas/main.py +++ b/src/pipelines/saas/main.py @@ -86,11 +86,15 @@ def safe_process_file(item, constants, run_timestamp): def main(testing=False, test_params={}): # Check if batch_id is specified if not config.BATCH_ID: - raise ValueError("batch_id is required. Please specify batch_id in your command") - + raise ValueError( + "batch_id is required. Please specify batch_id in your command" + ) + # Check if max_workers is not negative if config.MAX_WORKERS < 0: - raise ValueError(f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})") + raise ValueError( + f"Invalid configuration: MAX_WORKERS must be non-negative (got {config.MAX_WORKERS})" + ) # Set up per-file logging (creates separate log files for each document) logging_utils.setup_per_file_logging() diff --git a/src/pipelines/saas/prompts/prompt_calls.py b/src/pipelines/saas/prompts/prompt_calls.py index d0c213d..691a8b7 100644 --- a/src/pipelines/saas/prompts/prompt_calls.py +++ b/src/pipelines/saas/prompts/prompt_calls.py @@ -464,7 +464,7 @@ def prompt_dynamic(text: str, field_prompts, filename): cache=True, instruction=prompt_templates.EXHIBIT_LEVEL_INSTRUCTION(), ) # Returns dictionary of lists - + logging.debug(f"Dynamic answer for {filename}: {llm_answer_raw}") llm_answer_final = _parser(llm_answer_raw) return llm_answer_final @@ -964,11 +964,11 @@ def prompt_provider_info( # VALIDATION LAYER - Cross-check with regex findings page_text = text_dict[page_num] tin_npi_funcs.flag_missed_prov_info(page_text, providers, page_num, filename) - + return providers except ValueError as e: # Handle JSON parsing error logging.error(f"Error parsing JSON response from LLM: {str(e)}") logging.error(f"Raw response: {llm_answer_raw}") # Return a default value - return [{"TIN": "N/A", "NPI": "N/A", "NAME": "N/A"}] \ No newline at end of file + return [{"TIN": "N/A", "NPI": "N/A", "NAME": "N/A"}] diff --git a/src/pipelines/shared/extraction/tin_npi_funcs.py b/src/pipelines/shared/extraction/tin_npi_funcs.py index 9c5d93b..b27c8c4 100644 --- a/src/pipelines/shared/extraction/tin_npi_funcs.py +++ b/src/pipelines/shared/extraction/tin_npi_funcs.py @@ -30,8 +30,10 @@ def run_provider_info_fields( all_tins, all_npis, tin_matches, npi_matches = get_all_tins_and_npis(contract_text) - exact_tins, exact_npis, ocr_corrected_tins, ocr_corrected_npis = get_tin_extraction_quality(tin_matches, npi_matches, filename) - + exact_tins, exact_npis, ocr_corrected_tins, ocr_corrected_npis = ( + get_tin_extraction_quality(tin_matches, npi_matches, filename) + ) + relevant_pages = get_relevant_pages(all_tins, all_npis, text_dict, filename) prov_info_json = get_prov_info_json(relevant_pages, text_dict, filename, payer_name) @@ -70,7 +72,7 @@ def get_all_tins_and_npis(contract_text): all_npis = [npi for npi, quality in npi_matches] logging.debug(f"Extracted TINs: {all_tins}") logging.debug(f"Extracted NPIs: {all_npis}") - + return all_tins, all_npis, tin_matches, npi_matches @@ -207,7 +209,7 @@ def normalize_to_lists(provider: dict): tins = provider.get("TIN") npis = provider.get("NPI") names = provider.get("NAME") - + # Normalize to lists: if string, convert to list to avoid character splitting if isinstance(tins, str): tins = [tins] @@ -221,10 +223,13 @@ def normalize_to_lists(provider: dict): names = [names] elif not isinstance(names, list): names = [] - + return tins, npis, names -def deunknown_and_deduplicate(group_tins, group_npis, group_names, other_tins, other_npis, other_names): + +def deunknown_and_deduplicate( + group_tins, group_npis, group_names, other_tins, other_npis, other_names +): # De-Unknown GROUP fields group_tins = [v for v in group_tins if not string_utils.is_empty(v)] @@ -234,20 +239,25 @@ def deunknown_and_deduplicate(group_tins, group_npis, group_names, other_tins, o # Deduplicate while preserving order and remove any "UNKNOWN" entries and overlaps between group and other group_tins = list(dict.fromkeys(group_tins)) other_tins = list(dict.fromkeys(other_tins)) - other_tins = [v for v in other_tins if v not in group_tins and not string_utils.is_empty(v)] + other_tins = [ + v for v in other_tins if v not in group_tins and not string_utils.is_empty(v) + ] group_npis = list(dict.fromkeys(group_npis)) other_npis = list(dict.fromkeys(other_npis)) - other_npis = [v for v in other_npis if v not in group_npis and not string_utils.is_empty(v)] + other_npis = [ + v for v in other_npis if v not in group_npis and not string_utils.is_empty(v) + ] group_names = list(dict.fromkeys(group_names)) other_names = list(dict.fromkeys(other_names)) - other_names = [v for v in other_names if v not in group_names and not string_utils.is_empty(v)] + other_names = [ + v for v in other_names if v not in group_names and not string_utils.is_empty(v) + ] return group_tins, group_npis, group_names, other_tins, other_npis, other_names - def merge_provider_info_with_one_to_one(one_to_one_results: dict, filename: str): """ Merges provider_info into one_to_one_results. @@ -271,12 +281,12 @@ def merge_provider_info_with_one_to_one(one_to_one_results: dict, filename: str) provider_name = [provider_name] elif not isinstance(provider_name, list): provider_name = [] - - prov_info_json = one_to_one_results[ - "PROV_INFO_JSON" - ] # Returns list + + prov_info_json = one_to_one_results["PROV_INFO_JSON"] # Returns list prov_info_json_list = ( - json.loads(prov_info_json) if isinstance(prov_info_json, str) else prov_info_json + json.loads(prov_info_json) + if isinstance(prov_info_json, str) + else prov_info_json ) # Track if we found any name matches found_match = False @@ -316,8 +326,10 @@ def merge_provider_info_with_one_to_one(one_to_one_results: dict, filename: str) for name in provider_name: group_names.add(name) - group_tins, group_npis, group_names, other_tins, other_npis, other_names = deunknown_and_deduplicate( - group_tins, group_npis, group_names, other_tins, other_npis, other_names + group_tins, group_npis, group_names, other_tins, other_npis, other_names = ( + deunknown_and_deduplicate( + group_tins, group_npis, group_names, other_tins, other_npis, other_names + ) ) # Filter out records with NO_IDENTIFIERS_FOUND @@ -370,7 +382,11 @@ def deduplicate_providers( name = prov_info_dict.get("NAME") # Skip providers where all critical identification fields are unknown - if (string_utils.is_empty(tin) and string_utils.is_empty(npi) and string_utils.is_empty(name)): + if ( + string_utils.is_empty(tin) + and string_utils.is_empty(npi) + and string_utils.is_empty(name) + ): continue key = (tin, npi, name) @@ -380,6 +396,7 @@ def deduplicate_providers( return valid_providers + def flag_missed_prov_info(page_text, providers, page_num, filename): # Get all TINs and NPIs found via regex on this page regex_tins = [ @@ -423,7 +440,6 @@ def flag_missed_prov_info(page_text, providers, page_num, filename): ) - def get_relevant_pages(all_tins, all_npis, text_dict, filename): # If there are no identifiers, return early with default values if not all_tins and not all_npis: @@ -451,6 +467,7 @@ def get_relevant_pages(all_tins, all_npis, text_dict, filename): relevant_pages = [relevant_pages[0]] return relevant_pages + def get_prov_info_json(relevant_pages, text_dict, filename, payer_name): if not relevant_pages: prov_info_json = [ @@ -475,7 +492,6 @@ def get_prov_info_json(relevant_pages, text_dict, filename, payer_name): return prov_info_json - def get_tin_extraction_quality(tin_matches, npi_matches, filename): # Log OCR corrections for monitoring exact_tins = [tin for tin, quality in tin_matches if quality == "EXACT"] diff --git a/src/pipelines/shared/postprocessing/aarete_derived.py b/src/pipelines/shared/postprocessing/aarete_derived.py index 585057e..8baf3c8 100644 --- a/src/pipelines/shared/postprocessing/aarete_derived.py +++ b/src/pipelines/shared/postprocessing/aarete_derived.py @@ -232,13 +232,16 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants): to_field_answer_list.append( individual_from_field_value ) - + # Assign the result: convert to string if it's a single-value field with one element - if to_field_name in SINGLE_VALUE_FIELDS and len(to_field_answer_list) == 1: + if ( + to_field_name in SINGLE_VALUE_FIELDS + and len(to_field_answer_list) == 1 + ): answer_dict[to_field_name] = to_field_answer_list[0] else: answer_dict[to_field_name] = to_field_answer_list - + # DO NOT update from_field_name - preserve its original format # The crosswalk should only create/update the target field (to_field_name), # not modify the source field (from_field_name) diff --git a/src/tests/test_tin_npi_funcs.py b/src/tests/test_tin_npi_funcs.py index 897fb4e..ac9d46f 100644 --- a/src/tests/test_tin_npi_funcs.py +++ b/src/tests/test_tin_npi_funcs.py @@ -30,7 +30,6 @@ class TestTinNpiFuncs: # Test no matches assert tin_npi_funcs.get_all_matches("No TINs here", pattern) == [] - @patch("src.pipelines.saas.prompts.prompt_calls.provider_name_match_check") def test_merge_provider_info_with_hybrid_smart_chunking( self, mock_name_match_check diff --git a/src/utils/string_utils.py b/src/utils/string_utils.py index 0af298b..0b905f4 100644 --- a/src/utils/string_utils.py +++ b/src/utils/string_utils.py @@ -867,7 +867,9 @@ def normalize_one_to_one_field_value(field_name: str, value) -> str | list: return str(value) if value is not None else "N/A" -def normalize_one_to_one_answers_dict(answers_dict: dict[str, str | list]) -> dict[str, str | list]: +def normalize_one_to_one_answers_dict( + answers_dict: dict[str, str | list], +) -> dict[str, str | list]: """ Normalize all values in a 1:1 answers dictionary. @@ -893,16 +895,12 @@ def normalize_one_to_one_answers_dict(answers_dict: dict[str, str | list]) -> di "PROVIDER_NAME": ["Provider A", "Provider B"] } """ - DO_NOT_NORMALIZE = [ - "PROV_INFO_JSON", - "PAYER_STATE", - "PROVIDER_STATE" - ] + DO_NOT_NORMALIZE = ["PROV_INFO_JSON", "PAYER_STATE", "PROVIDER_STATE"] normalized = {} for field_name, value in answers_dict.items(): if field_name not in DO_NOT_NORMALIZE: normalized[field_name] = normalize_one_to_one_field_value(field_name, value) else: normalized[field_name] = value - + return normalized