jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

resolve hsc issue

Browse Source
This commit is contained in:
Katon Minhas
2026-02-02 22:23:23 -05:00
parent 9b30c5f282
commit 62605981c1
3 changed files with 15 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/pipelines/shared/extraction/tin_npi_funcs.py
+7 -7
View File
@@ -461,10 +461,10 @@ def get_provider_info(
provider_fields = FieldSet(
file_path=config.FIELD_JSON_PATH, field_type="provider_info"
)
prompt = prompt_templates.TIN_NPI_TEMPLATE(
prompt, _parser = prompt_templates.TIN_NPI_TEMPLATE(
chunk, provider_fields.print_prompt_dict(), payer_name
)
claude_answer_raw = llm_utils.invoke_claude(
llm_answer_raw = llm_utils.invoke_claude(
prompt,
"sonnet_latest",
filename,
@@ -474,12 +474,12 @@ def get_provider_info(
) # Sometimes rosters can have 50+ provider
logging.debug(
f"Raw LLM response for provider info on page {page_num}: {claude_answer_raw}"
f"Raw LLM response for provider info on page {page_num}: {llm_answer_raw}"
)
# Defensive programming for any json parsing errors
try:
providers = string_utils.universal_json_load(claude_answer_raw)
providers = _parser(llm_answer_raw)
logging.debug(f"Parsed provider info on page {page_num}: {providers}")
# Ensure we have a list of providers (not a dict or other type)
if isinstance(providers, dict):
@@ -521,13 +521,13 @@ def get_provider_info(
for provider in providers:
if provider.get("TIN") and provider["TIN"] != "UNKNOWN":
# Normalize for comparison (remove hyphens/dots)
raw_tins = [tin.strip() for tin in provider["TIN"].split("|")]
raw_tins = [tin.strip() for tin in provider["TIN"]]
normalized_tins = [
tin.replace("-", "").replace(".", "") for tin in raw_tins
]
llm_found_tins.extend(normalized_tins)
if provider.get("NPI") and provider["NPI"] != "UNKNOWN":
raw_npis = [npi.strip() for npi in provider["NPI"].split("|")]
raw_npis = [npi.strip() for npi in provider["NPI"]]
normalized_npis = [
npi.replace("-", "").replace(".", "") for npi in raw_npis
]
@@ -546,7 +546,7 @@ def get_provider_info(
except ValueError as e: # Handle JSON parsing error
logging.error(f"Error parsing JSON response from LLM: {str(e)}")
logging.error(f"Raw response: {claude_answer_raw}")
logging.error(f"Raw response: {llm_answer_raw}")
# Return a default value
return [{"TIN": "UNKNOWN", "NPI": "UNKNOWN", "NAME": "PARSING_ERROR"}]
src/pipelines/shared/postprocessing/aarete_derived.py
+3 -8
View File
@@ -145,11 +145,8 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
if not string_utils.is_empty(
from_field_value
) and string_utils.is_empty(to_field_value):
# Get list of values to map
if "|" in from_field_value:
from_field_value_list = from_field_value.split("|")
elif "," in from_field_value:
from_field_value_list = from_field_value.split(",")
if isinstance(from_field_value, list):
from_field_value_list = from_field_value
else:
from_field_value_list = [from_field_value]
@@ -159,7 +156,6 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
individual_from_field_value = (
individual_from_field_value.strip()
)
if individual_from_field_value in crosswalk.mapping.keys():
# Value is a key in the mapping - map it to the target value
to_field_answer_list.append(
@@ -176,6 +172,5 @@ def get_crosswalk_fields(answer_dicts: list, constants: Constants):
individual_from_field_value
)
)
answer_dict[to_field_name] = "|".join(to_field_answer_list)
answer_dict[to_field_name] = to_field_answer_list
return answer_dicts
src/pipelines/shared/preprocessing/hybrid_smart_chunking_funcs.py
+5 -16
View File
@@ -279,14 +279,6 @@ def run_hybrid_smart_chunked_fields(
answers_dict, field_name="PROVIDER_STATE"
)
# Normalize PAYER_STATE to standardized two-letter abbreviation
for state_field in ["PAYER_STATE"]:
if state_field in answers_dict:
answers_dict[state_field] = (
string_utils.normalize_state_to_abbreviation(
answers_dict[state_field]
)
)
# Normalize PAYER_STATE
answers_dict = string_utils.normalize_state_field(
answers_dict, field_name="PAYER_STATE"
@@ -313,7 +305,7 @@ def extract_amendment_num_from_filename(answer_dict: dict, filename: str) -> dic
or answer_dict.get("CONTRACT_AMENDMENT_NUM") == "UNKNOWN"
):
prompt = prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename)
prompt, _parser = prompt_templates.EXTRACT_AMENDMENT_NUM_FROM_FILENAME(filename)
llm_answer_raw = llm_utils.invoke_claude(
prompt,
"sonnet_latest",
@@ -324,7 +316,7 @@ def extract_amendment_num_from_filename(answer_dict: dict, filename: str) -> dic
)
try:
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
llm_answer_final = _parser(llm_answer_raw)
answer_dict["FILENAME_AMENDMENT_NUM"] = llm_answer_final.get(
"amendment_number", "N/A"
)
@@ -389,13 +381,10 @@ def prompt_hsc_single_field(
try:
llm_answer_final = _parser(llm_answer_raw) # returns dict
val = llm_answer_final.get(field.field_name)
if isinstance(val, list):
llm_answer_final[field.field_name] = val
elif isinstance(val, str):
llm_answer_final[field.field_name] = [val]
field_value = llm_answer_final.get(field.field_name)
if not isinstance(field_value, list):
field_value = [field_value]
field_value = llm_answer_final[field.field_name]
if field_value[0] == "N/A":
field.field_type = "full_context"
if field.field_name == "PAYER_NAME":