jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/remaining-dynamic-issues (pull request #543)

Browse Source 
Bugfix/remaining dynamic issues

* Add Care1st-->Medicaid mapping for Centene

* Streamlined fill_na_mapping

* Update prompt for reimbursement-level dynamic fields - remove unknown

* Merge branch 'main' into bugfix/remaining-dynamic-issues

* Remove test


Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
2025-05-22 19:53:15 +00:00
committed by Alex Galarce
parent f9b88959c3
commit 7a44c8575d
7 changed files with 20 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/crosswalk/mappings/crosswalk_product_lob.json
+1 -1
View File
@@ -10,7 +10,7 @@
"Medicare Select" : "Medicare"
},
"Centene" : {
"Care1st" : "Medicaid"
},
"Aetna" : {
fieldExtraction/crosswalk/mappings/crosswalk_program.json
+1 -1
View File
@@ -31,7 +31,7 @@
"Medicare-Medicaid Coordinated Plan (MMCP)" : "MMCP",
"Medicare-Medicaid Plan (MMP)" : "MMP",
"Program of All-Inclusive Care for the Elderly (PACE)" : "PACE",
"Medicare Advantage (MA)" : "MA",
"Medicare Advantage" : "MA",
"Medicare Part C" : "MA",
"Supplemental" : "Supp",
"Medigap" : "Supp",
fieldExtraction/src/investment/aarete_derived.py
+10 -20
View File
@@ -11,24 +11,12 @@ import os
from collections import defaultdict
def fill_na_from_field(results_dict, to_field, from_field, crosswalk_path):
if not string_utils.is_empty(results_dict.get(to_field)):
return results_dict.get(to_field)
from_str = results_dict.get(from_field)
if string_utils.is_empty(from_str):
return results_dict.get(to_field)
MAPPING_DICT = CrosswalkBuilder().from_json(crosswalk_path).mapping
from_list = [s.strip() for s in from_str.split('|')]
to_list = []
for from_value in from_list:
if from_value:
to_value = MAPPING_DICT.get(from_value)
if to_value:
to_list.append(to_value)
return '|'.join(list(set(to_list))) if to_list else results_dict.get(to_field)
to_value = apply_crosswalk(from_str, MAPPING_DICT)
return to_value if to_value else results_dict.get(to_field)
def fill_na_mapping(answer_dicts):
"""
@@ -40,16 +28,14 @@ def fill_na_mapping(answer_dicts):
Returns:
pd.DataFrame: DataFrame with filled missing values
"""
for answer_dict in answer_dicts:
# Fill AARETE_DERIVED_LOB from AARETE_DERIVED_PROGRAM
answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "crosswalk/mappings/crosswalk_program_lob.json")
if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")):
answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "AARETE_DERIVED_PROGRAM", "crosswalk/mappings/crosswalk_program_lob.json")
# Fill AARETE_DERIVED_LOB from PRODUCT
answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "crosswalk/mappings/crosswalk_product_lob.json")
# Add other mappable fields below
if string_utils.is_empty(answer_dict.get("AARETE_DERIVED_LOB")):
answer_dict["AARETE_DERIVED_LOB"] = fill_na_from_field(answer_dict, "AARETE_DERIVED_LOB", "PRODUCT", "crosswalk/mappings/crosswalk_product_lob.json")
return answer_dicts
@@ -57,6 +43,7 @@ def get_crosswalk_fields(answer_dicts):
crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
just_mapped = []
for to_field in crosswalk_fields.fields:
from_field = to_field.base_field
if from_field in just_mapped:
continue
@@ -71,10 +58,13 @@ def get_crosswalk_fields(answer_dicts):
from_field_value = answer_dict.get(from_field)
if not string_utils.is_empty(from_field_value):
to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping)
if not to_field_value:
to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping_reverse)
answer_dict[to_field.field_name] = to_field_value
just_mapped.append(to_field.field_name)
return answer_dicts
def get_aarete_derived(aarete_derived_fields: FieldSet,
fieldExtraction/src/investment/dynamic_funcs.py
+4 -2
View File
@@ -21,6 +21,8 @@ def prompt_dynamic(text, field_prompts, filename):
"""
prompt = investment_prompts.EXHIBIT_LEVEL(text, field_prompts)
llm_answer_raw = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename) # Returns dictionary of lists
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
return llm_answer_final
@@ -69,12 +71,13 @@ def get_dynamic_answers(exhibit_chunk, exhibit_header, filename, dynamic_fields
exhibit_header_results = prompt_dynamic(text=exhibit_header,
field_prompts=dynamic_fields.print_prompt_dict(),
filename=filename)
# Process Exhibit Header Results - update dynamic fields and add to reimbursement level fields
for field_name, answer in exhibit_header_results.items():
if not string_utils.is_empty(answer):
field = dynamic_fields.get_field(field_name)
field.update_valid_values(answer)
field.prompt = field.prompt + ". Ensure ALL values that apply to the specific reimbursement term are included."
reimbursement_level_fields.add_field(field)
dynamic_fields.remove_field(field_name)
@@ -83,7 +86,6 @@ def get_dynamic_answers(exhibit_chunk, exhibit_header, filename, dynamic_fields
exhibit_chunk_results = prompt_dynamic(text=exhibit_chunk,
field_prompts=dynamic_fields.print_prompt_dict(),
filename=filename)
# Process Exhibit Chunk Results - update dynamic fields and add to reimbursement level fields
for field_name, answer in exhibit_chunk_results.items():
if not string_utils.is_empty(answer):
fieldExtraction/src/investment/one_to_n_funcs.py
+1 -2
View File
@@ -104,11 +104,10 @@ def identify_reimbursement_exhibits(exhibit_headers: dict, filename: str) -> lis
def get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename):
field_prompts = reimbursement_level_fields.get_prompt_dict()
prompt = investment_prompts.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts)
# print(prompt) # Debugging line
llm_answer_raw = llm_utils.invoke_claude(
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=25000
)
# print(llm_answer_raw) # Debugging line
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
return llm_answer_final
fieldExtraction/src/prompts/investment_prompts.json
+2 -2
View File
@@ -243,7 +243,7 @@
"field_name": "REIMB_PROV_NAME",
"relationship": "one_to_n",
"field_type": "dynamic",
"prompt": "Identify any Provider Names that this reimbursement term applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'UNKNOWN'."
"prompt": "Identify any Provider Names that this reimbursement term applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'N/A'."
},
{
"field_name": "AARETE_DERIVED_EFFECTIVE_DT",
@@ -326,7 +326,7 @@
"field_name": "AARETE_DERIVED_PROV_TYPE",
"relationship": "one_to_n",
"field_type": "TBD",
"prompt": "What Provider Type does the Exhibit apply to? Choose ONLY from the following: {valid_values}. If a provider type is mentioned but doesn't match any values in the list, write 'UNKNOWN'. If no provider type is mentioned at all, write 'N/A'.",
"prompt": "What Provider Type does the Exhibit apply to? Choose ONLY from the following: {valid_values}. If no provider type form the list is mentioned, write 'N/A'.",
"valid_values": "VALID_PROV_TYPE"
},
{
fieldExtraction/src/prompts/investment_prompts.py
+1 -2
View File
@@ -349,8 +349,7 @@ Here are the attributes to be included in the dictionary, and instructions on ho
For each field, return ALL correct answers found. There may be more than one correct answer. If there are multiple, return them in a pipe-separated list ("|").
If a list of valid values is provided, ONLY answer with the EXACT value from that list.
If there are no correct answers for a given field because the requested information doesn't apply to this context, write 'N/A' as the answer for that field.
If there are no correct answers for a given field because the requested information should exist but cannot be clearly identified, write 'UNKNOWN' as the answer for that field.
If there are no correct answers for a given field, write 'N/A' as the answer for that field.
Here is the text to analyze: