Merged in bugfix/dynamic-primary-issues (pull request #555)
Bugfix/dynamic primary issues * Merge branch 'main' into bugfix/dynamic-primary-issues * Merged main into bugfix/dynamic-primary-issues * Crosswalk updates * Merge branch 'main' into bugfix/dynamic-primary-issues * dont include section information in Exhibit Header * Add Molina Health Benefit Exchange to Product-LOB mapping * PPO-POS separate mapping * Prevent inference of LOB from Program or Product * Add SoonerSelect * Update mapping rules to allow for direct values * update prompts * Prep for run * Merged main into bugfix/dynamic-primary-issues * Update unit tests * Update Moda Product mappings * Remove Care1st as Product for Centene * testing * Mapping update * Mapping updates * prep for merge * Merged main into bugfix/dynamic-primary-issues * Merged main into bugfix/dynamic-primary-issues * docstrings and formatting * Get group provider for > 1 * Merged main into bugfix/dynamic-primary-issues Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
@@ -5,6 +5,7 @@ import ast
|
||||
|
||||
from src.config import STATE
|
||||
from src.config import CLIENT
|
||||
import src.utils.string_utils as string_utils
|
||||
|
||||
class CrosswalkBuilder:
|
||||
def __init__(self):
|
||||
@@ -188,9 +189,15 @@ def apply_crosswalk(
|
||||
elif "," in val:
|
||||
values = [v.strip() for v in val.split(",")] # Split by comma if present
|
||||
elif "|" in val:
|
||||
values = [v.strip() for v in val.split("|")]
|
||||
values = [v.strip() for v in val.split("|")] # Split by pipe if present
|
||||
else:
|
||||
values = [val]
|
||||
|
||||
mapped_values = {mapping.get(v.strip(), default) for v in values}
|
||||
return "|".join(mapped_values) # Join back into a single string
|
||||
mapped_values.update({v for v in values if v in mapping.values()})
|
||||
mapped_values = [v for v in mapped_values if not string_utils.is_empty(v)]
|
||||
|
||||
if len(mapped_values) == 1:
|
||||
return list(mapped_values)[0]
|
||||
else:
|
||||
return "|".join(mapped_values) # Join back into a single string
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
"Preferred Provider Organization (PPO)": "PPO",
|
||||
"Exclusive Provider Organization (EPO)": "EPO",
|
||||
"Point of Service (POS)": "POS",
|
||||
"PPO-POS" : "PPO | POS",
|
||||
"High Deductible Health Plan (HDHP)": "HDHP",
|
||||
"Fee for Service (FFS)" : "FFS",
|
||||
"Indemnity" : "FFS"
|
||||
|
||||
@@ -10,21 +10,25 @@
|
||||
"Medicare Select" : "Medicare"
|
||||
},
|
||||
"Centene" : {
|
||||
"Care1st" : "Medicaid"
|
||||
|
||||
},
|
||||
"Aetna" : {
|
||||
|
||||
},
|
||||
"CHC" : {
|
||||
|
||||
},
|
||||
"Fidelis" : {
|
||||
"Medicaid Managed Care" : "Medicaid"
|
||||
},
|
||||
"Parkland" : {
|
||||
"HEALTH first" : "Medicaid",
|
||||
"KIDS first" : "Medicaid"
|
||||
"HEALTHfirst" : "Medicaid",
|
||||
"KIDSfirst" : "Medicaid"
|
||||
},
|
||||
"Molina" :{
|
||||
"Molina Medicare Options" : "Medicare",
|
||||
"Molina Medicare Options Plus" : "Medicare | Duals"
|
||||
"Molina Medicare Options Plus" : "Medicare | Duals",
|
||||
"Molina Health Benefit Exchange" : "Marketplace"
|
||||
},
|
||||
"Humana" : {
|
||||
|
||||
@@ -34,6 +38,9 @@
|
||||
"Connexus" : "Commercial",
|
||||
"Beacon" : "Commercial",
|
||||
"Community Care Network" : "Commercial",
|
||||
"OHSU" : "Commercial",
|
||||
"PEBB" : "Commercial",
|
||||
"OEBB" : "Commercial",
|
||||
"Affinity" : "Marketplace",
|
||||
"Moda Select" : "Marketplace"
|
||||
},
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
"Community First Choice (CFC)" : "CFC",
|
||||
"Foster Care (FC)" : "FC",
|
||||
"Home and Community Based Services Waiver (HCBS)" : "HCBS",
|
||||
"Home Health Services (HHS)" : "HHS",
|
||||
"Hospice Benefits (HOSPICE)" : "HOSPICE",
|
||||
"Intellectual and Developmental Disability (IDD)" : "IDD",
|
||||
"Institution for Medical Disease (IMD)" : "IMD",
|
||||
@@ -128,7 +127,7 @@
|
||||
},
|
||||
"LA" : {
|
||||
"Healthy Louisiana" : "HEALTHYLA",
|
||||
"Lousiana State Insurance Exchange" : "LASIE",
|
||||
"Louisiana State Insurance Exchange" : "LASIE",
|
||||
"LaCHIP" : "CHIP"
|
||||
},
|
||||
"ME" : {
|
||||
@@ -200,7 +199,8 @@
|
||||
"Healthy Start" : "CHIP"
|
||||
},
|
||||
"OK" : {
|
||||
"SoonerCare" : "SOONER"
|
||||
"SoonerCare" : "SOONER",
|
||||
"SoonerSelect" : "SOONER"
|
||||
},
|
||||
"OR" : {
|
||||
"Oregon Health Plan" : "ORHP",
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
"PU21" : "Medicaid",
|
||||
"SNAP" : "Medicaid",
|
||||
"TANF" : "Medicaid",
|
||||
"LTSS" : "Medicaid",
|
||||
"SNP" : "Duals",
|
||||
"CSNP" : "Duals",
|
||||
"DSNP" : "Duals",
|
||||
@@ -87,7 +88,7 @@
|
||||
"YHID" : "Marketplace"
|
||||
},
|
||||
"IL" : {
|
||||
"YHID" : "Medicaid",
|
||||
"HCIL" : "Marketplace",
|
||||
"GCIL" : "Marketplace"
|
||||
},
|
||||
"IN" : {
|
||||
@@ -150,7 +151,9 @@
|
||||
"WELLNM" : "Marketplace"
|
||||
},
|
||||
"NY" : {
|
||||
"NYSOH" : "Marketplace"
|
||||
"NYSOH" : "Marketplace",
|
||||
"HARP" : "Medicaid",
|
||||
"CHPLUS" : "Medicaid"
|
||||
},
|
||||
"NC" : {
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ def fill_na_mapping(answer_dicts):
|
||||
|
||||
def get_crosswalk_fields(answer_dicts):
|
||||
crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
|
||||
|
||||
just_mapped = []
|
||||
for to_field in crosswalk_fields.fields:
|
||||
|
||||
@@ -112,8 +113,7 @@ def get_aarete_derived(aarete_derived_fields: FieldSet,
|
||||
|
||||
# Get unique values from the base field
|
||||
unique_base_values = result_df[base_field].unique()
|
||||
print(f"Found {len(unique_base_values)} unique values out of {len(result_df)} total rows for {base_field}")
|
||||
|
||||
|
||||
# Process each unique value only once
|
||||
for unique_value in unique_base_values:
|
||||
# Skip if we've already processed this value
|
||||
|
||||
@@ -21,12 +21,9 @@ def prompt_dynamic(text, field_prompts, filename):
|
||||
"""
|
||||
prompt = investment_prompts.EXHIBIT_LEVEL(text, field_prompts)
|
||||
llm_answer_raw = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename) # Returns dictionary of lists
|
||||
|
||||
|
||||
llm_answer_final = string_utils.universal_json_load(llm_answer_raw)
|
||||
return llm_answer_final
|
||||
|
||||
|
||||
def get_dynamic_one_to_one_fields(answer_dicts):
|
||||
"""
|
||||
Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts
|
||||
@@ -45,8 +42,62 @@ def get_dynamic_one_to_one_fields(answer_dicts):
|
||||
continue
|
||||
return one_to_one_fields
|
||||
|
||||
def prompt_dynamic_primary(exhibit_text, field, filename):
|
||||
# Prompt Exhibit Header
|
||||
prompt = investment_prompts.DYNAMIC_PRIMARY(exhibit_text, field.field_name, field.get_prompt())
|
||||
claude_answer_raw = llm_utils.invoke_claude(prompt, config.MODEL_ID_CLAUDE35_SONNET, filename)
|
||||
claude_answer_final = string_utils.extract_text_from_delimiters(claude_answer_raw, string_utils.Delimiter.PIPE)
|
||||
if "," in claude_answer_final:
|
||||
claude_answer_final = "|".join([item.strip() for item in claude_answer_final.split(",")])
|
||||
return claude_answer_final
|
||||
|
||||
def get_dynamic_answers(exhibit_chunk, exhibit_header, filename, dynamic_fields : FieldSet):
|
||||
def get_dynamic_primary(exhibit_chunk : str, exhibit_header: str, filename: str, dynamic_fields : FieldSet):
|
||||
"""
|
||||
Processes dynamic primary fields one-by-one from exhibit header and chunk text.
|
||||
|
||||
Args:
|
||||
exhibit_chunk (str): The main text chunk of the exhibit
|
||||
exhibit_header (str): The header text of the exhibit
|
||||
filename (str): Name of the file being processed
|
||||
dynamic_fields (FieldSet): FieldSet containing the dynamic fields to process
|
||||
|
||||
Returns:
|
||||
tuple: (dict, FieldSet) containing:
|
||||
- exhibit_level_answer_dict: Dictionary of exhibit-level answers
|
||||
- reimbursement_level_fields: FieldSet of fields assigned to reimbursement level
|
||||
"""
|
||||
if not dynamic_fields.contains_fields():
|
||||
return {}, FieldSet()
|
||||
|
||||
reimbursement_level_fields = FieldSet()
|
||||
exhibit_level_answer_dict = {}
|
||||
for field in dynamic_fields.fields:
|
||||
exhibit_header_answer = prompt_dynamic_primary(exhibit_header, field, filename)
|
||||
# If there is exactly ONE Non-N/A answer in the header
|
||||
if not string_utils.is_empty(exhibit_header_answer) and "|" not in exhibit_header_answer:
|
||||
exhibit_level_answer_dict[field.field_name] = exhibit_header_answer
|
||||
# If there are multiple Non-N/A answers in the header
|
||||
elif not string_utils.is_empty(exhibit_header_answer) and "|" in exhibit_header_answer:
|
||||
field.update_valid_values(exhibit_header_answer)
|
||||
field.prompt = field.prompt + ". Ensure ALL values that apply to the specific reimbursement term are included."
|
||||
reimbursement_level_fields.add_field(field)
|
||||
dynamic_fields.remove_field(field.field_name)
|
||||
# If there are ZERO Non-N/A answers in the header
|
||||
else:
|
||||
exhibit_text_answer = prompt_dynamic_primary(exhibit_chunk, field, filename)
|
||||
# If there is at least ONE Non-N/A answers in the Exhibit
|
||||
if not string_utils.is_empty(exhibit_text_answer):
|
||||
field.update_valid_values(exhibit_text_answer + " | N/A")
|
||||
reimbursement_level_fields.add_field(field)
|
||||
dynamic_fields.remove_field(field.field_name)
|
||||
# If the field is not found, add to Exhibit-Level answers (the answer will be N/A or similar)
|
||||
else:
|
||||
exhibit_level_answer_dict[field.field_name] = exhibit_text_answer
|
||||
|
||||
return exhibit_level_answer_dict, reimbursement_level_fields
|
||||
|
||||
|
||||
def get_dynamic_answers(exhibit_chunk : str, exhibit_header: str, filename: str, dynamic_fields : FieldSet):
|
||||
"""
|
||||
Processes dynamic fields from exhibit header and chunk text.
|
||||
|
||||
|
||||
@@ -78,8 +78,6 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, d
|
||||
regex_answers_dict = one_to_one_funcs.run_provider_info_fields(contract_text, text_dict, filename)
|
||||
one_to_one_results = {}
|
||||
one_to_one_results['PROV_INFO_JSON'] = json.dumps(regex_answers_dict)
|
||||
## Easier formatting for customers:
|
||||
# Create newline-delimited version
|
||||
one_to_one_results['PROV_INFO_JSON_FORMATTED'] = '\n'.join([json.dumps(provider) for provider in regex_answers_dict])
|
||||
|
||||
one_to_one_results = merge_provider_info(one_to_one_results, regex_answers_dict)
|
||||
@@ -147,13 +145,21 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
|
||||
exhibit_level_answers['EXHIBIT_TITLE'] = exhibit_header
|
||||
|
||||
################## GET DYNAMIC-PRIMARY ANSWERS ##################
|
||||
dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = dynamic_funcs.get_dynamic_answers(exhibit_text,
|
||||
dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = dynamic_funcs.get_dynamic_primary(exhibit_text,
|
||||
exhibit_header,
|
||||
filename,
|
||||
FieldSet(relationship="one_to_n", field_type="dynamic", file_path=config.FIELD_JSON_PATH))
|
||||
FieldSet(relationship="one_to_n", field_type="dynamic_primary", file_path=config.FIELD_JSON_PATH))
|
||||
exhibit_level_answers.update(dynamic_to_exhibit_level_answers)
|
||||
reimbursement_level_fields.combine(dynamic_to_reimbursement_level_fields, inplace=True)
|
||||
|
||||
|
||||
################## GET DYNAMIC-REIMB-INFO ANSWERS ##################
|
||||
reimb_info_to_exhibit_level_answers, reimb_info_to_reimbursement_level_fields = dynamic_funcs.get_dynamic_answers(exhibit_text,
|
||||
exhibit_header,
|
||||
filename,
|
||||
FieldSet(relationship="one_to_n", field_type="dynamic_reimb_info", file_path=config.FIELD_JSON_PATH))
|
||||
exhibit_level_answers.update(reimb_info_to_exhibit_level_answers)
|
||||
reimbursement_level_fields.combine(reimb_info_to_reimbursement_level_fields, inplace=True)
|
||||
|
||||
################## GET DYNAMIC-CODE ANSWERS ##################
|
||||
code_to_exhibit_level_answers, code_to_reimbursement_level_fields = dynamic_funcs.get_dynamic_answers(exhibit_text,
|
||||
exhibit_header,
|
||||
@@ -168,7 +174,7 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, all_datase
|
||||
################# COMBINE ANSWERS ##################
|
||||
full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers)
|
||||
one_to_n_results += full_answer_dict
|
||||
|
||||
|
||||
################## Crosswalk Fields ##################
|
||||
one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results)
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ def get_exhibit_level_answers(exhibit_chunk, filename):
|
||||
|
||||
|
||||
def get_reimbursement_primary(reimbursement_level_fields, exhibit_text, filename):
|
||||
field_prompts = reimbursement_level_fields.get_prompt_dict()
|
||||
field_prompts = reimbursement_level_fields.print_prompt_dict()
|
||||
prompt = investment_prompts.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts)
|
||||
logging.debug(f"""Running reimbursement primary prompt for {filename} with prompt:
|
||||
{prompt}""")
|
||||
@@ -253,19 +253,16 @@ def methodology_breakout_single_row(
|
||||
prompt = methodology_prompt_template(
|
||||
service, contract_reimbursement_method, methodology_breakout_questions
|
||||
)
|
||||
# print(prompt) # Debugging line
|
||||
llm_response = llm_utils.invoke_claude(
|
||||
prompt,
|
||||
config.MODEL_ID_CLAUDE35_SONNET,
|
||||
filename,
|
||||
)
|
||||
# print(llm_response) # Debugging line
|
||||
try:
|
||||
methodology_breakout_answers = string_utils.universal_json_load(
|
||||
llm_response
|
||||
)
|
||||
except ValueError as e:
|
||||
# print(f"universal_string_load failed for input {llm_response} with error {e}\nassigning methodology_breakout_answer to []")
|
||||
methodology_breakout_answers = []
|
||||
|
||||
if isinstance(
|
||||
@@ -337,7 +334,6 @@ def get_methodology_breakout(
|
||||
for answer_dict in reimbursement_primary_answers:
|
||||
if isinstance(answer_dict, list): # if a nested list is found, flatten it
|
||||
# This shouldn't happen but is in for defensive programming purposes
|
||||
print(f"Found a nested list instead of a dict, flattening {answer_dict}")
|
||||
for nested_dict in answer_dict:
|
||||
_process_carveout_or_breakout(nested_dict, methodology_breakout_answers, filename)
|
||||
else: # normal processing for dict items
|
||||
|
||||
@@ -65,24 +65,8 @@ def run_provider_info_fields(contract_text: str,
|
||||
# Deduplicate providers based on TIN and NPI
|
||||
deduplicated_provider_info = deduplicate_providers(cleaned_provider_info)
|
||||
|
||||
# if payer_name:
|
||||
# # Filter out providers that are too similar to the payer name
|
||||
# deduplicated_provider_info = filter_payer_from_providers(deduplicated_provider_info, payer_name)
|
||||
# print(f"Filtered provider info for {filename}: {deduplicated_provider_info}")
|
||||
|
||||
if len(deduplicated_provider_info) > 1:
|
||||
# print(f"Identifying group provider for {filename}...")
|
||||
# This function modifies the provider list in place and marks one as IS_GROUP=True
|
||||
group_provider = identify_group_provider(text_dict, deduplicated_provider_info, filename)
|
||||
|
||||
# The returned group_provider is the one that was marked as IS_GROUP=True - but
|
||||
# we don't need to use it, as the deduplicated_provider_info list is already modified
|
||||
# and contains the IS_GROUP flag.
|
||||
# If you want to print the group provider, uncomment the following debugging lines:
|
||||
# if group_provider:
|
||||
# print(f"Group provider identified: {group_provider.get('NAME')} with TIN {group_provider.get('TIN')} and NPI {group_provider.get('NPI')}")
|
||||
# else:
|
||||
# print("No group provider identified.")
|
||||
elif len(deduplicated_provider_info) == 1:
|
||||
# If only one provider is found, mark it as the group provider
|
||||
deduplicated_provider_info[0]["IS_GROUP"] = True
|
||||
@@ -215,8 +199,6 @@ def run_smart_chunked_fields(one_to_one_fields: FieldSet,
|
||||
field_group_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, 8192
|
||||
)
|
||||
# print(f"\nfields: {fields}\nfield_group_answer_raw: {field_group_answer_raw}") # this is for debugging. Take this out before deployment.
|
||||
|
||||
if len(fields) == 1:
|
||||
field_group_answer_parsed = string_utils.extract_text_from_delimiters(field_group_answer_raw, Delimiter.PIPE)
|
||||
field_group_answer_dict = {fields[0].field_name: field_group_answer_parsed}
|
||||
@@ -225,8 +207,6 @@ def run_smart_chunked_fields(one_to_one_fields: FieldSet,
|
||||
field_group_answer_raw, [field.field_name for field in fields]
|
||||
)
|
||||
# Validate field names coming out of Claude:
|
||||
# check against fields in group, make sure they're all there
|
||||
# make sure there aren't any extras
|
||||
expected_fields = {field.field_name for field in fields}
|
||||
received_fields = set(field_group_answer_dict.keys())
|
||||
if expected_fields != received_fields:
|
||||
@@ -266,7 +246,6 @@ def check_and_update_effective_date(answers_dict, text_dict, filename):
|
||||
Returns:
|
||||
dict: Updated answers dictionary with the effective date.
|
||||
"""
|
||||
# print(f"Effective date for {filename} from smart chunking: {answers_dict.get('AARETE_DERIVED_EFFECTIVE_DT')}")
|
||||
|
||||
# Postprocess the effective date
|
||||
answers_dict["AARETE_DERIVED_EFFECTIVE_DT"] = generic_postprocessing_funcs.date_postprocessing(
|
||||
|
||||
@@ -58,7 +58,6 @@ def one_to_n_exhibit_chunking(text_dict, filename) -> tuple[dict, dict]:
|
||||
tuple: A tuple containing:
|
||||
- dict: A dictionary where keys are exhibit page numbers and values are the corresponding exhibit text chunks.
|
||||
- dict: A dictionary where keys are exhibit page numbers and values are the exhibit headers.
|
||||
|
||||
"""
|
||||
exhibit_pages, all_exhibit_headers = preprocessing_funcs.get_exhibit_pages(text_dict, filename)
|
||||
exhibit_pages, all_exhibit_headers = preprocessing_funcs.link_exhibit_pages(all_exhibit_headers, filename)
|
||||
|
||||
@@ -31,7 +31,6 @@ def get_image_array_based_answer(
|
||||
"""
|
||||
# Convert specified PDF pages to a base64-encoded image
|
||||
base64_images = process_pdf_pages_to_base64_images(pdf_path, filename, effective_date_page_numbers)
|
||||
# print("No of base64 images:", len(base64_images)) # Log a snippet of the base64 data for debugging
|
||||
|
||||
# Prepare the prompt for the LLM
|
||||
extended_prompt = prompts.effective_date_fix_prompt()
|
||||
|
||||
@@ -243,7 +243,7 @@ def get_exhibit_pages(text_dict: dict[str, str], filename: str) -> tuple[list[st
|
||||
if "." not in page_num or ".0" in page_num:
|
||||
prompt = investment_prompts.EXHIBIT_HEADER(page_content[0:400])
|
||||
claude_answer_raw = llm_utils.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE3_HAIKU, filename, max_tokens=150
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, filename, max_tokens=300
|
||||
)
|
||||
claude_answer_extracted = string_utils.extract_text_from_delimiters(
|
||||
claude_answer_raw, Delimiter.PIPE
|
||||
|
||||
@@ -64,14 +64,14 @@
|
||||
{
|
||||
"field_name": "REIMB_EFFECTIVE_DT",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic",
|
||||
"field_type": "dynamic_reimb_info",
|
||||
"prompt": "Identify the effective date for reimbursement rates:\n- Look for dates associated with:\n • 'Effective' or 'effective date'\n • 'In effect' or 'in effect on'\n • Date ranges with 'from/through' or 'start/end'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)",
|
||||
"format": "date"
|
||||
},
|
||||
{
|
||||
"field_name": "REIMB_TERMINATION_DT",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic",
|
||||
"field_type": "dynamic_reimb_info",
|
||||
"prompt": "Identify the termination date for reimbursement rates:\n- Look for dates associated with:\n • 'Termination' or 'termination date'\n • 'Expires' or 'expiration date'\n • 'Through' or 'until'\n- Dates may appear as:\n • Full dates (12/1/17, December 1, 2017)\n • Month and year (December 2017)\n • Year only (2017)",
|
||||
"format": "date"
|
||||
},
|
||||
@@ -110,8 +110,8 @@
|
||||
{
|
||||
"field_name" : "LOB",
|
||||
"relationship" : "one_to_n",
|
||||
"field_type" : "dynamic",
|
||||
"prompt" : "List any Lines of Business explicitly stated in the text. Choose ONLY from the following: {valid_values}. If there are multiple answers, return them in a pipe-separated list ('|')",
|
||||
"field_type" : "dynamic_primary",
|
||||
"prompt" : "Extract ONLY LOBs that are **explicitly mentioned** in the text. Do NOT extract based on Programs, Products, context, implications, or any other form of inference. Choose LOBs ONLY from the following values: {valid_values}. ",
|
||||
"valid_values" : "VALID_LOBS"
|
||||
},
|
||||
{
|
||||
@@ -125,8 +125,8 @@
|
||||
{
|
||||
"field_name" : "NETWORK",
|
||||
"relationship" : "one_to_n",
|
||||
"field_type" : "dynamic",
|
||||
"prompt" : "List any Networks explicitly stated in the text. Choose ONLY from the following: {valid_values}. If there are multiple answers, return them in a pipe-separated list ('|')",
|
||||
"field_type" : "dynamic_primary",
|
||||
"prompt" : "List any Networks explicitly stated in the text. Choose ONLY from the following: {valid_values}.",
|
||||
"valid_values" : "VALID_NETWORKS"
|
||||
},
|
||||
{
|
||||
@@ -140,8 +140,8 @@
|
||||
{
|
||||
"field_name" : "PRODUCT",
|
||||
"relationship" : "one_to_n",
|
||||
"field_type" : "dynamic",
|
||||
"prompt" : "List any Products explicitly stated in the text. Look for the brand name of a health plan or product. Do not include the name of the entire healthplan (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list. If there are multiple answers, return them in a pipe-separated list ('|')",
|
||||
"field_type" : "dynamic_primary",
|
||||
"prompt" : "List any Products explicitly stated in the text. Look for the brand name of a health plan or product. Do not include the name of the entire healthplan (such as 'Molina', 'Centene' etc) - rather, look for the Product that these healthplans offer. Valid products include, but are not limited to, the following: {valid_values}. If any of the values in this list are mentioned, return them exactly as written in the list. NEVER write an answer that is not explicitly stated in the text.",
|
||||
"valid_values" : "VALID_PRODUCTS"
|
||||
},
|
||||
{
|
||||
@@ -154,8 +154,8 @@
|
||||
{
|
||||
"field_name" : "PROGRAM",
|
||||
"relationship" : "one_to_n",
|
||||
"field_type" : "dynamic",
|
||||
"prompt" : "List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. If there are multiple answers, return them in a pipe-separated list ('|')",
|
||||
"field_type" : "dynamic_primary",
|
||||
"prompt" : "List any Programs explicitly stated in the text. Choose ONLY from the following: {valid_values}. Note that 'Medicare' does NOT mean 'Medicare Advantage'. If only 'Medicare' is seen, do not write 'Medicare Advantage'. Choose the answer from the list that most closely matches what is found in the text. NEVER write an answer that is not explicitly stated in the text.",
|
||||
"valid_values" : "VALID_PROGRAMS"
|
||||
},
|
||||
{
|
||||
@@ -243,20 +243,20 @@
|
||||
{
|
||||
"field_name": "REIMB_PROV_TIN",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "TBD",
|
||||
"prompt": "What TIN (Tax ID Number) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable TINs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.). Choose only from the following: {valid_values}."
|
||||
"field_type": "dynamic_prov_info",
|
||||
"prompt": "What TIN (Tax ID Number) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable TINs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.)."
|
||||
},
|
||||
{
|
||||
"field_name": "REIMB_PROV_NPI",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "TBD",
|
||||
"prompt": "What NPI (National Provider Identifier) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable NPIs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.). Choose only from the following: {valid_values}."
|
||||
"field_type": "dynamic_prov_info",
|
||||
"prompt": "What NPI (National Provider Identifier) does the reimbursement term apply to? When the term applies to multiple providers simultaneously, list ALL applicable NPIs separated by pipes (|). Look for language patterns indicating multiple providers (phrases like 'following providers', 'these facilities', 'listed providers', etc.)."
|
||||
},
|
||||
{
|
||||
"field_name": "REIMB_PROV_NAME",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic",
|
||||
"prompt": "Identify any Provider Names that this reimbursement term applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'N/A'."
|
||||
"field_type": "dynamic_prov_info",
|
||||
"prompt": "Identify any Provider Names that this reimbursement applies to. When the term applies to multiple providers simultaneously, list ALL applicable names separated by pipes (|). Look for language patterns indicating multiple providers. Extract only the official names of healthcare providers (doctors, hospitals, clinics, etc.) - not payers or insurance plans. If unable to determine applicable provider names, respond with 'N/A'."
|
||||
},
|
||||
{
|
||||
"field_name": "AARETE_DERIVED_EFFECTIVE_DT",
|
||||
@@ -409,7 +409,7 @@
|
||||
{
|
||||
"field_name": "PATIENT_AGE_RANGE",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic",
|
||||
"field_type": "dynamic_reimb_info",
|
||||
"prompt": "List any patient age ranges described in the text that relate to a reimbursement term. Write the age ranges mentioned in the format 'X-Y' (e.g., '0-18', '18-65'). If the age range is not specified but the text indicates a specific age group (e.g., 'pediatric', 'adult'), return that exact text instead. If an age is listed as simply 'Over X', write 'X-None', to indicate that there is no upper bound to the age range. If no age ranges are mentioned, return N/A."
|
||||
},
|
||||
{
|
||||
|
||||
@@ -259,22 +259,20 @@ class FieldSet:
|
||||
def EXHIBIT_HEADER(context):
|
||||
return f"""Analyze the following contract excerpt and extract the full header found.
|
||||
|
||||
If an Exhibit Header is present, you must capture the complete hierarchy of document identifiers present in the text. Include all of the following when present:
|
||||
Include ALL text that appears to be part of the exhibit/attachment header. Include all of the following when present:
|
||||
* Full Attachment names and numbers (e.g., "Attachment C: Commercial-Exchange")
|
||||
* Complete Exhibit numbers/letters with full titles (e.g., "Exhibit 1 - Medicare")
|
||||
* Any other relevant subtitles and section identifiers, including but not limited to 'Article', 'Amendment', 'Schedule', 'Addendum', 'Appendix', or similar.
|
||||
* Any other relevant subtitles and identifiers, including but not limited to 'Article', 'Amendment', 'Schedule', 'Addendum', 'Appendix', or similar.
|
||||
* Any words in the header that may specify what type of information is contained, such as "Compensation Schedule" or "Definitions".
|
||||
* Provider/Entity names when listed with exhibit information
|
||||
|
||||
If there is no Exhibit Header present, return |N/A|. Do not fill in values from the examples above, these are only examples.
|
||||
Here are some very important rules:
|
||||
* Do NOT abbreviate or summarize any part of the exhibit names
|
||||
* Do NOT include page numbers
|
||||
* Do NOT include any Section language from your extraction of the header, even if it appears to be a part of the header. Sections are not considered part of the header.
|
||||
* Do NOT include docusign IDs and other metadata. These do not constitute a header.
|
||||
|
||||
- Rules for inclusion:
|
||||
* Do NOT abbreviate or summarize any part of the exhibit names
|
||||
* Do NOT include page numbers
|
||||
* Do NOT cherry-pick only certain parts - capture the complete exhibit hierarchy
|
||||
* Include ALL text that appears to be part of the exhibit/attachment header
|
||||
* Keep exact capitalization and formatting as shown in document
|
||||
* Note that docusign IDs and other metadata do not constitute a header and should be excluded
|
||||
If there is no Exhibit Header present, return |N/A|. Do not fill in values from the examples above, these are only examples.
|
||||
|
||||
Here is the text to analyze:
|
||||
{context.replace('"', "'")}
|
||||
@@ -354,6 +352,27 @@ Explain each answer before putting the final answer in the JSON dictionary forma
|
||||
"""
|
||||
|
||||
|
||||
def DYNAMIC_PRIMARY(context, field_name, field_prompt):
|
||||
return f"""You are an information extractor. Your job is to extract attributes **strictly and literally** from the provided text, paying close attention to detail.
|
||||
|
||||
[ATTRIBUTE TO EXTRACT]
|
||||
Here is the attribute, and instructions on how to correctly answer:
|
||||
{field_name} : {field_prompt}
|
||||
|
||||
[GENERAL INSTRUCTIONS]
|
||||
- Return all valid, explicitly stated values. If multiple values are found, list them separated by commas.
|
||||
- If none of the valid values appear explicitly in the text, return N/A.
|
||||
- Never guess. Never assume. Only return values that are a direct, word-for-word match from the valid list.
|
||||
- If a value appears in the text but is not in the list of valid values, ignore it.
|
||||
|
||||
Here is the text to analyze:
|
||||
|
||||
{context.replace('"', "'")}
|
||||
|
||||
Briefly explain your answer before putting the final answer in |pipes|.
|
||||
"""
|
||||
|
||||
|
||||
def REIMBURSEMENT_LEVEL_PRIMARY(context, fields):
|
||||
return f"""[OBJECTIVE]
|
||||
Extract reimbursement attributes from this payer-provider contract section.
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
# import pytest
|
||||
# from unittest.mock import patch, MagicMock
|
||||
# from src.investment.aarete_derived import get_aarete_derived
|
||||
|
||||
# @pytest.fixture
|
||||
# def mock_config():
|
||||
# with patch('src.investment.aarete_derived.config') as mock_config:
|
||||
# mock_config.FIELD_JSON_PATH = 'mock_path'
|
||||
# mock_config.MODEL_ID_CLAUDE35_SONNET = 'mock_model_id'
|
||||
# yield mock_config
|
||||
|
||||
# @pytest.fixture
|
||||
# def mock_field_set():
|
||||
# mock_field = MagicMock()
|
||||
# mock_field.field_name = 'derived_field'
|
||||
# mock_field.base_field = 'base_field'
|
||||
# mock_field.valid_values = 'valid_values_list'
|
||||
|
||||
# mock_field_set = MagicMock()
|
||||
# mock_field_set.fields = [mock_field]
|
||||
|
||||
# with patch('src.investment.aarete_derived.FieldSet', return_value=mock_field_set):
|
||||
# yield mock_field_set
|
||||
|
||||
# @pytest.fixture
|
||||
# def mock_investment_values():
|
||||
# with patch('src.investment.aarete_derived.investment_values') as mock_investment_values:
|
||||
# mock_investment_values.valid_values_list = ['valid_value1', 'valid_value2']
|
||||
# yield mock_investment_values
|
||||
|
||||
# @pytest.fixture
|
||||
# def mock_llm_utils():
|
||||
# with patch('src.investment.aarete_derived.llm_utils') as mock_llm_utils:
|
||||
# mock_llm_utils.invoke_claude.return_value = '|extracted_value|'
|
||||
# yield mock_llm_utils
|
||||
|
||||
# @pytest.fixture
|
||||
# def mock_string_utils():
|
||||
# with patch('src.investment.aarete_derived.string_utils') as mock_string_utils:
|
||||
# mock_string_utils.extract_text_from_delimiters.return_value = 'extracted_value'
|
||||
# yield mock_string_utils
|
||||
|
||||
# def test_get_aarete_derived(mock_config, mock_field_set, mock_investment_values, mock_llm_utils, mock_string_utils):
|
||||
# contract_answers = {'base_field': 'base_value'}
|
||||
# filename = 'test_file.txt'
|
||||
|
||||
# result = get_aarete_derived(contract_answers, filename)
|
||||
|
||||
# assert result['derived_field'] == 'extracted_value'
|
||||
# mock_llm_utils.invoke_claude.assert_called_once()
|
||||
# mock_string_utils.extract_text_from_delimiters.assert_called_once_with('|extracted_value|', mock_field_set.fields[0].Delimiter.PIPE)
|
||||
@@ -1,113 +0,0 @@
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from src.client.smart_chunking_funcs import DATE_PATTERN
|
||||
|
||||
legal_traditional_dates = [
|
||||
"This 12th day of December, 2024",
|
||||
"The 23RD DAY of JANUARY, 2026",
|
||||
"The 0th dAy of JaNuArY, 1000",
|
||||
"12th of December, Two Thousand and Twenty-Four",
|
||||
"Thirteenth of DeCeMbEr, Two Thousand and Sixteen",
|
||||
"Thirteenth of DeCeMbEr, Nineteen Hundred and Sixty Seven",
|
||||
"Thirteenth of DeCeMbEr, One Thousand, Nine Hundred, and Sixty Seven",
|
||||
"The 12th of December, 2024",
|
||||
"The 145th of January, 2023",
|
||||
"12th day of December, 2024",
|
||||
"1st day of January, 2025",
|
||||
"2nd day of JaNuArY, 2025",
|
||||
"3rd day of JANUARY, 2025",
|
||||
"21st day of January, 2025",
|
||||
"22nd day of January, 2025",
|
||||
"On 12 December 2025",
|
||||
]
|
||||
|
||||
numeric_dates = [
|
||||
"20.12.2024",
|
||||
"01.01.1000",
|
||||
"05.12.06",
|
||||
"12.18.2024",
|
||||
"20241212",
|
||||
"11/18/2014",
|
||||
"11-18-2024",
|
||||
"11/18/24",
|
||||
"11-18-24",
|
||||
"2024/11/18",
|
||||
"2024-11-18",
|
||||
"18/11/24",
|
||||
"1224",
|
||||
"2024.12.09",
|
||||
]
|
||||
|
||||
abbreviated_dates = [
|
||||
"18-Nov-2024",
|
||||
"Nov-18-2024",
|
||||
"Dec/12/2024",
|
||||
"Dec.12.2024",
|
||||
"Dec/2024",
|
||||
]
|
||||
|
||||
contextual_dates = [
|
||||
"Friday, December 18, 2024",
|
||||
"Fri, Dec 18, 2024",
|
||||
]
|
||||
|
||||
compact_dates = [
|
||||
"12-December-2024",
|
||||
"December 2024",
|
||||
"Dec-12-2024",
|
||||
]
|
||||
|
||||
case_variations = [
|
||||
"friday, december 18, 2024",
|
||||
"FRI, DEC 18, 2024",
|
||||
"DECEMBER 2024",
|
||||
# Multiple spaces
|
||||
"Friday, December 18, 2024",
|
||||
"Dec 12, 2024",
|
||||
# Multiple newlines
|
||||
"Friday,\nDecember\n18,\n2024",
|
||||
"Dec\n12,\n2024",
|
||||
# Original formats with newlines
|
||||
"1 day\nof December\n,\n2006",
|
||||
"1\nday\nof\nDecember\n,\n2006",
|
||||
"1st day\nof\nDecember,\n2006",
|
||||
]
|
||||
|
||||
ordinal_dates = [
|
||||
"1st day of December, 2006",
|
||||
"1st Day of December,2006",
|
||||
"2nd day of January, 2024",
|
||||
"2nd Day of Jan,2024",
|
||||
"3rd day of November, 1999",
|
||||
"3rd Day of Nov,1999",
|
||||
"4th day of March, 2023",
|
||||
"4th Day of Mar,2023",
|
||||
"21st day of April, 2022",
|
||||
"22nd day of May,2022",
|
||||
"23rd day of June, 2021",
|
||||
"24th Day of July,2021",
|
||||
]
|
||||
|
||||
invalid_dates = [
|
||||
"Invalid date",
|
||||
"Not a date",
|
||||
"Decide", # make sure "dec" doesn't accidentally match to "decide" or other words
|
||||
"decided",
|
||||
"Novel", # make sure "nov" doesn't accidentally match to other words
|
||||
"novelty",
|
||||
]
|
||||
|
||||
|
||||
def test_valid_dates():
|
||||
for date_list in [legal_traditional_dates, numeric_dates, abbreviated_dates,
|
||||
contextual_dates, compact_dates, case_variations, ordinal_dates]:
|
||||
for date in date_list:
|
||||
match = re.findall(DATE_PATTERN, date)
|
||||
assert match
|
||||
|
||||
def test_invalid_dates():
|
||||
for date in invalid_dates:
|
||||
match = re.findall(DATE_PATTERN, date)
|
||||
assert not match
|
||||
@@ -1,207 +0,0 @@
|
||||
import math
|
||||
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from pandas.testing import assert_frame_equal
|
||||
|
||||
# from scripts.cnc_hotfixes.hotfix_helper_funcs import state_check
|
||||
|
||||
|
||||
# class TestStateCheck:
|
||||
|
||||
# @pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
# ("Multiple", "N/A",["N/A"]),
|
||||
# ("Peach State", "N/A",["N/A"]),
|
||||
# ("Statewide", "N/A",["N/A"]),
|
||||
# ("Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
# ("The Contract Text Indicates This Health Plan Is For The Sunshine State, Which Is Florida. Therefore, The Answer Is:\n\nFlorida->N/A", "N/A", ["N/A"]),
|
||||
# ("Sunshine State", "N/A", ["N/A"]),
|
||||
# ("The Health Plan Is For The Sunshine State, Which Refers To Florida. However, Since The State Is Not Explicitly Specified, The Most Accurate Answer Based Solely On The Information Provided Is:\n\nN/A", "N/A", ["N/A"]),
|
||||
# ])
|
||||
|
||||
# def test_invalid_state_names(self, input_state, expected_df_output, expected_non_df_output):
|
||||
# """Test invalid inputs"""
|
||||
|
||||
# assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# def test_hlorida_correction(self):
|
||||
# """Test fix - Hlorida -> Florida"""
|
||||
# assert state_check("Hlorida",is_dataframe=False) == ["Florida"]
|
||||
|
||||
# @pytest.mark.parametrize("input_state,expected_df_output,expected_non_df_output", [
|
||||
# ("AL", "Alabama",["Alabama"]),
|
||||
# ("Florida", "Florida",["Florida"]),
|
||||
# ("FLORIDA", "Florida",["Florida"]),
|
||||
# ("florida", "Florida",["Florida"]),
|
||||
# ])
|
||||
|
||||
# def test_valid_states(self, input_state, expected_df_output, expected_non_df_output):
|
||||
# """Test valid inputs"""
|
||||
# assert state_check(input_state,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_state,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# def test_list_inputs(self):
|
||||
# """Test a list of input"""
|
||||
# input_states = ["AL", "Florida", "Hlorida", "Multiple"]
|
||||
|
||||
# expected_df_output = "Alabama, Florida, Florida, N/A"
|
||||
# expected_non_df_output = ["Alabama", "Florida", "Florida", "N/A"]
|
||||
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# @pytest.mark.parametrize("invalid_input", [
|
||||
# 123,
|
||||
# {"state": "Florida"},
|
||||
# True
|
||||
# ])
|
||||
|
||||
# def test_invalid_input_types(self, invalid_input):
|
||||
# """Test invalid input types"""
|
||||
|
||||
# with pytest.raises(TypeError):
|
||||
# state_check(invalid_input)
|
||||
|
||||
# @pytest.mark.parametrize("empty_input,expected_df_output,expected_non_df_output", [
|
||||
# ("", "N/A",["N/A"]),
|
||||
# ([], "",[]),
|
||||
# ([" "], "N/A",["N/A"])
|
||||
# ])
|
||||
|
||||
# def test_empty_inputs(self, empty_input, expected_df_output, expected_non_df_output):
|
||||
# """Test empty inputs"""
|
||||
# assert state_check(empty_input,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(empty_input,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# def test_multiple_invalid_states(self):
|
||||
# """Test multiple invalid inputs"""
|
||||
|
||||
# input_states = [
|
||||
# "Multiple",
|
||||
# "Peach State",
|
||||
# "The Health Plan Is For The Sunshine State, Which Refers To Florida. So The Answer Is:\n\nFlorida->N/A",
|
||||
# "The Health Plan Is For The Florida State, Which Refers To Florida",
|
||||
# "Sunshine State Health Plan Suggests This Is For Florida, But Since The State Is Not Explicitly Specified, The Most Appropriate Answer Is:\n\nN/A->N/A",
|
||||
# "Sunshine State Health Plan->N/A",
|
||||
# "Sunshine State->N/A",
|
||||
# "The Contract Text Indicates This Health Plan Is For Florida, As It Mentions Sunshine State Health Plan, Inc. Which Operates In Florida. However, To Strictly Follow The Instructions, I Will Provide The Answer:\n\nN/A->N/A"
|
||||
|
||||
# ]
|
||||
|
||||
# expected_df_output = ", ".join(["N/A"]*8)
|
||||
# expected_non_df_output = ["N/A"]*8
|
||||
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# def test_mixed_states(self):
|
||||
# """Test mixed input"""
|
||||
# input_states = ["AL", "Multiple", "Florida", "Hlorida", "Florida State Health Plan"]
|
||||
|
||||
# expected_df_output = "Alabama, N/A, Florida, Florida, N/A"
|
||||
# expected_non_df_output = ["Alabama", "N/A", "Florida", "Florida", "N/A"]
|
||||
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# def concatenated_string_states(self):
|
||||
# """Test concatentated input"""
|
||||
# input_states = ["Arizona, Louisiana","NY, NJ, Connecticut"]
|
||||
|
||||
# expected_df_output = "Arizona, Louisiana, New York, New Jersey"
|
||||
# expected_non_df_output = ["Arizona", "Louisiana", "New York", "New Jersey"]
|
||||
|
||||
# assert state_check(input_states,is_dataframe=True) == expected_df_output
|
||||
# assert state_check(input_states,is_dataframe=False) == expected_non_df_output
|
||||
|
||||
# class TestApplyStateCheckToDataFrame:
|
||||
|
||||
# def test_basic_state_processing(self):
|
||||
# """Test basic state code to full name conversion"""
|
||||
# df = pd.DataFrame({
|
||||
# 'Health Plan State': ['NY', 'CA', 'TX', ['CO','CT'], 'NJ, AR']
|
||||
# })
|
||||
|
||||
# expected_df = pd.DataFrame({
|
||||
# 'Health Plan State': ['NY', 'CA', 'TX', ['CO','CT'], 'NJ, AR'],
|
||||
# 'Health Plan State_corrected': ['New York', 'California', 'Texas', 'Colorado, Connecticut', 'New Jersey, Arkansas']
|
||||
# })
|
||||
|
||||
# result_df = apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# assert_frame_equal(result_df, expected_df)
|
||||
|
||||
# def test_mixed_case_states(self):
|
||||
# """Test handling of mixed case input"""
|
||||
# df = pd.DataFrame({
|
||||
# 'Health Plan State': ['ny', 'Ca', 'TeXaS']
|
||||
# })
|
||||
|
||||
# expected_df = pd.DataFrame({
|
||||
# 'Health Plan State': ['ny', 'Ca', 'TeXaS'],
|
||||
# 'Health Plan State_corrected': ['New York', 'California', 'Texas']
|
||||
# })
|
||||
|
||||
# result_df = apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# assert_frame_equal(result_df, expected_df)
|
||||
|
||||
# def test_invalid_states(self):
|
||||
# """Test handling of invalid state codes"""
|
||||
|
||||
# df = pd.DataFrame({
|
||||
# 'Health Plan State': [['SH', 'IO'], 'LM', 'XY']
|
||||
# })
|
||||
|
||||
# expected_df = pd.DataFrame({
|
||||
# 'Health Plan State': [['SH', 'IO'], 'LM', 'XY'],
|
||||
# 'Health Plan State_corrected': ['N/A, N/A', 'N/A', 'N/A']
|
||||
# })
|
||||
|
||||
# result_df = apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# assert_frame_equal(result_df, expected_df)
|
||||
|
||||
# def test_already_full_state_names(self):
|
||||
# """Test handling of already expanded state names"""
|
||||
# df = pd.DataFrame({
|
||||
# 'Health Plan State': ['Vermont', 'Tennessee', 'Mississippi']
|
||||
# })
|
||||
|
||||
# expected_df = pd.DataFrame({
|
||||
# 'Health Plan State': ['Vermont', 'Tennessee', 'Mississippi'],
|
||||
# 'Health Plan State_corrected': ['Vermont', 'Tennessee', 'Mississippi']
|
||||
# })
|
||||
|
||||
# result_df = apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# assert_frame_equal(result_df, expected_df)
|
||||
|
||||
# def test_empty_dataframe(self):
|
||||
# """Test handling of empty DataFrame"""
|
||||
# df = pd.DataFrame({'Health Plan State': []})
|
||||
# expected_df = pd.DataFrame({
|
||||
# 'Health Plan State': [],
|
||||
# 'Health Plan State_corrected': []
|
||||
# })
|
||||
|
||||
# result_df = apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# assert_frame_equal(result_df, expected_df)
|
||||
|
||||
# def test_wrong_column_name(self):
|
||||
# """Test error handling for non-existent column"""
|
||||
# df = pd.DataFrame({
|
||||
# 'Heaalth Plans state': ['NY', 'CA'],
|
||||
# })
|
||||
|
||||
# with pytest.raises(KeyError) as exc_info:
|
||||
# apply_state_check_to_df(df,is_dataframe=True)
|
||||
|
||||
# def test_is_dataframe_parameter_false(self):
|
||||
# """Test 'is_dataframe' parameter as false"""
|
||||
# df = pd.DataFrame({'Health Plan State': ['NY', 'CA']})
|
||||
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_state_check_to_df(df,is_dataframe=False)
|
||||
@@ -1,22 +0,0 @@
|
||||
import pytest
|
||||
|
||||
# from hotfix_helper_funcs import apply_date_formatting_fix
|
||||
|
||||
# def test_change_date_formatting():
|
||||
# assert apply_date_formatting_fix("31/12/2012") == "12/31/2012"
|
||||
# assert apply_date_formatting_fix("20/07/2025") == "07/20/2025"
|
||||
# assert apply_date_formatting_fix("13/12/1992") == "12/13/1992"
|
||||
|
||||
# def test_date_formatting_no_change():
|
||||
# assert apply_date_formatting_fix("11/21/2024") == "11/21/2024"
|
||||
# assert apply_date_formatting_fix("07/20/1992") == "07/20/1992"
|
||||
# assert apply_date_formatting_fix("01/02/2000") == "01/02/2000"
|
||||
|
||||
# def test_date_formatting_na():
|
||||
# assert apply_date_formatting_fix("N/A") == "N/A"
|
||||
|
||||
# def test_invalid_date():
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("7/20/1992")
|
||||
# with pytest.raises(ValueError):
|
||||
# apply_date_formatting_fix("some string")
|
||||
@@ -1,91 +0,0 @@
|
||||
import os
|
||||
|
||||
import pandas as pd
|
||||
|
||||
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
||||
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
||||
|
||||
print("Head of df_batch_1_clean_abc['Contract_Name']:")
|
||||
print(df_batch_1_clean_abc['Contract_Name'].head())
|
||||
|
||||
fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
||||
df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc)
|
||||
|
||||
df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name']
|
||||
print("Head of df_batch_2_clean_abc['Contract_Name']:")
|
||||
print(df_batch_2_clean_abc['Contract_Name'].head())
|
||||
|
||||
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
||||
print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
||||
|
||||
print("batch 1 columns: ")
|
||||
for column in df_batch_1_clean_abc.columns:
|
||||
print(column)
|
||||
|
||||
print("\nbatch 2 columns: ")
|
||||
for column in df_batch_2_clean_abc.columns:
|
||||
print(column)
|
||||
|
||||
print("\n columns in batch 2 not in batch 1: ")
|
||||
b1cols = set(df_batch_1_clean_abc.columns)
|
||||
b2cols = set(df_batch_2_clean_abc.columns)
|
||||
print(b2cols.difference(b1cols))
|
||||
|
||||
quit()
|
||||
|
||||
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
||||
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
||||
|
||||
dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
||||
fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
||||
for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
||||
|
||||
|
||||
pre_concat_list_batch_1 = []
|
||||
pre_concat_list_batch_2 = []
|
||||
print("starting concatenation for batch 1")
|
||||
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
||||
|
||||
for fn in fn_ac2_batch_1_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_batch_1)) + " files post-CSV reading")
|
||||
df_concat_batch_1_indiv_outputs = pd.concat(pre_concat_list_batch_1)
|
||||
|
||||
print("batch 1 concatenation complete")
|
||||
|
||||
print("starting concatenation for batch 2")
|
||||
print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading")
|
||||
for fn in fn_ac2_batch_2_outputs_list:
|
||||
try:
|
||||
pre_concat_list_batch_2.append(pd.read_csv(fn))
|
||||
except Exception as e:
|
||||
print(f"Appending failed on {fn} with exception {e}")
|
||||
|
||||
print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading")
|
||||
df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2)
|
||||
|
||||
print("batch 2 concatenation complete")
|
||||
|
||||
print("starting merging for batch 1")
|
||||
|
||||
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
|
||||
batch_1_merged = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
batch_1_merged.to_csv("batch_1_merged.csv", index=False)
|
||||
print("batch 1 merge completed. Merge status counts:")
|
||||
print(batch_1_merged['_merge'].value_counts())
|
||||
print("Batch 1 merged and written to CSV")
|
||||
|
||||
print("starting merging for batch 2")
|
||||
df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt'
|
||||
batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name',
|
||||
right_on='Contract Name', indicator=True)
|
||||
batch_2_merged.to_csv("batch_2_merged.csv", index=False)
|
||||
print("batch 2 merge completed. Merge status counts:")
|
||||
print(batch_2_merged['_merge'].value_counts())
|
||||
print("Batch 2 merged and written to CSV")
|
||||
@@ -110,47 +110,3 @@ class TestPreprocessingFuncs:
|
||||
contract, quick_review = filter_quick_review(input_dict)
|
||||
assert contract == expected_contract
|
||||
assert quick_review == expected_quick_review
|
||||
|
||||
|
||||
@patch('src.utils.llm_utils.invoke_claude')
|
||||
@patch('src.utils.string_utils.extract_text_from_delimiters')
|
||||
def test_get_exhibit_pages(self, mock_extract_text, mock_invoke_claude):
|
||||
"""Test exhibit page detection with both base and decimal pages"""
|
||||
# Setup test data
|
||||
text_dict = {
|
||||
"1": "First page content",
|
||||
"2.0": "Second page content with exhibit header",
|
||||
"2.1": "Continuation of second page",
|
||||
"2.2": "More continuation of second page",
|
||||
"3.0": "Third page content with exhibit header",
|
||||
"4": "Regular page content"
|
||||
}
|
||||
|
||||
filename = "test_contract.pdf"
|
||||
|
||||
# Configure mocks
|
||||
mock_invoke_claude.return_value = "raw_llm_response"
|
||||
mock_extract_text.side_effect = [
|
||||
"Exhibit A", # For page 2.0
|
||||
"N/A", # For page 4
|
||||
"Exhibit B" # For page 3.0
|
||||
]
|
||||
|
||||
# Execute
|
||||
exhibit_pages, exhibit_headers = get_exhibit_pages(text_dict, filename)
|
||||
|
||||
# Assert
|
||||
expected_exhibit_pages = ["1", "2.0", "2.1", "2.2", '4']
|
||||
|
||||
assert exhibit_pages == expected_exhibit_pages
|
||||
|
||||
# Verify LLM was only called for base pages
|
||||
assert mock_invoke_claude.call_count == 3
|
||||
|
||||
# Verify prompt content and model
|
||||
mock_invoke_claude.assert_any_call(
|
||||
investment_prompts.EXHIBIT_HEADER("Second page content with exhibit header"[0:400]),
|
||||
config.MODEL_ID_CLAUDE3_HAIKU,
|
||||
filename,
|
||||
max_tokens=150
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user