From 8c99f5d073c9fbefa4636daa7281326c33dfd11b Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Thu, 14 Aug 2025 16:46:28 +0000 Subject: [PATCH] Merged in bugfix/dynamic-primary-issues (pull request #664) Bugfix/dynamic primary issues * format and restructure * checkpoint * Merge branch 'main' into bugfix/dynamic-primary-issues * Update constants attributes to base class * Revert "Update constants attributes to base class" This reverts commit 936b1e41f2e2d445e69e840b69af08eac6661120. * Fix mapping * E2E test * remove prints * Merge branch 'main' into bugfix/dynamic-primary-issues * Black * Update any dynamic * Prompt update - more leniency * Remove prints * Merged main into bugfix/dynamic-primary-issues * update crosswalk * Merged main into bugfix/dynamic-primary-issues Approved-by: Alex Galarce --- .../constants/mappings/crosswalk_network.json | 2 +- .../constants/mappings/crosswalk_program.json | 6 +- .../src/investment/aarete_derived.py | 13 +- .../src/investment/dynamic_funcs.py | 137 ++++++++++++------ .../src/investment/file_processing.py | 105 +++----------- .../src/investment/one_to_n_funcs.py | 123 ++++++++++++++-- .../src/investment/one_to_one_funcs.py | 14 +- fieldExtraction/src/investment/postprocess.py | 1 - .../src/investment/preprocessing_funcs.py | 4 +- fieldExtraction/src/investment/row_funcs.py | 7 +- .../src/investment/smart_chunking_funcs.py | 4 +- fieldExtraction/src/investment/test.py | 59 -------- fieldExtraction/src/prompts/fieldset.py | 23 +-- .../src/prompts/prompt_templates.py | 6 +- fieldExtraction/src/utils/qa_qc_utils.py | 1 - 15 files changed, 272 insertions(+), 233 deletions(-) delete mode 100644 fieldExtraction/src/investment/test.py diff --git a/fieldExtraction/constants/mappings/crosswalk_network.json b/fieldExtraction/constants/mappings/crosswalk_network.json index b811539..fa22bf3 100644 --- a/fieldExtraction/constants/mappings/crosswalk_network.json +++ b/fieldExtraction/constants/mappings/crosswalk_network.json @@ -8,7 +8,7 @@ "Preferred Provider Organization (PPO)": "PPO", "Exclusive Provider Organization (EPO)": "EPO", "Point of Service (POS)": "POS", - "PPO-POS" : "PPO | POS", + "PPO-POS" : "PPO|POS", "High Deductible Health Plan (HDHP)": "HDHP", "Indemnity" : "FFS" } diff --git a/fieldExtraction/constants/mappings/crosswalk_program.json b/fieldExtraction/constants/mappings/crosswalk_program.json index 4c76dc1..c3bf60f 100644 --- a/fieldExtraction/constants/mappings/crosswalk_program.json +++ b/fieldExtraction/constants/mappings/crosswalk_program.json @@ -21,9 +21,9 @@ "Temporary Assistance Needy Families (TANF)" : "TANF", "Long-Term Services and Supports (LTSS)" : "LTSS", "Special Needs Plan (SNP)" : "SNP", - "Medicare Advantage Special Needs Plan (MA-SNP)" : "MASNP", - "Chronic Condition Special Needs Plan (C-SNP)" : "CSNP", - "Dual Eligible Special Needs Plan (D-SNP)" : "DSNP", + "Medicare Advantage Special Needs Plan (MASNP)" : "MASNP", + "Chronic Condition Special Needs Plan (CSNP)" : "CSNP", + "Dual Eligible Special Needs Plan (DSNP)" : "DSNP", "Fully Integrated Dual Eligible Special Needs Plan (FIDE-SNP)" : "FIDE SNP", "Highly Integrated Dual Eligible Special Needs Plan (HIDE-SNP)" : "HIDE SNP", "Institutional Special Needs Plan (I-SNP)" : "ISNP", diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 9d7a27b..24a63bd 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -73,13 +73,18 @@ def get_crosswalk_fields(answer_dicts): from_field_value = answer_dict.get(from_field) if not string_utils.is_empty(from_field_value): - to_field_value = apply_crosswalk(from_field_value, crosswalk_mapping) - - if not to_field_value or to_field_value == from_field_value: + if from_field_value in crosswalk_mapping.values(): + to_field_value = from_field_value + else: to_field_value = apply_crosswalk( - from_field_value, crosswalk_mapping_reverse + from_field_value, crosswalk_mapping ) + if not to_field_value or to_field_value == from_field_value: + to_field_value = apply_crosswalk( + from_field_value, crosswalk_mapping_reverse + ) + answer_dict[to_field.field_name] = to_field_value just_mapped.append(to_field.field_name) diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index 3ae382a..3171ff5 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -26,13 +26,29 @@ def prompt_dynamic(text, field_prompts, filename): return llm_answer_final +def add_full_context_field(one_to_one_fields, field_to_add): + field_to_add.field_type = "full_context" + field_to_add.relationship = "one_to_one" + + if field_to_add.allow_na: + field_to_add.prompt = ( + field_to_add.prompt + + " Only answer if the answer applies to the ENTIRE contract. Otherwise, answer N/A." + ) + else: + field_to_add.prompt = ( + field_to_add.prompt + + " Do NOT leave this field N/A. Choose the most appropriate answer based on the context." + ) + one_to_one_fields.add_field(field_to_add) + return one_to_one_fields + + def get_dynamic_one_to_one_fields(answer_dicts): """ Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts """ - fieldset = FieldSet( - config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True - ) + rate_escalator_fields = FieldSet( config.FIELD_JSON_PATH, relationship="one_to_n", @@ -45,50 +61,59 @@ def get_dynamic_one_to_one_fields(answer_dicts): field_type="exhibit_level", field_name="OUTLIER_TERMS", ) - fieldset.combine(rate_escalator_fields, inplace=True) - fieldset.combine(outlier_terms, inplace=True) - one_to_one_fields = FieldSet() - for field in fieldset.fields: + all_empty_fields = FieldSet( + config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True + ) # These fields get passed to 1:1 if ALL of the answers are empty + any_empty_fields = rate_escalator_fields.combine( + outlier_terms, inplace=False + ) # These fields get passed to 1:1 if ANY of the answers are empty + one_to_one_fields = FieldSet() # Empty FieldSet to populate if criteria are met + + # Handle ANY empty fields + for field in any_empty_fields.fields: field_name = field.field_name - if "REIMB" not in field_name: # Don't do this for the REIMB_ fields - for answer_dict in answer_dicts: - if ( - string_utils.is_empty(answer_dict.get(field_name)) - or field_name not in answer_dict - ): - if field_name == "RATE_ESCALATOR_DESC": - # If the field is RATE_ESCALATOR_DESC, we need to add RATE_ESCALATOR_STATEMENT field and change its field type and relationship - field_to_add = Field.load_from_file( - file_path=config.FIELD_JSON_PATH, - field_name="RATE_ESCALATOR_STATEMENT", - ) - elif field_name == "OUTLIER_TERMS": - field_to_add = Field.load_from_file( - file_path=config.FIELD_JSON_PATH, - field_name="OUTLIER_TERMS", - ) - else: - field_to_add = Field.load_from_file( - file_path=config.FIELD_JSON_PATH, - field_name=field.base_field, - ) + if "REIMB" in field_name: # Don't do this for the REIMB_ fields + continue + for answer_dict in answer_dicts: + if string_utils.is_empty(answer_dict.get(field_name)): + if field_name == "RATE_ESCALATOR_DESC": + # If the field is RATE_ESCALATOR_DESC, we need to add RATE_ESCALATOR_STATEMENT field and change its field type and relationship + field_to_add = Field.load_from_file( + file_path=config.FIELD_JSON_PATH, + field_name="RATE_ESCALATOR_STATEMENT", + ) + elif field_name == "OUTLIER_TERMS": + field_to_add = Field.load_from_file( + file_path=config.FIELD_JSON_PATH, + field_name="OUTLIER_TERMS", + ) + else: + field_to_add = Field.load_from_file( + file_path=config.FIELD_JSON_PATH, + field_name=field.base_field, + ) + one_to_one_fields = add_full_context_field( + one_to_one_fields, field_to_add + ) + continue - field_to_add.field_type = "full_context" - field_to_add.relationship = "one_to_one" - - if field_to_add.allow_na: - field_to_add.prompt = ( - field_to_add.prompt - + " Only answer if the answer applies to the ENTIRE contract. Otherwise, answer N/A." - ) - else: - field_to_add.prompt = ( - field_to_add.prompt - + " Do NOT leave this field N/A. Choose the most appropriate answer based on the context." - ) - one_to_one_fields.add_field(field_to_add) - continue + # Handle ALL empty fields + for field in all_empty_fields.fields: + field_name = field.field_name + if "REIMB" in field_name: # Don't do this for the REIMB_ fields + continue + if all( + [ + string_utils.is_empty(answer_dict.get(field_name)) + for answer_dict in answer_dicts + ] + ): + field_to_add = Field.load_from_file( + file_path=config.FIELD_JSON_PATH, + field_name=field.base_field, + ) + one_to_one_fields = add_full_context_field(one_to_one_fields, field_to_add) return one_to_one_fields @@ -109,7 +134,11 @@ def prompt_dynamic_primary(exhibit_text, constants, field, filename): def get_dynamic_primary( - exhibit_chunk: str, exhibit_header: str, constants: Constants, filename: str, dynamic_fields: FieldSet + exhibit_chunk: str, + exhibit_header: str, + constants: Constants, + filename: str, + dynamic_fields: FieldSet, ): """ Processes dynamic primary fields one-by-one from exhibit header and chunk text. @@ -131,7 +160,13 @@ def get_dynamic_primary( reimbursement_level_fields = FieldSet() exhibit_level_answer_dict = {} for field in dynamic_fields.fields: - exhibit_header_answer = prompt_dynamic_primary(exhibit_header, constants, field, filename) + if string_utils.is_empty(exhibit_header): + exhibit_header_answer = "N/A" + else: + exhibit_header_answer = prompt_dynamic_primary( + exhibit_header, constants, field, filename + ) + # If there is exactly ONE Non-N/A answer in the header if ( not string_utils.is_empty(exhibit_header_answer) @@ -152,7 +187,9 @@ def get_dynamic_primary( dynamic_fields.remove_field(field.field_name) # If there are ZERO Non-N/A answers in the header else: - exhibit_text_answer = prompt_dynamic_primary(exhibit_chunk, constants, field, filename) + exhibit_text_answer = prompt_dynamic_primary( + exhibit_chunk, constants, field, filename + ) # If there is at least ONE Non-N/A answers in the Exhibit if not string_utils.is_empty(exhibit_text_answer): field.update_valid_values(exhibit_text_answer + " | N/A") @@ -166,7 +203,11 @@ def get_dynamic_primary( def get_dynamic_answers( - exhibit_chunk: str, exhibit_header: str, filename: str, dynamic_fields: FieldSet, constants: Constants + exhibit_chunk: str, + exhibit_header: str, + filename: str, + dynamic_fields: FieldSet, + constants: Constants, ): """ Processes dynamic fields from exhibit header and chunk text. diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 1788af0..833b95f 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -52,7 +52,12 @@ def process_file(file_object, constants: Constants, run_timestamp): ################## ONE TO ONE ################## one_to_one_results = run_one_to_one_prompts( - filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields, constants + filename, + contract_text, + text_dict, + top_sheet_dict, + dynamic_one_to_one_fields, + constants, ) # Return dict one_to_one_results["FILE_NAME"] = filename print(f"{datetime_str()} One to One Complete - {filename}") @@ -64,6 +69,7 @@ def process_file(file_object, constants: Constants, run_timestamp): ################## CODES ################## final_results = code_funcs.code_breakout(merged_results, constants) + print(f"{datetime_str()} Codes Complete - {filename}") ################## POSTPROCESS ################## final_df = postprocess.postprocess(final_results, constants) @@ -80,7 +86,12 @@ def process_file(file_object, constants: Constants, run_timestamp): def run_one_to_one_prompts( - filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields, constants + filename, + contract_text, + text_dict, + top_sheet_dict, + dynamic_one_to_one_fields, + constants, ): ################## INITIALIZE FIELDS ################## @@ -114,7 +125,9 @@ def run_one_to_one_prompts( one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results) if "OUTLIER_TERMS" in one_to_one_results: - outlier_results = one_to_n_funcs.outlier_terms_breakout(one_to_one_results["OUTLIER_TERMS"],filename) + outlier_results = one_to_n_funcs.outlier_terms_breakout( + one_to_one_results["OUTLIER_TERMS"], filename + ) one_to_one_results.update(outlier_results) return one_to_one_results[0] @@ -133,89 +146,15 @@ def run_one_to_n_prompts(filename, exhibit_dict, all_exhibit_headers, constants) exhibit_text = exhibit_dict[exhibit_page] ################# GET EXHIBIT-LEVEL ANSWERS ################## - if exhibit_page in all_exhibit_headers.keys(): - exhibit_header = all_exhibit_headers[exhibit_page] - elif ( - ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys() - ): # Take the parent header if it exists - exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])] - else: - exhibit_header = "No Header Found" - - exhibit_level_answers = one_to_n_funcs.get_exhibit_level_answers( - exhibit_text, constants, filename - ) - exhibit_level_answers["EXHIBIT_PAGE"] = exhibit_page - exhibit_level_answers["EXHIBIT_TITLE"] = exhibit_header - - # Extract exhibit lesser-of statement for passing to reimbursement_level - exhibit_lesser_of = exhibit_level_answers.get( - "EXHIBIT_LESSER_OF_STATEMENT", "N/A" - ) - - ################## INITIALIZE FIELDS ################## - reimbursement_level_fields = FieldSet( - relationship="one_to_n", - field_type="reimbursement_level", - file_path=FIELD_JSON_PATH, - ) - - ################## GET DYNAMIC-PRIMARY ANSWERS ################## - dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = ( - dynamic_funcs.get_dynamic_primary( - exhibit_text, - exhibit_header, - constants, - filename, - FieldSet( - relationship="one_to_n", - field_type="dynamic_primary", - file_path=config.FIELD_JSON_PATH, - ), + exhibit_level_answers, exhibit_lesser_of, exhibit_header = ( + one_to_n_funcs.get_exhibit_level( + exhibit_text, exhibit_page, all_exhibit_headers, constants, filename ) ) - exhibit_level_answers.update(dynamic_to_exhibit_level_answers) - reimbursement_level_fields.combine( - dynamic_to_reimbursement_level_fields, inplace=True - ) - ################## GET DYNAMIC-REIMB-INFO ANSWERS ################## - ( - reimb_info_to_exhibit_level_answers, - reimb_info_to_reimbursement_level_fields, - ) = dynamic_funcs.get_dynamic_answers( - exhibit_text, - exhibit_header, - filename, - FieldSet( - relationship="one_to_n", - field_type="dynamic_reimb_info", - file_path=config.FIELD_JSON_PATH, - ), - constants - ) - exhibit_level_answers.update(reimb_info_to_exhibit_level_answers) - reimbursement_level_fields.combine( - reimb_info_to_reimbursement_level_fields, inplace=True - ) - - ################## GET DYNAMIC-CODE ANSWERS ################## - code_to_exhibit_level_answers, code_to_reimbursement_level_fields = ( - dynamic_funcs.get_dynamic_answers( - exhibit_text, - exhibit_header, - filename, - FieldSet( - relationship="one_to_n", - field_type="dynamic_code", - file_path=config.FIELD_JSON_PATH, - ), - constants - ) - ) - exhibit_level_answers.update(code_to_exhibit_level_answers) - reimbursement_level_fields.combine( - code_to_reimbursement_level_fields, inplace=True + ################# GET DYNAMIC ANSWERS ################## + exhibit_level_answers, reimbursement_level_fields = one_to_n_funcs.get_dynamic( + exhibit_text, exhibit_header, exhibit_level_answers, constants, filename ) ################## GET REIMBURSEMENT-LEVEL ANSWERS (INCLUDING DYNAMIC) ################## diff --git a/fieldExtraction/src/investment/one_to_n_funcs.py b/fieldExtraction/src/investment/one_to_n_funcs.py index 4cd6779..2e2e153 100644 --- a/fieldExtraction/src/investment/one_to_n_funcs.py +++ b/fieldExtraction/src/investment/one_to_n_funcs.py @@ -16,6 +16,108 @@ from src import config from src.config import FIELD_JSON_PATH from src.prompts.prompt_templates import EXHIBIT_LEVEL from src.prompts.fieldset import FieldSet +import src.investment.dynamic_funcs as dynamic_funcs + + +def get_exhibit_level( + exhibit_text: str, + exhibit_page: str, + all_exhibit_headers: dict, + constants: Constants, + filename: str, +): + + if exhibit_page in all_exhibit_headers.keys(): + exhibit_header = all_exhibit_headers[exhibit_page] + elif ( + ".".join(exhibit_page.split(".")[:-1]) in all_exhibit_headers.keys() + ): # Take the parent header if it exists + exhibit_header = all_exhibit_headers[".".join(exhibit_page.split(".")[:-1])] + else: + exhibit_header = "No Header Found" + + exhibit_level_answers = get_exhibit_level_answers(exhibit_text, constants, filename) + exhibit_level_answers["EXHIBIT_PAGE"] = exhibit_page + exhibit_level_answers["EXHIBIT_TITLE"] = exhibit_header + + # Extract exhibit lesser-of statement for passing to reimbursement_level + exhibit_lesser_of = exhibit_level_answers.get("EXHIBIT_LESSER_OF_STATEMENT", "N/A") + + return exhibit_level_answers, exhibit_lesser_of, exhibit_header + + +def get_dynamic( + exhibit_text: str, + exhibit_header: str, + exhibit_level_answers: dict, + constants: Constants, + filename: str, +): + + ################## INITIALIZE FIELDS ################## + reimbursement_level_fields = FieldSet( + relationship="one_to_n", + field_type="reimbursement_level", + file_path=FIELD_JSON_PATH, + ) + + ################## GET DYNAMIC-PRIMARY ANSWERS ################## + dynamic_to_exhibit_level_answers, dynamic_to_reimbursement_level_fields = ( + dynamic_funcs.get_dynamic_primary( + exhibit_text, + exhibit_header, + constants, + filename, + FieldSet( + relationship="one_to_n", + field_type="dynamic_primary", + file_path=config.FIELD_JSON_PATH, + ), + ) + ) + exhibit_level_answers.update(dynamic_to_exhibit_level_answers) + reimbursement_level_fields.combine( + dynamic_to_reimbursement_level_fields, inplace=True + ) + + ################## GET DYNAMIC-REIMB-INFO ANSWERS ################## + ( + reimb_info_to_exhibit_level_answers, + reimb_info_to_reimbursement_level_fields, + ) = dynamic_funcs.get_dynamic_answers( + exhibit_text, + exhibit_header, + filename, + FieldSet( + relationship="one_to_n", + field_type="dynamic_reimb_info", + file_path=config.FIELD_JSON_PATH, + ), + constants, + ) + exhibit_level_answers.update(reimb_info_to_exhibit_level_answers) + reimbursement_level_fields.combine( + reimb_info_to_reimbursement_level_fields, inplace=True + ) + + ################## GET DYNAMIC-CODE ANSWERS ################## + code_to_exhibit_level_answers, code_to_reimbursement_level_fields = ( + dynamic_funcs.get_dynamic_answers( + exhibit_text, + exhibit_header, + filename, + FieldSet( + relationship="one_to_n", + field_type="dynamic_code", + file_path=config.FIELD_JSON_PATH, + ), + constants, + ) + ) + exhibit_level_answers.update(code_to_exhibit_level_answers) + reimbursement_level_fields.combine(code_to_reimbursement_level_fields, inplace=True) + + return exhibit_level_answers, reimbursement_level_fields def get_exhibit_level_answers(exhibit_chunk, constants, filename): @@ -56,6 +158,7 @@ def get_reimbursement_primary( ) -> list[dict]: field_prompts = reimbursement_level_fields.print_prompt_dict(constants) prompt = prompt_templates.REIMBURSEMENT_LEVEL_PRIMARY(exhibit_text, field_prompts) + logging.debug( f"""Running reimbursement primary prompt for {filename} with prompt: {prompt}""" @@ -1165,9 +1268,7 @@ def process_special_cases( special_case_results.update(trigger_cap_results) elif case == "OUTLIER": logging.info(f"Running OUTLIERS breakout for {service} - {methodology}") - outlier_results = run_outliers_breakout( - service, methodology, filename - ) + outlier_results = run_outliers_breakout(service, methodology, filename) special_case_results.update(outlier_results) else: # Other/future special cases can go here pass @@ -1225,6 +1326,7 @@ def run_trigger_cap_breakout(service: str, methodology: str, filename: str) -> d ) return string_utils.universal_json_load(llm_response) or {} + def run_outliers_breakout(service: str, methodology: str, filename: str) -> dict: """ Runs focused OUTLIERS analysis using specialized prompt. @@ -1241,17 +1343,14 @@ def run_outliers_breakout(service: str, methodology: str, filename: str) -> dict file_path=config.FIELD_JSON_PATH, field_type="outlier" ).print_prompt_dict() - prompt = prompt_templates.OUTLIER_BREAKOUT( - service, methodology, outlier_questions - ) + prompt = prompt_templates.OUTLIER_BREAKOUT(service, methodology, outlier_questions) logging.debug(f"Running OUTLIERS breakout in {filename} with prompt: {prompt}") llm_response = llm_utils.invoke_claude(prompt, "sonnet_latest", filename) - logging.debug( - f"LLM Response for OUTLIERS breakout in {filename}: {llm_response}" - ) + logging.debug(f"LLM Response for OUTLIERS breakout in {filename}: {llm_response}") return string_utils.universal_json_load(llm_response) or {} + def outlier_terms_breakout(outlier_term, filename: str) -> dict: """Extract outlier fields from outlier term. @@ -1261,8 +1360,10 @@ def outlier_terms_breakout(outlier_term, filename: str) -> dict: Returns: dict: A dictionary containing the extracted outlier terms. """ - outlier_questions = FieldSet(file_path=config.FIELD_JSON_PATH, field_type="outlier").print_prompt_dict() + outlier_questions = FieldSet( + file_path=config.FIELD_JSON_PATH, field_type="outlier" + ).print_prompt_dict() prompt = prompt_templates.OUTLIER_TERMS_BREAKOUT(outlier_term, outlier_questions) llm_response = llm_utils.invoke_claude(prompt, "sonnet_latest", filename) outlier_answers = string_utils.universal_json_load(llm_response) or {} - return outlier_answers \ No newline at end of file + return outlier_answers diff --git a/fieldExtraction/src/investment/one_to_one_funcs.py b/fieldExtraction/src/investment/one_to_one_funcs.py index 6b59570..0576ec5 100644 --- a/fieldExtraction/src/investment/one_to_one_funcs.py +++ b/fieldExtraction/src/investment/one_to_one_funcs.py @@ -102,7 +102,11 @@ def filter_payer_from_providers( def run_smart_chunked_fields( - one_to_one_fields: FieldSet, constants, contract_text: str, filename: str, text_dict: dict + one_to_one_fields: FieldSet, + constants, + contract_text: str, + filename: str, + text_dict: dict, ) -> dict[str, str]: """Process fields that require smart chunking and return the extracted answers. @@ -383,9 +387,11 @@ def run_global_lesser_of_breakout( mock_service = "Global Lesser Of (all services)" prompt = REIMB_TERM_BREAKOUT( - mock_service, global_lesser_of_stmt, FieldSet( - file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout" - ).print_prompt_dict(constants) + mock_service, + global_lesser_of_stmt, + FieldSet( + file_path=config.FIELD_JSON_PATH, field_type="methodology_breakout" + ).print_prompt_dict(constants), ) logging.debug( f"Running global lesser of breakout for {filename} with prompt: {prompt}" diff --git a/fieldExtraction/src/investment/postprocess.py b/fieldExtraction/src/investment/postprocess.py index 41bc1e0..90668f1 100644 --- a/fieldExtraction/src/investment/postprocess.py +++ b/fieldExtraction/src/investment/postprocess.py @@ -4,7 +4,6 @@ from constants.investment_columns import COLUMN_ORDER from src.investment import postprocessing_funcs - def postprocess(df, constants: Constants): if df.shape[0] > 0: diff --git a/fieldExtraction/src/investment/preprocessing_funcs.py b/fieldExtraction/src/investment/preprocessing_funcs.py index 2513ff7..5a0ed16 100644 --- a/fieldExtraction/src/investment/preprocessing_funcs.py +++ b/fieldExtraction/src/investment/preprocessing_funcs.py @@ -331,9 +331,7 @@ def link_exhibit_pages( exhibit_is_different = "Y" previous_header = page_header else: - prompt = prompt_templates.EXHIBIT_LINKAGE( - page_header, previous_header - ) + prompt = prompt_templates.EXHIBIT_LINKAGE(page_header, previous_header) claude_answer_raw = llm_utils.invoke_claude( prompt, "legacy_sonnet", filename ) diff --git a/fieldExtraction/src/investment/row_funcs.py b/fieldExtraction/src/investment/row_funcs.py index fda6a29..b8cc514 100644 --- a/fieldExtraction/src/investment/row_funcs.py +++ b/fieldExtraction/src/investment/row_funcs.py @@ -51,7 +51,10 @@ def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results, constan def inject_global_lesser_of_rows( - one_to_n_results: pd.DataFrame, global_lesser_of_stmt: str, constants: Constants, filename: str + one_to_n_results: pd.DataFrame, + global_lesser_of_stmt: str, + constants: Constants, + filename: str, ) -> pd.DataFrame: """Create new rows with global lesser of constraint for REIMB_IDs that lack lesser of logic. @@ -82,7 +85,7 @@ def inject_global_lesser_of_rows( else: one_to_n_results["LESSER_OF_IND"] = "N" rows_needing_injection = one_to_n_results.copy() - + if rows_needing_injection.empty: logging.debug(f"No rows needing global lesser of injection for {filename}.") # Still need to set tracking flag for all existing rows diff --git a/fieldExtraction/src/investment/smart_chunking_funcs.py b/fieldExtraction/src/investment/smart_chunking_funcs.py index d539803..9dcc9e4 100644 --- a/fieldExtraction/src/investment/smart_chunking_funcs.py +++ b/fieldExtraction/src/investment/smart_chunking_funcs.py @@ -111,7 +111,9 @@ def stitch_chunks(chunks: list[str], overlap_size: int = CHUNK_OVERLAP) -> str: return result -def field_context(one_to_one_fields: FieldSet, constants: Constants, contract_text: str, filename: str): +def field_context( + one_to_one_fields: FieldSet, constants: Constants, contract_text: str, filename: str +): """this function creates chunks from whole contract text and returns relevant chunks for each field based on sementic search and keyword search diff --git a/fieldExtraction/src/investment/test.py b/fieldExtraction/src/investment/test.py deleted file mode 100644 index 58b422f..0000000 --- a/fieldExtraction/src/investment/test.py +++ /dev/null @@ -1,59 +0,0 @@ - - - -one_to_n_results = [{'SERVICE_TERM': 'Covered Services provided under the Medicaid and CHIP Products', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) an amount equivalent to the Medicaid Fee-for-Service Program allowable payment rate as set forth by the State of Nevada ('Medicaid Rate')", 'LOB': 'Medicaid', 'PROGRAM': "Children's Health Insurance Program (CHIP)", 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '100', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services provided under the Medicaid and CHIP Products', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) an amount equivalent to the Medicaid Fee-for-Service Program allowable payment rate as set forth by the State of Nevada ('Medicaid Rate')", 'LOB': 'Medicaid', 'PROGRAM': "Children's Health Insurance Program (CHIP)", 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Fee Schedule', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'Medicaid Fee-for-Service Program allowable payment rate', 'AARETE_DERIVED_FEE_SCHEDULE': 'State', 'FEE_SCHEDULE_VERSION': 'in effect for the Date of Service', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'current'}, {'SERVICE_TERM': 'Covered Services where there is no Medicaid Rate under Medicaid and CHIP Products', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Medicaid', 'PROGRAM': "Children's Health Insurance Program (CHIP)", 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 30, 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services where there is no Medicaid Rate under Medicaid and CHIP Products', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Medicaid', 'PROGRAM': "Children's Health Insurance Program (CHIP)", 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Flat Rate', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 75.0, 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services provided under the Medicare Advantage Product', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) an amount equivalent to the Medicare Fee-for-Service Program allowable payment rate, geographically adjusted ('Medicare Rate')", 'LOB': 'N/A', 'PROGRAM': 'Medicare Advantage', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 100, 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services provided under the Medicare Advantage Product', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) an amount equivalent to the Medicare Fee-for-Service Program allowable payment rate, geographically adjusted ('Medicare Rate')", 'LOB': 'N/A', 'PROGRAM': 'Medicare Advantage', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Fee Schedule', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'Medicare Fee-for-Service Program allowable payment rate', 'AARETE_DERIVED_FEE_SCHEDULE': 'CMS', 'FEE_SCHEDULE_VERSION': 'in effect for the Date of Service', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'current'}, {'SERVICE_TERM': 'Covered Services where there is no Medicare Rate under Medicare Advantage Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'N/A', 'PROGRAM': 'Medicare Advantage', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 30, 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services where there is no Medicare Rate under Medicare Advantage Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'N/A', 'PROGRAM': 'Medicare Advantage', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Flat Rate', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 75.0, 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services which are covered by or are primary to Medicare under the Medicare-Medicaid Program Product', 'REIMB_TERM': "lesser of Provider's billed charges or an amount equivalent to the Medicare Fee-for-Service Program allowable payment rate, geographically adjusted, ('Medicare Rate') and any portion, if any, that the Medicaid agency or Health Plan would have been responsible for paying if the Member was enrolled in the Medicare Fee-for-Service Program", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '100', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services which are covered by or are primary to Medicare under the Medicare-Medicaid Program Product', 'REIMB_TERM': "lesser of Provider's billed charges or an amount equivalent to the Medicare Fee-for-Service Program allowable payment rate, geographically adjusted, ('Medicare Rate') and any portion, if any, that the Medicaid agency or Health Plan would have been responsible for paying if the Member was enrolled in the Medicare Fee-for-Service Program", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Fee Schedule', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'Medicare Fee-for-Service Program allowable payment rate', 'AARETE_DERIVED_FEE_SCHEDULE': 'CMS', 'FEE_SCHEDULE_VERSION': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'current'}, {'SERVICE_TERM': 'Covered Services which are covered by or are primary to Medicaid under the Medicare-Medicaid Program Product', 'REIMB_TERM': "lesser of Provider's billed charges or an amount equivalent to the Medicaid Fee-for-Service Program allowable payment rate as set forth by the State of Nevada ('Medicaid Rate')", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '100', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services which are covered by or are primary to Medicaid under the Medicare-Medicaid Program Product', 'REIMB_TERM': "lesser of Provider's billed charges or an amount equivalent to the Medicaid Fee-for-Service Program allowable payment rate as set forth by the State of Nevada ('Medicaid Rate')", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'PAYMENT_CARVEOUT', 'CARVEOUT_IND': 'Y', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Fee Schedule', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'Medicaid Fee-for-Service Program', 'AARETE_DERIVED_FEE_SCHEDULE': 'State', 'FEE_SCHEDULE_VERSION': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'current'}, {'SERVICE_TERM': 'Covered Services where there is no Medicaid Rate or Medicare Rate under Medicare-Medicaid Program Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'DEFAULT_TERM_TERMS': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '30', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'DISCOUNT_PERCENT_RATE': 'N/A', 'DISCOUNT_RATE_CHANGE_INTERVAL': 'N/A', 'DISCOUNT_START_DT': 'N/A', 'DISCOUNT_END_DT': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services where there is no Medicaid Rate or Medicare Rate under Medicare-Medicaid Program Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Medicaid', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'DEFAULT_TERM_TERMS': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Flat Rate', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': '75.00', 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'DISCOUNT_PERCENT_RATE': 'N/A', 'DISCOUNT_RATE_CHANGE_INTERVAL': 'N/A', 'DISCOUNT_START_DT': 'N/A', 'DISCOUNT_END_DT': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services provided under the Health Insurance Marketplace Product', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) one hundred percent (100%) of the 2021 Medicare fee schedule, which does not include geographic adjustment ('Marketplace Medicare Rate')", 'LOB': 'Marketplace', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '100', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services provided under the Health Insurance Marketplace Product', 'REIMB_TERM': "lesser of the following amounts in effect for the Date of Service: (i) Provider's billed charges; or (ii) one hundred percent (100%) of the 2021 Medicare fee schedule, which does not include geographic adjustment ('Marketplace Medicare Rate')", 'LOB': 'Marketplace', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'BASE_COVERED_SERVICES', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'N', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Fee Schedule', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': '100', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': '2021 Medicare fee schedule', 'AARETE_DERIVED_FEE_SCHEDULE': 'CMS', 'FEE_SCHEDULE_VERSION': '2021', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': '2021'}, {'SERVICE_TERM': 'Covered Services where there is no Marketplace Medicare Rate under Health Insurance Marketplace Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Marketplace', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Billed Charges', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 'N/A', 'REIMB_PCT_RATE': 30, 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}, {'SERVICE_TERM': 'Covered Services where there is no Marketplace Medicare Rate under Health Insurance Marketplace Product', 'REIMB_TERM': "thirty percent (30%) of Provider's billed charges, not to exceed seventy-five dollars ($75.00)", 'LOB': 'Marketplace', 'PROGRAM': 'N/A', 'PLACE_OF_SERVICE_CD_DESC': 'N/A', 'BILL_TYPE_CD_DESC': 'N/A', 'CARVEOUT_CD': 'DEFAULT_TERM', 'CARVEOUT_IND': 'N', 'DEFAULT_IND': 'Y', 'LESSER_OF_IND': 'Y', 'AARETE_DERIVED_REIMB_METHOD': 'Flat Rate', 'UNIT_OF_MEASURE': 'N/A', 'REIMB_FEE_RATE': 75.0, 'REIMB_PCT_RATE': 'N/A', 'REIMB_CONVERSION_FACTOR': 'N/A', 'FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE': 'N/A', 'AARETE_DERIVED_FEE_SCHEDULE_VERSION': 'N/A'}] - -import src.investment.aarete_derived as aarete_derived -import src.investment.postprocessing_funcs as postprocessing_funcs -import src.investment.dynamic_funcs as dynamic_funcs -import src.investment.one_to_n_funcs as one_to_n_funcs -import src.investment.file_processing as file_processing -import src.utils.io_utils as io_utils -from constants.constants import Constants -import src.investment.preprocess as preprocess -import src.investment.row_funcs as row_funcs -filename = "" -import pandas as pd - - -################## Crosswalk Fields ################## -one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results) - -################## Fill NA Mapping ################## -one_to_n_results = aarete_derived.fill_na_mapping(one_to_n_results) - -################## Update LOB for Duals ################## -one_to_n_results = postprocessing_funcs.update_lob_for_duals(one_to_n_results) - -################## Add N/A Dynamic or Exhibit to One-to-One ################## -dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields(one_to_n_results) -print(dynamic_one_to_one_fields) - -################### Split REIMB_DATES ################## -one_to_n_results = one_to_n_funcs.split_reimb_dates(one_to_n_results, filename) - -################## CONVERT TO DF ################## -one_to_n_df = pd.DataFrame(one_to_n_results) - -input_dict = io_utils.read_input() - -constants = Constants() - -# Get contract text from the input dictionary -contract_text = list(input_dict.values())[0] - -contract_text = preprocess.clean_text(contract_text) -text_dict, top_sheet_dict = preprocess.split_text(contract_text) - -one_to_one_results = file_processing.run_one_to_one_prompts( - filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields, constants - ) # Return dict -one_to_one_results["FILE_NAME"] = filename - - -merged_results = row_funcs.merge_one_to_one_into_one_to_n( - one_to_n_df, one_to_one_results, constants -) - -print(merged_results.to_csv("test_results.csv")) - diff --git a/fieldExtraction/src/prompts/fieldset.py b/fieldExtraction/src/prompts/fieldset.py index 6bc35af..1601980 100644 --- a/fieldExtraction/src/prompts/fieldset.py +++ b/fieldExtraction/src/prompts/fieldset.py @@ -1,8 +1,8 @@ - import json from constants.constants import Constants import src.utils.string_utils as string_utils + class Field: def __init__(self, field_dict): # Base Attributes @@ -94,17 +94,17 @@ class Field: f"Regex Pattern: {'None specified' if not self.regex_pattern else self.regex_pattern}" ) - def get_prompt_dict(self, constants: Constants=None): + def get_prompt_dict(self, constants: Constants = None): """Returns a dictionary with the field name as the key and its prompt as the value.""" return {self.field_name: self.get_prompt(constants)} - def get_prompt(self, constants: Constants=None): + def get_prompt(self, constants: Constants = None): """Resolves placeholders in the prompt using the valid_values property.""" # If no valid_values to resolve - if string_utils.is_empty(self.valid_values): # + if string_utils.is_empty(self.valid_values): # return self.prompt - + if not string_utils.is_empty(self.valid_values) and not constants: raise AttributeError( "Must pass Constants object to resolve placeholders for fields with valid_values" @@ -112,7 +112,7 @@ class Field: resolved_prompt = self.prompt if "{valid_values}" in resolved_prompt: - resolved_value = self.valid_values + resolved_value = self.valid_values if hasattr(constants, resolved_value): try: resolved_prompt = resolved_prompt.replace( @@ -188,7 +188,9 @@ class FieldSet: if field.field_name == field_name: setattr(field, attr, value) - def get_prompt_dict(self, constants:Constants=None, field_type=None, relationship=None): + def get_prompt_dict( + self, constants: Constants = None, field_type=None, relationship=None + ): return { field.field_name: field.get_prompt(constants) for field in self.fields @@ -196,9 +198,12 @@ class FieldSet: and (relationship is None or field.relationship == relationship) } - def print_prompt_dict(self, constants: Constants=None): + def print_prompt_dict(self, constants: Constants = None): return "\n".join( - [f"{field.field_name} : {field.get_prompt(constants)}" for field in self.fields] + [ + f"{field.field_name} : {field.get_prompt(constants)}" + for field in self.fields + ] ) def print(self): diff --git a/fieldExtraction/src/prompts/prompt_templates.py b/fieldExtraction/src/prompts/prompt_templates.py index 9399e7f..83438b3 100644 --- a/fieldExtraction/src/prompts/prompt_templates.py +++ b/fieldExtraction/src/prompts/prompt_templates.py @@ -213,7 +213,7 @@ Explain each answer before putting the final answer in the JSON dictionary forma def DYNAMIC_PRIMARY(context, field_name, field_prompt): - return f"""You are an information extractor. Your job is to extract attributes **strictly and literally** from the provided text, paying close attention to detail. + return f"""Extract attributes from the provided text, paying close attention to detail. [ATTRIBUTE TO EXTRACT] Here is the attribute, and instructions on how to correctly answer: @@ -222,8 +222,8 @@ Here is the attribute, and instructions on how to correctly answer: [GENERAL INSTRUCTIONS] - Return all valid, explicitly stated values. If multiple values are found, list them separated by commas. - If none of the valid values appear explicitly in the text, return N/A. -- Never guess. Never assume. Only return values that are a direct, word-for-word match from the valid list. -- If a value appears in the text but is not in the list of valid values, ignore it. +- Make appropriate and obvious assumptions, but don't take huge leaps of logic. + - For example, you can assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "CHIP Perinate", but do not assume that "Children's Health Insurance Program Perinate (CHIP-P)" is equivalent to "Medicaid". Here is the text to analyze: diff --git a/fieldExtraction/src/utils/qa_qc_utils.py b/fieldExtraction/src/utils/qa_qc_utils.py index af1b32f..62fe8ae 100644 --- a/fieldExtraction/src/utils/qa_qc_utils.py +++ b/fieldExtraction/src/utils/qa_qc_utils.py @@ -119,7 +119,6 @@ def yn_check(yn: str) -> tuple[str, bool]: return yn, (yn == "Y" or yn == "N") - def pages_pagenum_check(pages: int, pagenum: int) -> bool: if not isinstance(pages, int):