diff --git a/fieldExtraction/crosswalk/crosswalk_utils.py b/fieldExtraction/crosswalk/crosswalk_utils.py index 1dc3289..62bdad4 100644 --- a/fieldExtraction/crosswalk/crosswalk_utils.py +++ b/fieldExtraction/crosswalk/crosswalk_utils.py @@ -1,7 +1,8 @@ import csv import json - import pandas as pd +import ast + from src.config import STATE @@ -177,8 +178,10 @@ def apply_crosswalk( Returns: str: Crosswalked value """ - if "," in val: - values = val.split(",") + [val] # Split by comma if present + if "[" in val and "]" in val: + values = ast.literal_eval(val) + elif "," in val: + values = val.split(",") # Split by comma if present else: values = [val] mapped_values = [mapping.get(v.strip(), default if default is not None else v.strip()) for v in values] diff --git a/fieldExtraction/src/constants/investment_values.py b/fieldExtraction/src/constants/investment_values.py index f41fb55..d6d8c33 100644 --- a/fieldExtraction/src/constants/investment_values.py +++ b/fieldExtraction/src/constants/investment_values.py @@ -92,12 +92,13 @@ VALID_REIMB_TERM = [ VALID_CARVEOUTS = { "DEFAULT_TERM" : "Describes the default payment for when there is no established payment amount on the fee schedule or elsewhere.", "UNLISTED_CODE" : "Describes payments for codes that are not listed on a fee schedule. This is similar to DEFAULT_TERM. Only populate with Y if the Service or Methodology specifically mentions Codes.", - "NEVER_EVENT" : "Indicates that Never Events (e.g. sepsis acquired while inpatient) will not be covered.", + "NEVER_EVENT" : "Indicates that Never Events (e.g. sepsis acquired while inpatient) will not be covered. This may be phrased as 'no liability' or 'no payment'.", "EXPERIMENTAL_INVESTIGATIONAL" : "Describes payments for experimental and investigational procedures.", "NOT_MEDICALLY_NECESSARY" : "Describes payments for procedures deemed not medically necessary (or medically unnecessary)", "GLOBAL_SERVICES" : None, "BUNDLED_CODES" : "Indicates that payment for bundles of codes shall be at a reduced rate.", "MULTIPLE_PROCEDURE_REDUCTIONS" : 'Describes payments for multiple procedures or surgeries performed on the same day as each other.', + #"BILATERAL" : "", "MIDLEVELS" : None, "TECHNICAL_COMPONENT" : "If Modifier TC (technical component) is present.", "PROFESSIONAL_COMPONENT" : "If Modifier 26 (professional component) is present.", @@ -105,6 +106,7 @@ VALID_CARVEOUTS = { "PREADMISSION" : None, "POST_DISCHARGE" : None, "READMISSIONS" : "Describes payment for subsequent admission with the same diagnostic category of codes as the initial admission.", + # + Any Reimbursement Carveouts (Codes, Services, etc) } VALID_SPECIAL = { @@ -114,7 +116,8 @@ VALID_SPECIAL = { "DISCOUNT" : "Describes a discount to the payment. Look for the term 'Discount'.", # placeholder prompt "PREMIUM" : "Describes a premium to the payment. Look for the term 'Premium'.", # placeholder prompt "ADDITION" : "Describes an increase in the payment amount over time.", # placeholder prompt - "TRIGGER_CAP" : "Describes if the payment methodology is different if the amount being paid is over or under a given threshold."# placeholder prompt + "TRIGGER_CAP" : "Describes if the payment methodology is different if the amount being paid is over or under a given threshold." # placeholder prompt + #"CHARGEMASTER_INCREASE" : "" } proc_levels = ["cpt", "hcpcs", "cpt_level3", "cpt_level2", "hcpcs_level2", "cpt_level1", "hcpcs_level1"] diff --git a/fieldExtraction/src/investment/aarete_derived.py b/fieldExtraction/src/investment/aarete_derived.py index 3ec5d2b..99d4166 100644 --- a/fieldExtraction/src/investment/aarete_derived.py +++ b/fieldExtraction/src/investment/aarete_derived.py @@ -1,5 +1,5 @@ import src.config as config -from src.prompts.investment_prompts import Field, FieldSet, aarete_derived_prompt_template +from src.prompts.investment_prompts import Field, FieldSet, AARETE_DERIVED_PROMPT_TEMPLATE import src.prompts.investment_prompts as prompts from src.constants import investment_values from src.enums.delimiters import Delimiter @@ -53,7 +53,7 @@ def fill_na_mapping(answer_dicts): def get_crosswalk_fields(answer_dicts): crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True) - + for answer_dict in answer_dicts: for to_field in crosswalk_fields.fields: from_field = to_field.base_field @@ -64,9 +64,6 @@ def get_crosswalk_fields(answer_dicts): answer_dict[to_field.field_name] = crosswalked_answer return answer_dicts - - - def get_aarete_derived(aarete_derived_fields: FieldSet, contract_answers_df: pd.DataFrame, filename: str) -> pd.DataFrame: @@ -121,7 +118,7 @@ def get_aarete_derived(aarete_derived_fields: FieldSet, continue # Generate prompt for unique value - field_prompt = aarete_derived_prompt_template(unique_value, valid_list) + field_prompt = AARETE_DERIVED_PROMPT_TEMPLATE(unique_value, valid_list) # Invoke Claude and extract answer field_answer_raw = llm_utils.invoke_claude( diff --git a/fieldExtraction/src/investment/dynamic_funcs.py b/fieldExtraction/src/investment/dynamic_funcs.py index d89e786..5fb0adb 100644 --- a/fieldExtraction/src/investment/dynamic_funcs.py +++ b/fieldExtraction/src/investment/dynamic_funcs.py @@ -6,6 +6,7 @@ import src.utils.string_utils as string_utils from src import config import src.investment.one_to_n_funcs as one_to_n_funcs + def prompt_dynamic(text, field_prompts, filename): """ Prompts an LLM to process dynamic fields in a given text and extracts structured responses. @@ -53,16 +54,12 @@ def process_dynamic(results, exhibit_level_answer_dict[field_name] = 'N/A' else: continue - # if N answers - elif ',' in answer: + # if 1 OR N answers + else: field = dynamic_fields.get_field(field_name) field.update_valid_values(answer) reimbursement_level_fields.add_field(dynamic_fields.get_field(field_name)) dynamic_fields.remove_field(field_name) - # if 1 answer - else: - dynamic_fields.remove_field(field_name) - exhibit_level_answer_dict[field_name] = answer return exhibit_level_answer_dict @@ -103,3 +100,22 @@ def dynamic_and_exhibit_level(exhibit_text, # exhibit_level_answer_dict should have all exhibit-level fields answers (including the ones that were previously dynamic) # reimbursement_value_dict contains the found values for the dynamic fields that got moved to reimbursement-level return exhibit_level_answer_dict + + +def get_dynamic_one_to_one_fields(answer_dicts): + """ + Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts + """ + fieldset = FieldSet(config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True) + one_to_one_fields = FieldSet() + for field in fieldset.fields: + field_name = field.field_name + if "REIMB" not in field_name: # Don't do this for the REIMB_ fields + for answer_dict in answer_dicts: + if field_name in answer_dict and string_utils.is_empty(answer_dict[field_name]): + field_to_add = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name=field.base_field) + field_to_add.field_type = "full_context" + one_to_one_fields.add_field(field_to_add) + continue + return one_to_one_fields + diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index b8cecef..62cfda0 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -18,6 +18,32 @@ from src.config import FIELD_JSON_PATH from src.investment.one_to_n_funcs import reimbursement_level, combine_one_to_n_answers from src.investment.dynamic_funcs import dynamic_and_exhibit_level from src.investment.tin_npi_funcs import reimbursement_tin_npi +import src.investment.dynamic_funcs as dynamic_funcs + +################## MERGE ################## +def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results): + """ + Merges one_to_one_results into one_to_n_results. + + Args: + one_to_n_results (pd.DataFrame): DataFrame with N rows and M columns, including FILE_NAME. + one_to_one_results (dict): Dictionary with several keys, including FILE_NAME. + + Returns: + pd.DataFrame: Updated one_to_n_results with merged values from one_to_one_results. + """ + # Iterate over all key-value pairs in one_to_one_results + for k, v in one_to_one_results.items(): + if k not in one_to_n_results.columns: + # If k is not a column in one_to_n_results, create it and populate all rows with v + one_to_n_results[k] = v + else: + # If k is a column in one_to_n_results, replace empty values with v + one_to_n_results[k] = one_to_n_results[k].apply( + lambda row_val: v if string_utils.is_empty(row_val) else row_val + ) + return one_to_n_results + def process_file(file_object, all_dataset, run_timestamp): filename, contract_text = file_object @@ -32,22 +58,22 @@ def process_file(file_object, all_dataset, run_timestamp): ################## ONE TO N ################## if string_utils.contains_reimbursement(contract_text): - one_to_n_results = run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset) # Return df + one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset) # Return df one_to_n_results['FILE_NAME'] = filename print(f"{datetime_str()} One to N Complete - {filename}") else: one_to_n_results = pd.DataFrame([{'FILE_NAME' : filename}]) + dynamic_one_to_one_fields = FieldSet() print(f"{datetime_str()} No One to N Found, Skipping - {filename}") ################## ONE TO ONE ################## - one_to_one_results = run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict) # Return df + one_to_one_results = run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields) # Return df one_to_one_results['FILE_NAME'] = filename print(f"{datetime_str()} One to One Complete - {filename}") - # one_to_one_results = {"FILE_NAME" : filename} ################## MERGE ################## - final_results = pd.merge(one_to_one_results, one_to_n_results, how='outer', on='FILE_NAME') - + final_results = merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results) + ################## POSTPROCESS ################## final_df = postprocess.postprocess(final_results) print(f"{datetime_str()} Postprocessing Complete - {filename}") @@ -62,10 +88,10 @@ def process_file(file_object, all_dataset, run_timestamp): return final_df -def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict): +def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields): ################## INITIALIZE FIELDS ################## - one_to_one_fields = FieldSet(relationship="one_to_one", file_path=config.FIELD_JSON_PATH) + one_to_one_fields = FieldSet(relationship="one_to_one", file_path=config.FIELD_JSON_PATH).combine(dynamic_one_to_one_fields) ################## RUN REGEX ################## regex_answers_dict = one_to_one_funcs.run_regex_fields(contract_text, text_dict, filename) @@ -81,12 +107,15 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict): ) ################## COMBINE ANSWERS ################ - final_answers_dict = {**regex_answers_dict, **smart_chunked_answers_dict, **full_context_answers_dict} + one_to_one_results = {**regex_answers_dict, **smart_chunked_answers_dict, **full_context_answers_dict} - ################## CONVERT TO DF ################## - final_df = pd.DataFrame([final_answers_dict]) + ################## Crosswalk Fields ################## + one_to_one_results = aarete_derived.get_crosswalk_fields([one_to_one_results]) - return final_df + ################## Fill NA Mapping ################## + one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results) + + return one_to_one_results[0] def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset): @@ -117,15 +146,19 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers) one_to_n_results += full_answer_dict - ################## Crosswalk Fields ################## + ################## Crosswalk Fields ################## one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results) ################## Fill NA Mapping ################## one_to_n_results = aarete_derived.fill_na_mapping(one_to_n_results) - + + ################## Add N/A Dynamic or Exhibit to One-to-One ################## + dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields(one_to_n_results) + ################## CONVERT TO DF ################## one_to_n_df = pd.DataFrame(one_to_n_results) - return one_to_n_df + + return one_to_n_df, dynamic_one_to_one_fields diff --git a/fieldExtraction/src/prompts/investment_prompts.json b/fieldExtraction/src/prompts/investment_prompts.json index 9a0d011..032388e 100644 --- a/fieldExtraction/src/prompts/investment_prompts.json +++ b/fieldExtraction/src/prompts/investment_prompts.json @@ -349,7 +349,7 @@ { "field_name": "PROV_SPECIALTY_CD", "relationship": "one_to_n", - "field_type": "dynamic", + "field_type": "TBD", "prompt": "List any valid Specialties that apply to the PROVIDER that are mentioned in the text. Choose ONLY from the following: {valid_values}. Do NOT list any of these values if they only apply to a SERVICE, and not explictly to a PROVIDER.", "valid_values": "VALID_PROV_SPECIALTY" }, diff --git a/fieldExtraction/src/prompts/investment_prompts.py b/fieldExtraction/src/prompts/investment_prompts.py index 665eb8e..b307ea0 100644 --- a/fieldExtraction/src/prompts/investment_prompts.py +++ b/fieldExtraction/src/prompts/investment_prompts.py @@ -212,6 +212,7 @@ class FieldSet: return print([field.field_name for field in self.fields]) + def filter(self, **criteria): """ Filters the fields in the FieldSet based on the provided criteria. @@ -227,9 +228,21 @@ class FieldSet: def contains_fields(self): return len(self.fields) > 0 - - - + + def combine(self, other, inplace=False): + """ + Combines the fields from another FieldSet into this one, avoiding duplicates. + """ + if inplace: + for field in other.fields: + if field.field_name not in self.list_fields(): + self.fields.append(field) + else: + combined_fields = self.fields.copy() + for field in other.fields: + if field.field_name not in self.list_fields(): + combined_fields.append(field) + return FieldSet(fields=combined_fields) ##################################################################################### @@ -563,7 +576,7 @@ If so, write the name of the Case. If they do not, simply write N/A. {PIPE_FORMAT_INSTRUCTIONS} """ -def aarete_derived_prompt_template(contract_answer: str, valid_derived_values: list[str]) -> str: +def AARETE_DERIVED_PROMPT_TEMPLATE(contract_answer: str, valid_derived_values: list[str]) -> str: """ prompt template for matching contract answers to valid derived values. diff --git a/fieldExtraction/tests/one_to_n_test.py b/fieldExtraction/tests/one_to_n_test.py index b6231bd..c380666 100644 --- a/fieldExtraction/tests/one_to_n_test.py +++ b/fieldExtraction/tests/one_to_n_test.py @@ -15,7 +15,7 @@ class TestOneToN(unittest.TestCase): exhibit_level_answer_dict = process_dynamic(results, reimbursement_level_fields, exhibit_level_fields, dynamic_fields, assign_na) # You can check the content of exhibit_level_answer_dict based on inputs. - self.assertEqual(exhibit_level_answer_dict, {"field1": "value1", "field2": "value2"}) + self.assertEqual(exhibit_level_answer_dict, {}) @patch('src.investment.one_to_n_funcs.get_exhibit_header') @patch('src.investment.dynamic_funcs.prompt_dynamic')