Merged in feature/dynamic-one-to-one (pull request #461)
Feature/dynamic one to one * Pass N/A to one-to-one * E2E test passed * Merged main into feature/dynamic-one-to-one * Pass ANY non N/A answer to reimbursement-level * Update test * Remove test * Merge branch 'main' into feature/dynamic-one-to-one * Wrap clean_dates * Revert postprocess * Merged main into feature/dynamic-one-to-one * Merged main into feature/dynamic-one-to-one * Remove prints Approved-by: Alex Galarce
This commit is contained in:
Katon Minhas
committed by
Alex Galarce
Alex Galarce
parent
273d124d09
commit
a9da72871e
@@ -1,7 +1,8 @@
|
||||
import csv
|
||||
import json
|
||||
|
||||
import pandas as pd
|
||||
import ast
|
||||
|
||||
from src.config import STATE
|
||||
|
||||
|
||||
@@ -177,8 +178,10 @@ def apply_crosswalk(
|
||||
Returns:
|
||||
str: Crosswalked value
|
||||
"""
|
||||
if "," in val:
|
||||
values = val.split(",") + [val] # Split by comma if present
|
||||
if "[" in val and "]" in val:
|
||||
values = ast.literal_eval(val)
|
||||
elif "," in val:
|
||||
values = val.split(",") # Split by comma if present
|
||||
else:
|
||||
values = [val]
|
||||
mapped_values = [mapping.get(v.strip(), default if default is not None else v.strip()) for v in values]
|
||||
|
||||
@@ -92,12 +92,13 @@ VALID_REIMB_TERM = [
|
||||
VALID_CARVEOUTS = {
|
||||
"DEFAULT_TERM" : "Describes the default payment for when there is no established payment amount on the fee schedule or elsewhere.",
|
||||
"UNLISTED_CODE" : "Describes payments for codes that are not listed on a fee schedule. This is similar to DEFAULT_TERM. Only populate with Y if the Service or Methodology specifically mentions Codes.",
|
||||
"NEVER_EVENT" : "Indicates that Never Events (e.g. sepsis acquired while inpatient) will not be covered.",
|
||||
"NEVER_EVENT" : "Indicates that Never Events (e.g. sepsis acquired while inpatient) will not be covered. This may be phrased as 'no liability' or 'no payment'.",
|
||||
"EXPERIMENTAL_INVESTIGATIONAL" : "Describes payments for experimental and investigational procedures.",
|
||||
"NOT_MEDICALLY_NECESSARY" : "Describes payments for procedures deemed not medically necessary (or medically unnecessary)",
|
||||
"GLOBAL_SERVICES" : None,
|
||||
"BUNDLED_CODES" : "Indicates that payment for bundles of codes shall be at a reduced rate.",
|
||||
"MULTIPLE_PROCEDURE_REDUCTIONS" : 'Describes payments for multiple procedures or surgeries performed on the same day as each other.',
|
||||
#"BILATERAL" : "",
|
||||
"MIDLEVELS" : None,
|
||||
"TECHNICAL_COMPONENT" : "If Modifier TC (technical component) is present.",
|
||||
"PROFESSIONAL_COMPONENT" : "If Modifier 26 (professional component) is present.",
|
||||
@@ -105,6 +106,7 @@ VALID_CARVEOUTS = {
|
||||
"PREADMISSION" : None,
|
||||
"POST_DISCHARGE" : None,
|
||||
"READMISSIONS" : "Describes payment for subsequent admission with the same diagnostic category of codes as the initial admission.",
|
||||
# + Any Reimbursement Carveouts (Codes, Services, etc)
|
||||
}
|
||||
|
||||
VALID_SPECIAL = {
|
||||
@@ -114,7 +116,8 @@ VALID_SPECIAL = {
|
||||
"DISCOUNT" : "Describes a discount to the payment. Look for the term 'Discount'.", # placeholder prompt
|
||||
"PREMIUM" : "Describes a premium to the payment. Look for the term 'Premium'.", # placeholder prompt
|
||||
"ADDITION" : "Describes an increase in the payment amount over time.", # placeholder prompt
|
||||
"TRIGGER_CAP" : "Describes if the payment methodology is different if the amount being paid is over or under a given threshold."# placeholder prompt
|
||||
"TRIGGER_CAP" : "Describes if the payment methodology is different if the amount being paid is over or under a given threshold." # placeholder prompt
|
||||
#"CHARGEMASTER_INCREASE" : ""
|
||||
}
|
||||
|
||||
proc_levels = ["cpt", "hcpcs", "cpt_level3", "cpt_level2", "hcpcs_level2", "cpt_level1", "hcpcs_level1"]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import src.config as config
|
||||
from src.prompts.investment_prompts import Field, FieldSet, aarete_derived_prompt_template
|
||||
from src.prompts.investment_prompts import Field, FieldSet, AARETE_DERIVED_PROMPT_TEMPLATE
|
||||
import src.prompts.investment_prompts as prompts
|
||||
from src.constants import investment_values
|
||||
from src.enums.delimiters import Delimiter
|
||||
@@ -53,7 +53,7 @@ def fill_na_mapping(answer_dicts):
|
||||
|
||||
def get_crosswalk_fields(answer_dicts):
|
||||
crosswalk_fields = FieldSet(file_path=config.FIELD_JSON_PATH, crosswalk=True)
|
||||
|
||||
|
||||
for answer_dict in answer_dicts:
|
||||
for to_field in crosswalk_fields.fields:
|
||||
from_field = to_field.base_field
|
||||
@@ -64,9 +64,6 @@ def get_crosswalk_fields(answer_dicts):
|
||||
answer_dict[to_field.field_name] = crosswalked_answer
|
||||
return answer_dicts
|
||||
|
||||
|
||||
|
||||
|
||||
def get_aarete_derived(aarete_derived_fields: FieldSet,
|
||||
contract_answers_df: pd.DataFrame,
|
||||
filename: str) -> pd.DataFrame:
|
||||
@@ -121,7 +118,7 @@ def get_aarete_derived(aarete_derived_fields: FieldSet,
|
||||
continue
|
||||
|
||||
# Generate prompt for unique value
|
||||
field_prompt = aarete_derived_prompt_template(unique_value, valid_list)
|
||||
field_prompt = AARETE_DERIVED_PROMPT_TEMPLATE(unique_value, valid_list)
|
||||
|
||||
# Invoke Claude and extract answer
|
||||
field_answer_raw = llm_utils.invoke_claude(
|
||||
|
||||
@@ -6,6 +6,7 @@ import src.utils.string_utils as string_utils
|
||||
from src import config
|
||||
import src.investment.one_to_n_funcs as one_to_n_funcs
|
||||
|
||||
|
||||
def prompt_dynamic(text, field_prompts, filename):
|
||||
"""
|
||||
Prompts an LLM to process dynamic fields in a given text and extracts structured responses.
|
||||
@@ -53,16 +54,12 @@ def process_dynamic(results,
|
||||
exhibit_level_answer_dict[field_name] = 'N/A'
|
||||
else:
|
||||
continue
|
||||
# if N answers
|
||||
elif ',' in answer:
|
||||
# if 1 OR N answers
|
||||
else:
|
||||
field = dynamic_fields.get_field(field_name)
|
||||
field.update_valid_values(answer)
|
||||
reimbursement_level_fields.add_field(dynamic_fields.get_field(field_name))
|
||||
dynamic_fields.remove_field(field_name)
|
||||
# if 1 answer
|
||||
else:
|
||||
dynamic_fields.remove_field(field_name)
|
||||
exhibit_level_answer_dict[field_name] = answer
|
||||
return exhibit_level_answer_dict
|
||||
|
||||
|
||||
@@ -103,3 +100,22 @@ def dynamic_and_exhibit_level(exhibit_text,
|
||||
# exhibit_level_answer_dict should have all exhibit-level fields answers (including the ones that were previously dynamic)
|
||||
# reimbursement_value_dict contains the found values for the dynamic fields that got moved to reimbursement-level
|
||||
return exhibit_level_answer_dict
|
||||
|
||||
|
||||
def get_dynamic_one_to_one_fields(answer_dicts):
|
||||
"""
|
||||
Returns a FieldSet object of dynamic fields that have are empty for at least one dict in answer_dicts
|
||||
"""
|
||||
fieldset = FieldSet(config.FIELD_JSON_PATH, relationship="one_to_n", base_field=True)
|
||||
one_to_one_fields = FieldSet()
|
||||
for field in fieldset.fields:
|
||||
field_name = field.field_name
|
||||
if "REIMB" not in field_name: # Don't do this for the REIMB_ fields
|
||||
for answer_dict in answer_dicts:
|
||||
if field_name in answer_dict and string_utils.is_empty(answer_dict[field_name]):
|
||||
field_to_add = Field.load_from_file(file_path=config.FIELD_JSON_PATH, field_name=field.base_field)
|
||||
field_to_add.field_type = "full_context"
|
||||
one_to_one_fields.add_field(field_to_add)
|
||||
continue
|
||||
return one_to_one_fields
|
||||
|
||||
|
||||
@@ -18,6 +18,32 @@ from src.config import FIELD_JSON_PATH
|
||||
from src.investment.one_to_n_funcs import reimbursement_level, combine_one_to_n_answers
|
||||
from src.investment.dynamic_funcs import dynamic_and_exhibit_level
|
||||
from src.investment.tin_npi_funcs import reimbursement_tin_npi
|
||||
import src.investment.dynamic_funcs as dynamic_funcs
|
||||
|
||||
################## MERGE ##################
|
||||
def merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results):
|
||||
"""
|
||||
Merges one_to_one_results into one_to_n_results.
|
||||
|
||||
Args:
|
||||
one_to_n_results (pd.DataFrame): DataFrame with N rows and M columns, including FILE_NAME.
|
||||
one_to_one_results (dict): Dictionary with several keys, including FILE_NAME.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: Updated one_to_n_results with merged values from one_to_one_results.
|
||||
"""
|
||||
# Iterate over all key-value pairs in one_to_one_results
|
||||
for k, v in one_to_one_results.items():
|
||||
if k not in one_to_n_results.columns:
|
||||
# If k is not a column in one_to_n_results, create it and populate all rows with v
|
||||
one_to_n_results[k] = v
|
||||
else:
|
||||
# If k is a column in one_to_n_results, replace empty values with v
|
||||
one_to_n_results[k] = one_to_n_results[k].apply(
|
||||
lambda row_val: v if string_utils.is_empty(row_val) else row_val
|
||||
)
|
||||
return one_to_n_results
|
||||
|
||||
|
||||
def process_file(file_object, all_dataset, run_timestamp):
|
||||
filename, contract_text = file_object
|
||||
@@ -32,22 +58,22 @@ def process_file(file_object, all_dataset, run_timestamp):
|
||||
|
||||
################## ONE TO N ##################
|
||||
if string_utils.contains_reimbursement(contract_text):
|
||||
one_to_n_results = run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset) # Return df
|
||||
one_to_n_results, dynamic_one_to_one_fields = run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset) # Return df
|
||||
one_to_n_results['FILE_NAME'] = filename
|
||||
print(f"{datetime_str()} One to N Complete - {filename}")
|
||||
else:
|
||||
one_to_n_results = pd.DataFrame([{'FILE_NAME' : filename}])
|
||||
dynamic_one_to_one_fields = FieldSet()
|
||||
print(f"{datetime_str()} No One to N Found, Skipping - {filename}")
|
||||
|
||||
################## ONE TO ONE ##################
|
||||
one_to_one_results = run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict) # Return df
|
||||
one_to_one_results = run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields) # Return df
|
||||
one_to_one_results['FILE_NAME'] = filename
|
||||
print(f"{datetime_str()} One to One Complete - {filename}")
|
||||
# one_to_one_results = {"FILE_NAME" : filename}
|
||||
|
||||
################## MERGE ##################
|
||||
final_results = pd.merge(one_to_one_results, one_to_n_results, how='outer', on='FILE_NAME')
|
||||
|
||||
final_results = merge_one_to_one_into_one_to_n(one_to_n_results, one_to_one_results)
|
||||
|
||||
################## POSTPROCESS ##################
|
||||
final_df = postprocess.postprocess(final_results)
|
||||
print(f"{datetime_str()} Postprocessing Complete - {filename}")
|
||||
@@ -62,10 +88,10 @@ def process_file(file_object, all_dataset, run_timestamp):
|
||||
return final_df
|
||||
|
||||
|
||||
def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict):
|
||||
def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict, dynamic_one_to_one_fields):
|
||||
|
||||
################## INITIALIZE FIELDS ##################
|
||||
one_to_one_fields = FieldSet(relationship="one_to_one", file_path=config.FIELD_JSON_PATH)
|
||||
one_to_one_fields = FieldSet(relationship="one_to_one", file_path=config.FIELD_JSON_PATH).combine(dynamic_one_to_one_fields)
|
||||
|
||||
################## RUN REGEX ##################
|
||||
regex_answers_dict = one_to_one_funcs.run_regex_fields(contract_text, text_dict, filename)
|
||||
@@ -81,12 +107,15 @@ def run_one_to_one_prompts(filename, contract_text, text_dict, top_sheet_dict):
|
||||
)
|
||||
|
||||
################## COMBINE ANSWERS ################
|
||||
final_answers_dict = {**regex_answers_dict, **smart_chunked_answers_dict, **full_context_answers_dict}
|
||||
one_to_one_results = {**regex_answers_dict, **smart_chunked_answers_dict, **full_context_answers_dict}
|
||||
|
||||
################## CONVERT TO DF ##################
|
||||
final_df = pd.DataFrame([final_answers_dict])
|
||||
################## Crosswalk Fields ##################
|
||||
one_to_one_results = aarete_derived.get_crosswalk_fields([one_to_one_results])
|
||||
|
||||
return final_df
|
||||
################## Fill NA Mapping ##################
|
||||
one_to_one_results = aarete_derived.fill_na_mapping(one_to_one_results)
|
||||
|
||||
return one_to_one_results[0]
|
||||
|
||||
|
||||
def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mapping, all_dataset):
|
||||
@@ -117,15 +146,19 @@ def run_one_to_n_prompts(filename, text_dict, exhibit_pages, exhibit_chunk_mappi
|
||||
full_answer_dict = combine_one_to_n_answers(exhibit_level_answers, reimbursement_level_answers, tin_npi_answers)
|
||||
one_to_n_results += full_answer_dict
|
||||
|
||||
################## Crosswalk Fields ##################
|
||||
################## Crosswalk Fields ##################
|
||||
one_to_n_results = aarete_derived.get_crosswalk_fields(one_to_n_results)
|
||||
|
||||
################## Fill NA Mapping ##################
|
||||
one_to_n_results = aarete_derived.fill_na_mapping(one_to_n_results)
|
||||
|
||||
|
||||
################## Add N/A Dynamic or Exhibit to One-to-One ##################
|
||||
dynamic_one_to_one_fields = dynamic_funcs.get_dynamic_one_to_one_fields(one_to_n_results)
|
||||
|
||||
################## CONVERT TO DF ##################
|
||||
one_to_n_df = pd.DataFrame(one_to_n_results)
|
||||
return one_to_n_df
|
||||
|
||||
return one_to_n_df, dynamic_one_to_one_fields
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -349,7 +349,7 @@
|
||||
{
|
||||
"field_name": "PROV_SPECIALTY_CD",
|
||||
"relationship": "one_to_n",
|
||||
"field_type": "dynamic",
|
||||
"field_type": "TBD",
|
||||
"prompt": "List any valid Specialties that apply to the PROVIDER that are mentioned in the text. Choose ONLY from the following: {valid_values}. Do NOT list any of these values if they only apply to a SERVICE, and not explictly to a PROVIDER.",
|
||||
"valid_values": "VALID_PROV_SPECIALTY"
|
||||
},
|
||||
|
||||
@@ -212,6 +212,7 @@ class FieldSet:
|
||||
return
|
||||
print([field.field_name for field in self.fields])
|
||||
|
||||
|
||||
def filter(self, **criteria):
|
||||
"""
|
||||
Filters the fields in the FieldSet based on the provided criteria.
|
||||
@@ -227,9 +228,21 @@ class FieldSet:
|
||||
|
||||
def contains_fields(self):
|
||||
return len(self.fields) > 0
|
||||
|
||||
|
||||
|
||||
|
||||
def combine(self, other, inplace=False):
|
||||
"""
|
||||
Combines the fields from another FieldSet into this one, avoiding duplicates.
|
||||
"""
|
||||
if inplace:
|
||||
for field in other.fields:
|
||||
if field.field_name not in self.list_fields():
|
||||
self.fields.append(field)
|
||||
else:
|
||||
combined_fields = self.fields.copy()
|
||||
for field in other.fields:
|
||||
if field.field_name not in self.list_fields():
|
||||
combined_fields.append(field)
|
||||
return FieldSet(fields=combined_fields)
|
||||
|
||||
|
||||
#####################################################################################
|
||||
@@ -563,7 +576,7 @@ If so, write the name of the Case. If they do not, simply write N/A.
|
||||
{PIPE_FORMAT_INSTRUCTIONS}
|
||||
"""
|
||||
|
||||
def aarete_derived_prompt_template(contract_answer: str, valid_derived_values: list[str]) -> str:
|
||||
def AARETE_DERIVED_PROMPT_TEMPLATE(contract_answer: str, valid_derived_values: list[str]) -> str:
|
||||
"""
|
||||
prompt template for matching contract answers to valid derived values.
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ class TestOneToN(unittest.TestCase):
|
||||
exhibit_level_answer_dict = process_dynamic(results, reimbursement_level_fields, exhibit_level_fields, dynamic_fields, assign_na)
|
||||
|
||||
# You can check the content of exhibit_level_answer_dict based on inputs.
|
||||
self.assertEqual(exhibit_level_answer_dict, {"field1": "value1", "field2": "value2"})
|
||||
self.assertEqual(exhibit_level_answer_dict, {})
|
||||
|
||||
@patch('src.investment.one_to_n_funcs.get_exhibit_header')
|
||||
@patch('src.investment.dynamic_funcs.prompt_dynamic')
|
||||
|
||||
Reference in New Issue
Block a user