jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Merged in bugfix/lob-fixes (pull request #692)

Browse Source 
Bugfix/lob fixes

* Refactor update_lob_for_duals and update_grouper_base_rate_and_grouper_pct_rate functions for improved clarity and logic; add handling for 'neither' and 'n/a' cases.

* Change logging level from info to debug for various functions to reduce log verbosity.


Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
2025-09-03 20:29:43 +00:00
parent 95806f5782
commit 0320d6633a
3 changed files with 44 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
fieldExtraction/src/investment/one_to_n_funcs.py
+9 -9
View File
@@ -822,7 +822,7 @@ def deduplicate_with_accumulator(
)
if duplicates_found > 0:
logging.info(
logging.debug(
f"Removed {duplicates_found} duplicate pairs from exhibit {exhibit_page} (base page {base_page}) in {filename}"
)
@@ -864,7 +864,7 @@ def filter_services_without_reimbursements(
filtered_count = len(answers) - len(valid_reimbursements)
if filtered_count > 0:
logging.info(
logging.debug(
f"Filtered {filtered_count} services without clear reimbursement methodologies in {filename}."
)
@@ -939,13 +939,13 @@ def split_compound_reimbursements(
split_result = split_compound_reimbursement_llm(answer_dict, filename)
if split_result and len(split_result) > 1:
logging.info(
logging.debug(
f"Split compound reimbursement in {filename}: "
f"{answer_dict['SERVICE_TERM']} - {answer_dict['REIMB_TERM']}"
f" into {len(split_result)} parts."
)
logging.info(f"Split results: {split_result}")
logging.debug(f"Split results: {split_result}")
split_answers.extend(split_result)
else: # If LLM didn't split, keep the original
@@ -1176,7 +1176,7 @@ def split_reimb_dates(one_to_n_results: list, filename: str) -> list:
"""
if not one_to_n_results or not isinstance(one_to_n_results, list):
logging.info("No valid data to process")
logging.debug("`split_reimb_dates`: No valid data to process")
return one_to_n_results
# Filter records that have non-empty REIMB_DATES
@@ -1195,7 +1195,7 @@ def split_reimb_dates(one_to_n_results: list, filename: str) -> list:
records_to_process.append((i, record))
if not records_to_process:
logging.info("No valid date ranges to process")
logging.debug("`split_reimb_dates`: No valid date ranges to process")
# Still remove the REIMB_DATES field from all records
for record in one_to_n_results:
if "REIMB_DATES" in record:
@@ -1261,17 +1261,17 @@ def process_special_cases(
case = case.upper()
if case == "TRIGGER_CAP":
logging.info(f"Running TRIGGER_CAP breakout for {service} - {methodology}")
logging.debug(f"Running TRIGGER_CAP breakout for {service} - {methodology}")
trigger_cap_results = run_trigger_cap_breakout(
service, methodology, filename
)
special_case_results.update(trigger_cap_results)
elif case == "OUTLIER":
logging.info(f"Running OUTLIERS breakout for {service} - {methodology}")
logging.debug(f"Running OUTLIERS breakout for {service} - {methodology}")
outlier_results = run_outliers_breakout(service, methodology, filename)
special_case_results.update(outlier_results)
elif case == "GROUPER":
logging.info(f"Running GROUPER breakout for {service} - {methodology}")
logging.debug(f"Running GROUPER breakout for {service} - {methodology}")
grouper_results = run_grouper_breakout(service, methodology, filename)
special_case_results.update(grouper_results)
else: # Other/future special cases can go here
fieldExtraction/src/investment/postprocessing_funcs.py
+29 -22
View File
@@ -598,7 +598,13 @@ def update_lob_for_duals(answer_dicts):
claude_answer_extracted = string_utils.extract_text_from_delimiters(
claude_answer_raw, Delimiter.PIPE
)
answer_dict["AARETE_DERIVED_LOB"] = claude_answer_extracted
if (
claude_answer_extracted.lower() == "neither"
or claude_answer_extracted.lower() == "n/a"
): # If neither, take original AARETE_DERIVED_LOB
pass
else:
answer_dict["AARETE_DERIVED_LOB"] = claude_answer_extracted
final_answer_dicts.append(answer_dict)
return final_answer_dicts
@@ -743,50 +749,51 @@ def deduplicate_provider_columns(df: pd.DataFrame) -> pd.DataFrame:
)
return df
def update_grouper_base_rate_and_grouper_pct_rate(df):
"""
Update GROUPER_PCT_RATE and GROUPER_BASE_RATE columns based on grouper conditions.
Creates new columns and populates them when:
- AARETE_DERIVED_REIMB_METHOD is 'Grouper', OR
- AARETE_DERIVED_REIMB_METHOD is 'Grouper', OR
- GROUPER_TYPE is not empty (using string_utils.is_empty) AND not empty array/empty array string, OR
- GROUPER_CD is not empty (using string_utils.is_empty) AND not empty array/empty array string
Args:
df: DataFrame to update
Returns:
DataFrame with updated GROUPER_PCT_RATE and GROUPER_BASE_RATE columns
"""
# Required columns for the operation
required_cols = [
"AARETE_DERIVED_REIMB_METHOD",
"REIMB_PCT_RATE",
"REIMB_CONVERSION_FACTOR",
"GROUPER_TYPE"
"GROUPER_CD"
"AARETE_DERIVED_REIMB_METHOD",
"REIMB_PCT_RATE",
"REIMB_CONVERSION_FACTOR",
"GROUPER_TYPE" "GROUPER_CD",
]
# Check if all required columns exist
if not all(col in df.columns for col in required_cols):
# Initialize empty columns and return if required columns are missing
df["GROUPER_PCT_RATE"] = ""
df["GROUPER_BASE_RATE"] = ""
return df
# Create grouper condition mask
grouper_mask = (
(df["AARETE_DERIVED_REIMB_METHOD"] == "Grouper") & (
(~string_utils.is_empty(df["GROUPER_TYPE"]) & df["GROUPER_TYPE"].ne("[]")) |
(~string_utils.is_empty(df["GROUPER_CD"]) & df["GROUPER_CD"].ne("[]"))
))
grouper_mask = (df["AARETE_DERIVED_REIMB_METHOD"] == "Grouper") & (
(~string_utils.is_empty(df["GROUPER_TYPE"]) & df["GROUPER_TYPE"].ne("[]"))
| (~string_utils.is_empty(df["GROUPER_CD"]) & df["GROUPER_CD"].ne("[]"))
)
# Initialize new columns with empty strings
df["GROUPER_PCT_RATE"] = ""
df["GROUPER_BASE_RATE"] = ""
# Update values where grouper condition is met
df.loc[grouper_mask, "GROUPER_PCT_RATE"] = df.loc[grouper_mask, "REIMB_PCT_RATE"]
df.loc[grouper_mask, "GROUPER_BASE_RATE"] = df.loc[grouper_mask, "REIMB_CONVERSION_FACTOR"]
return df
df.loc[grouper_mask, "GROUPER_BASE_RATE"] = df.loc[
grouper_mask, "REIMB_CONVERSION_FACTOR"
]
return df
fieldExtraction/src/prompts/prompt_templates.py
+6 -11
View File
@@ -219,6 +219,7 @@ Here is the text to analyze:
Explain each answer before putting the final answer in the JSON dictionary format.
"""
def DYNAMIC_PRIMARY_HEADER(context, field_name, field_prompt):
return f"""Extract attributes from the provided text, paying close attention to detail.
@@ -238,6 +239,7 @@ Here is the text to analyze:
Briefly explain your answer before putting the final answer in |pipes|.
"""
def DYNAMIC_PRIMARY_TEXT(context, field_name, field_prompt):
return f"""Extract attributes from the provided text, paying close attention to detail.
@@ -770,6 +772,7 @@ def DUAL_LOB_CHECK(service_term):
If the Service is only for Medicaid, return 'Medicaid'
If the Service is only for Medicare, return 'Medicare'
If the Service is for both Medicare and Medicaid, return 'Duals'
If the service is for neither Medicare nor Medicaid, return 'neither'
If it is unclear, write 'N/A'.
Here is the Service:
@@ -1118,9 +1121,8 @@ Return your answers in valid JSON format using the exact field names from the qu
**Important:** Base your answers ONLY on the outlier terms provided. Do not make assumptions or add information not present in the contract language.
"""
def GROUPER_BREAKOUT(
service, methodology, questions
):
def GROUPER_BREAKOUT(service, methodology, questions):
"""
Use this prompt for grouper fields that are derived directly from the combination of Service and Methodology
@@ -1151,6 +1153,7 @@ Return your answers in valid JSON format using the exact field names from the qu
}}
"""
def LOB_RELATIONSHIP(exhibit_text, dynamic_primary_values):
return f"""You will be presented with an exhibit from a Payer-Provider contract.
@@ -1185,11 +1188,3 @@ def FULL_CONTEXT_ADDITIONAL_INSTRUCTIONS(allow_na):
return " Only provide an answer if that answer clearly applies to the ENTIRE contract. Otherwise, answer N/A."
else:
return " Do NOT leave this field N/A. Choose the most appropriate answer based on the context."