Katon Minhas
afb6d5185d
Feature/lesser table caching refactor hybrid * chore: Remove unused duplicate main.py from shared pipeline * fix: Correct crosswalk paths in aarete_derived.py * chore: Remove unused documentation files from fieldExtraction * docs: Add documentation files to documentation folder * docs: Update README with uv setup, expanded project structure, and branching conventions * docs: Add uv installation steps with Ubuntu/WSL emphasis * Enable prompt caching for all remaining LLM calls - Add _INSTRUCTION() functions for: EXHIBIT_HEADER, EXHIBIT_LINKAGE, EXHIBIT_TITLE_MATCH, DATE_FIX, DERIVED_TERM_DATE, CHECK_PROVIDER_NAME_MATCH, SPECIAL_CASE_ASSIGNMENT - Update all invoke_claude() calls in saas and clover pipelines to use cache=True with corresponding _INSTRUCTION() functions - Add new instructions to get_cacheable_instructions() for cache warming - Update tests for new instruction functions Functions now using caching: - prompt_exhibit_level - prompt_exhibit_lesser (EXHIBIT_LEVEL_LESSER_OF) - prompt_fee_schedule_breakout - prompt_grouper_breakout - prompt_special_case_assignment - prompt_exhibit_linkage - prompt_exhibit_header - prompt_smart_chunked (ONE_TO_ONE templates) - prompt_date_fix - prompt_derived_term_date - prompt_exhibit_title_match - provider_name_match_check 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Reorder * feat: Add bcbs_promise client pipeline with OFFSET_TERM extraction - Add new bcbs_promise client with HSC-based OFFSET_TERM field extraction - Extract full paragraph text of offset/recoupment provisions from contracts - Derive OFFSET_INDICATOR (Y/N) from OFFSET_TERM presence - Fix reorder_columns to preserve extra columns not in COLUMN_ORDER - Update QC/QA output path to outputs/qc_qa/ * fix: Update dev deps and test assertions for QC/QA output path - Add pytest/pytest-mock to dev dependencies for mypy type checking - Update test assertions to expect outputs/qc_qa instead of qa_qc_output * style: Apply black formatting to prompt_templates.py * Merge main, move scripts * Archive some scripts * update py version * remove .py version file * Remove ASCII characters * Restore testbed code * restore tracking * Update testbed metrics * Enable prompt caching for CODE_LAST_CHECK, FILL_BILL_TYPE, DUAL_LOB_CHECK, and GROUPER_BREAKOUT - Add CODE_LAST_CHECK_INSTRUCTION() for service specificity classification - Add FILL_BILL_TYPE_INSTRUCTION() for bill type code determination - Add DUAL_LOB_CHECK_INSTRUCTION() for Medicare/Medicaid classification - Update code_funcs.py to use caching for CODE_LAST_CHECK, FILL_BILL_TYPE, GROUPER_BREAKOUT - Update postprocessing_funcs.py to use caching for DUAL_LOB_CHECK - Add new instructions to get_cacheable_instructions() for cache warming - Add unit tests for new instruction functions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * Fix postprocessing_funcs to remove invalid columns * Merge branch 'main' into feature/lesser-table-caching-refactor-hybrid * Revert prompt caching changes from aed1b73c * update formatting * Update imports Approved-by: Sha Brown Approved-by: Praneel Panchigar
190 lines
6.3 KiB
Python
190 lines
6.3 KiB
Python
import pandas as pd
|
|
import os
|
|
|
|
import valid
|
|
from postprocess import ac_postprocess
|
|
import postprocessing_funcs
|
|
|
|
"""
|
|
This script consolidates a&c fields. Its data sources are `reference_files/CNC-Batch [1/2] Output_File1.xlsx`
|
|
as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv.
|
|
"""
|
|
|
|
def b_postprocess(combined_df, filename):
|
|
if combined_df.shape[0] > 0:
|
|
# Add Single Code, Multiple Rate
|
|
combined_df = postprocessing_funcs.add_scmr(combined_df)
|
|
|
|
# Filter Add Ons
|
|
combined_df = postprocessing_funcs.filter_add_ons(combined_df)
|
|
|
|
# Clean MSR
|
|
combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
|
|
|
|
# Clean Default
|
|
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
|
|
|
# Clean Prov 2
|
|
combined_df = postprocessing_funcs.clean_prov_2(combined_df)
|
|
|
|
# Clean Lesser Of Rate
|
|
combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
|
|
|
|
# Clean LOB
|
|
combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
|
|
|
# Rename and reorder
|
|
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
|
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
|
final_df = combined_df[column_order]
|
|
return final_df
|
|
else:
|
|
return combined_df
|
|
|
|
|
|
##### Batch 1 #####
|
|
# New B Outputs
|
|
print("Starting B")
|
|
dir_batch_1_individual_b_outputs = 'output_individual/cnc_batch1_b/'
|
|
|
|
fn_b_batch_1_outputs_list = [dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
|
for f in os.listdir(dir_batch_1_individual_b_outputs)]
|
|
pre_concat_list_b_batch_1 = []
|
|
print("starting B concatenation for batch 1")
|
|
print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading")
|
|
|
|
for fn in fn_b_batch_1_outputs_list:
|
|
try:
|
|
file_b_df = pd.read_csv(fn)
|
|
file_b_df = b_postprocess(file_b_df, fn)
|
|
pre_concat_list_b_batch_1.append(file_b_df)
|
|
except Exception as e:
|
|
print(f"Appending failed on {fn} with exception {e}")
|
|
|
|
print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading")
|
|
abc = pd.concat(pre_concat_list_b_batch_1)
|
|
|
|
abc.drop(['Exclusions', 'Unnamed: 0'], axis=1, inplace=True)
|
|
print(abc.shape)
|
|
print(abc.columns)
|
|
|
|
# Original AC1 Outputs - All files
|
|
df_batch_1_ac1 = pd.read_csv('reference_files/CNC-1-AC.csv')
|
|
print(df_batch_1_ac1.shape)
|
|
print(df_batch_1_ac1.columns)
|
|
|
|
# quit()
|
|
|
|
# Cleaned ABC original
|
|
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
|
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
|
print("Head of df_batch_1_clean_abc['Contract_Name']:")
|
|
print(df_batch_1_clean_abc['Contract_Name'].head())
|
|
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
|
|
|
|
|
|
|
|
|
|
|
# New AC2 Outputs
|
|
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
|
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
|
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
|
pre_concat_list_batch_1 = []
|
|
print("starting AC2 concatenation for batch 1")
|
|
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
|
for fn in fn_ac2_batch_1_outputs_list:
|
|
try:
|
|
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
|
except Exception as e:
|
|
print(f"Appending failed on {fn} with exception {e}")
|
|
print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading")
|
|
df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1))
|
|
|
|
|
|
|
|
|
|
"""
|
|
1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes)
|
|
2. Updae AC 1 column names - Add AC1 original to the end of abc
|
|
3. Merge AC2 to abc
|
|
4. Postprocess
|
|
|
|
|
|
Just doing:
|
|
1. Read B
|
|
2. Postprocess as we read them in before we concatenate them ac_postprocess and b_postprocess (for b postprocess we comment out contract name, parent agreement, pages)
|
|
3. Just merge new AC2 outputs
|
|
|
|
"""
|
|
|
|
|
|
print("starting merging for batch 1")
|
|
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
|
|
batch_1_merged_ac = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
|
|
right_on='Contract Name', indicator=True)
|
|
|
|
# batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False)
|
|
|
|
print("Batch1 AC2 merge completed. Merge status counts:")
|
|
print(batch_1_merged['_merge'].value_counts())
|
|
|
|
|
|
batch_1_abc = pd.merge(batch_1_merged_ac, df_concat_batch_1_indiv_b_outputs, how='left', left_on='Contract_Name',
|
|
right_on='Contract Name', indicator=True)
|
|
print(batch_1_abc.shape)
|
|
|
|
print("Batch 1 merged and written to CSV")
|
|
|
|
batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
##### Batch 2 #####
|
|
|
|
# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
|
# df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc)
|
|
|
|
# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name']
|
|
# print("Head of df_batch_2_clean_abc['Contract_Name']:")
|
|
# print(df_batch_2_clean_abc['Contract_Name'].head())
|
|
|
|
|
|
# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
|
|
|
|
|
|
|
# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
|
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
|
# for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
|
|
|
# pre_concat_list_batch_2 = []
|
|
|
|
# print("starting concatenation for batch 2")
|
|
# print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading")
|
|
# for fn in fn_ac2_batch_2_outputs_list:
|
|
# try:
|
|
# pre_concat_list_batch_2.append(pd.read_csv(fn))
|
|
# except Exception as e:
|
|
# print(f"Appending failed on {fn} with exception {e}")
|
|
|
|
# print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading")
|
|
# df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2)
|
|
|
|
# print("batch 2 concatenation complete")
|
|
|
|
|
|
# print("starting merging for batch 2")
|
|
# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt'
|
|
# batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name',
|
|
# right_on='Contract Name', indicator=True)
|
|
# batch_2_merged.to_csv("batch_2_merged.csv", index=False)
|
|
# print("batch 2 merge completed. Merge status counts:")
|
|
# print(batch_2_merged['_merge'].value_counts())
|
|
# print("Batch 2 merged and written to CSV")
|