Alex Galarce
3740588efa
Refactor/daip2-9 code refactor * add missing import * refactor ac_smart_chunking.py * update tests * removed old and unused imports * removed outdated import * forgot to import re * refactor bottom up funcs * remove unused imports from file_processing.py * refactor dict_operations.py * remove unused import from conditional_funcs.py * replace top_down_funcs with one_to_n_funcs and remove top_down_funcs.py * remove duplicate import from one_to_n_funcs.py * refactor all utils.py imports * refactor error handling by removing last code in utils and integrating InvalidDateException into postprocessing_funcs * refactor import in claude_funcs.py to use string_funcs directly and (hopefully) resolve circular import * refactor regex_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor hotfix_helper_funcs.py to use postprocessing_funcs for add_hyphen_if_needed calls * refactor postprocess.py to remove unused imports * add numpy import to string_funcs * fix tests after utils refactor * move adhoc from tests to scripts * remove unused imports * remove scripts from mypy checking * isort * Merged main into refactor/daip-2-9-code-refactor Approved-by: Katon Minhas
190 lines
6.3 KiB
Python
190 lines
6.3 KiB
Python
import pandas as pd
|
|
import os
|
|
|
|
import valid
|
|
from postprocess import ac_postprocess
|
|
import postprocessing_funcs
|
|
|
|
"""
|
|
This script consolidates a&c fields. Its data sources are `reference_files/CNC-Batch [1/2] Output_File1.xlsx`
|
|
as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv.
|
|
"""
|
|
|
|
def b_postprocess(combined_df, filename):
|
|
if combined_df.shape[0] > 0:
|
|
# Add Single Code, Multiple Rate
|
|
combined_df = postprocessing_funcs.add_scmr(combined_df)
|
|
|
|
# Filter Add Ons
|
|
combined_df = postprocessing_funcs.filter_add_ons(combined_df)
|
|
|
|
# Clean MSR
|
|
combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
|
|
|
|
# Clean Default
|
|
combined_df = postprocessing_funcs.clean_default_term(combined_df)
|
|
|
|
# Clean Prov 2
|
|
combined_df = postprocessing_funcs.clean_prov_2(combined_df)
|
|
|
|
# Clean Lesser Of Rate
|
|
combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
|
|
|
|
# Clean LOB
|
|
combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
|
|
|
|
# Rename and reorder
|
|
combined_df.rename(columns=valid.B_MAPPING, inplace=True)
|
|
column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
|
|
final_df = combined_df[column_order]
|
|
return final_df
|
|
else:
|
|
return combined_df
|
|
|
|
|
|
##### Batch 1 #####
|
|
# New B Outputs
|
|
print("Starting B")
|
|
dir_batch_1_individual_b_outputs = 'output_individual/cnc_batch1_b/'
|
|
|
|
fn_b_batch_1_outputs_list = [dir_batch_1_individual_b_outputs + f + "/b_output.csv"
|
|
for f in os.listdir(dir_batch_1_individual_b_outputs)]
|
|
pre_concat_list_b_batch_1 = []
|
|
print("starting B concatenation for batch 1")
|
|
print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading")
|
|
|
|
for fn in fn_b_batch_1_outputs_list:
|
|
try:
|
|
file_b_df = pd.read_csv(fn)
|
|
file_b_df = b_postprocess(file_b_df, fn)
|
|
pre_concat_list_b_batch_1.append(file_b_df)
|
|
except Exception as e:
|
|
print(f"Appending failed on {fn} with exception {e}")
|
|
|
|
print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading")
|
|
abc = pd.concat(pre_concat_list_b_batch_1)
|
|
|
|
abc.drop(['Exclusions', 'Unnamed: 0'], axis=1, inplace=True)
|
|
print(abc.shape)
|
|
print(abc.columns)
|
|
|
|
# Original AC1 Outputs - All files
|
|
df_batch_1_ac1 = pd.read_csv('reference_files/CNC-1-AC.csv')
|
|
print(df_batch_1_ac1.shape)
|
|
print(df_batch_1_ac1.columns)
|
|
|
|
# quit()
|
|
|
|
# Cleaned ABC original
|
|
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
|
|
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
|
|
print("Head of df_batch_1_clean_abc['Contract_Name']:")
|
|
print(df_batch_1_clean_abc['Contract_Name'].head())
|
|
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))
|
|
|
|
|
|
|
|
|
|
|
|
# New AC2 Outputs
|
|
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
|
|
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv"
|
|
for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
|
|
pre_concat_list_batch_1 = []
|
|
print("starting AC2 concatenation for batch 1")
|
|
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
|
|
for fn in fn_ac2_batch_1_outputs_list:
|
|
try:
|
|
pre_concat_list_batch_1.append(pd.read_csv(fn))
|
|
except Exception as e:
|
|
print(f"Appending failed on {fn} with exception {e}")
|
|
print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading")
|
|
df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1))
|
|
|
|
|
|
|
|
|
|
"""
|
|
1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes)
|
|
2. Updae AC 1 column names - Add AC1 original to the end of abc
|
|
3. Merge AC2 to abc
|
|
4. Postprocess
|
|
|
|
|
|
Just doing:
|
|
1. Read B
|
|
2. Postprocess as we read them in before we concatenate them ac_postprocess and b_postprocess (for b postprocess we comment out contract name, parent agreement, pages)
|
|
3. Just merge new AC2 outputs
|
|
|
|
"""
|
|
|
|
|
|
print("starting merging for batch 1")
|
|
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
|
|
batch_1_merged_ac = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
|
|
right_on='Contract Name', indicator=True)
|
|
|
|
# batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False)
|
|
|
|
print("Batch1 AC2 merge completed. Merge status counts:")
|
|
print(batch_1_merged['_merge'].value_counts())
|
|
|
|
|
|
batch_1_abc = pd.merge(batch_1_merged_ac, df_concat_batch_1_indiv_b_outputs, how='left', left_on='Contract_Name',
|
|
right_on='Contract Name', indicator=True)
|
|
print(batch_1_abc.shape)
|
|
|
|
print("Batch 1 merged and written to CSV")
|
|
|
|
batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
##### Batch 2 #####
|
|
|
|
# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
|
|
# df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc)
|
|
|
|
# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name']
|
|
# print("Head of df_batch_2_clean_abc['Contract_Name']:")
|
|
# print(df_batch_2_clean_abc['Contract_Name'].head())
|
|
|
|
|
|
# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))
|
|
|
|
|
|
|
|
# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
|
|
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv"
|
|
# for f in os.listdir(dir_batch_2_individual_ac2_outputs)]
|
|
|
|
# pre_concat_list_batch_2 = []
|
|
|
|
# print("starting concatenation for batch 2")
|
|
# print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading")
|
|
# for fn in fn_ac2_batch_2_outputs_list:
|
|
# try:
|
|
# pre_concat_list_batch_2.append(pd.read_csv(fn))
|
|
# except Exception as e:
|
|
# print(f"Appending failed on {fn} with exception {e}")
|
|
|
|
# print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading")
|
|
# df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2)
|
|
|
|
# print("batch 2 concatenation complete")
|
|
|
|
|
|
# print("starting merging for batch 2")
|
|
# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt'
|
|
# batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name',
|
|
# right_on='Contract Name', indicator=True)
|
|
# batch_2_merged.to_csv("batch_2_merged.csv", index=False)
|
|
# print("batch 2 merge completed. Merge status counts:")
|
|
# print(batch_2_merged['_merge'].value_counts())
|
|
# print("Batch 2 merged and written to CSV")
|