doczyai-pipelines/fieldExtraction/tests/adhoc/consolidate_ac2.py

import pandas as pd
import os

import valid
from postprocess import ac_postprocess
import postprocessing_funcs

"""
This script consolidates a&c fields.  Its data sources are `reference_files/CNC-Batch [1/2] Output_File1.xlsx`
as well as .csv files from `output_individual/cnc_batch[1/2]_ac/[name of provider]/ac_output.csv.
"""

def b_postprocess(combined_df, filename):
    if combined_df.shape[0] > 0:
        # Add Single Code, Multiple Rate
        combined_df = postprocessing_funcs.add_scmr(combined_df)

        # Filter Add Ons
        combined_df = postprocessing_funcs.filter_add_ons(combined_df)

        # Clean MSR
        combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
        
        # Clean Default
        combined_df = postprocessing_funcs.clean_default_term(combined_df)

        # Clean Prov 2
        combined_df = postprocessing_funcs.clean_prov_2(combined_df)

        # Clean Lesser Of Rate
        combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)

        # Clean LOB
        combined_df = postprocessing_funcs.clean_lob(combined_df, filename)

        # Rename and reorder
        combined_df.rename(columns=valid.B_MAPPING, inplace=True)
        column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
        final_df = combined_df[column_order]
        return final_df
    else:
        return combined_df


##### Batch 1 #####
# New B Outputs
print("Starting B")
dir_batch_1_individual_b_outputs = 'output_individual/cnc_batch1_b/'

fn_b_batch_1_outputs_list = [dir_batch_1_individual_b_outputs + f + "/b_output.csv" 
												for f in os.listdir(dir_batch_1_individual_b_outputs)]
pre_concat_list_b_batch_1 = []
print("starting B concatenation for batch 1")
print(str(len(fn_b_batch_1_outputs_list)) + " B files pre-CSV reading")

for fn in fn_b_batch_1_outputs_list:
	try:
		file_b_df = pd.read_csv(fn)
		file_b_df = b_postprocess(file_b_df, fn)
		pre_concat_list_b_batch_1.append(file_b_df)
	except Exception as e:
		print(f"Appending failed on {fn} with exception {e}")

print(str(len(pre_concat_list_b_batch_1)) + " B files post-CSV reading")
abc = pd.concat(pre_concat_list_b_batch_1)

abc.drop(['Exclusions', 'Unnamed: 0'], axis=1, inplace=True)
print(abc.shape)
print(abc.columns)

# Original AC1 Outputs - All files
df_batch_1_ac1 = pd.read_csv('reference_files/CNC-1-AC.csv')
print(df_batch_1_ac1.shape)
print(df_batch_1_ac1.columns)

# quit()

# Cleaned ABC original
fn_batch_1_clean_abc = 'reference_files/CNC-Batch 1 Output_File1.xlsx'
df_batch_1_clean_abc = pd.read_excel(fn_batch_1_clean_abc)
print("Head of df_batch_1_clean_abc['Contract_Name']:")
print(df_batch_1_clean_abc['Contract_Name'].head())
print("Batch 1 clean shape: " + str(df_batch_1_clean_abc.shape))





# New AC2 Outputs
dir_batch_1_individual_ac2_outputs = 'output_individual/cnc_batch1_ac/'
fn_ac2_batch_1_outputs_list = [dir_batch_1_individual_ac2_outputs + f + "/ac_output.csv" 
												for f in os.listdir(dir_batch_1_individual_ac2_outputs)]
pre_concat_list_batch_1 = []
print("starting AC2 concatenation for batch 1")
print(str(len(fn_ac2_batch_1_outputs_list)) + " files pre-CSV reading")
for fn in fn_ac2_batch_1_outputs_list:
	try:
		pre_concat_list_batch_1.append(pd.read_csv(fn))
	except Exception as e:
		print(f"Appending failed on {fn} with exception {e}")
print(str(len(pre_concat_list_batch_1)) + " AC2 files post-CSV reading")
df_concat_batch_1_indiv_outputs = ac_postprocess(pd.concat(pre_concat_list_batch_1))




"""
1. Read B - create abc - remove old columns (Exclusions, Add-On, MedNes)
2. Updae AC 1 column names - Add AC1 original to the end of abc
3. Merge AC2 to abc
4. Postprocess


Just doing:
1. Read B
2. Postprocess as we read them in before we concatenate them ac_postprocess and b_postprocess (for b postprocess we comment out contract name, parent agreement, pages)
3. Just merge new AC2 outputs

"""


print("starting merging for batch 1")
df_batch_1_clean_abc['Contract_Name'] = df_batch_1_clean_abc['Contract_Name'] + '.txt'
batch_1_merged_ac = pd.merge(df_batch_1_clean_abc, df_concat_batch_1_indiv_outputs, how='left', left_on='Contract_Name',
                          right_on='Contract Name', indicator=True)

# batch_1_merged.to_csv("output_consolidated/CNC-1-AC2-Added.csv", index=False)

print("Batch1 AC2 merge completed.  Merge status counts:")
print(batch_1_merged['_merge'].value_counts())


batch_1_abc = pd.merge(batch_1_merged_ac, df_concat_batch_1_indiv_b_outputs, how='left', left_on='Contract_Name',
                          right_on='Contract Name', indicator=True)
print(batch_1_abc.shape)

print("Batch 1 merged and written to CSV")

batch_1_abc.to_csv("batch_1_merged_abc.csv", index=False)








##### Batch 2 #####

# fn_batch_2_clean_abc = 'reference_files/CNC-Batch 2 Output_File1.xlsx'
# df_batch_2_clean_abc = pd.read_excel(fn_batch_2_clean_abc)

# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract Name']
# print("Head of df_batch_2_clean_abc['Contract_Name']:")
# print(df_batch_2_clean_abc['Contract_Name'].head())


# print("Batch 2 clean shape: " + str(df_batch_2_clean_abc.shape))



# dir_batch_2_individual_ac2_outputs = 'output_individual/cnc_batch2_ac/'
# fn_ac2_batch_2_outputs_list = [dir_batch_2_individual_ac2_outputs + f + "/ac_output.csv" 
# 															for f in os.listdir(dir_batch_2_individual_ac2_outputs)]

# pre_concat_list_batch_2 = []

# print("starting concatenation for batch 2")
# print(str(len(fn_ac2_batch_2_outputs_list)) + " files pre-CSV reading")
# for fn in fn_ac2_batch_2_outputs_list:
# 	try:
# 		pre_concat_list_batch_2.append(pd.read_csv(fn))
# 	except Exception as e:
# 		print(f"Appending failed on {fn} with exception {e}")

# print(str(len(pre_concat_list_batch_2)) + " files post-CSV reading")
# df_concat_batch_2_indiv_outputs = pd.concat(pre_concat_list_batch_2)

# print("batch 2 concatenation complete")


# print("starting merging for batch 2")
# df_batch_2_clean_abc['Contract_Name'] = df_batch_2_clean_abc['Contract_Name'] + '.txt'
# batch_2_merged = pd.merge(df_batch_2_clean_abc, df_concat_batch_2_indiv_outputs, how='left', left_on='Contract_Name',
# 													right_on='Contract Name', indicator=True)
# batch_2_merged.to_csv("batch_2_merged.csv", index=False)
# print("batch 2 merge completed.  Merge status counts:")
# print(batch_2_merged['_merge'].value_counts())
# print("Batch 2 merged and written to CSV")