doczyai-pipelines/archive/scripts/adhoc/cnc_adhoc_stitching_2.py

import os
import pandas as pd
import logging
import concurrent.futures
logging.getLogger().setLevel("ERROR")

import warnings
warnings.filterwarnings("ignore")

import postprocessing_funcs
import valid
import config

def b_postprocess(filename, combined_df, pages):
    if combined_df.shape[0] > 0:
        # Metadata fields
        # combined_df['Contract Name'] = filename
        # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename)
        # combined_df['Pages'] = pages

        # Add Single Code, Multiple Rate
        combined_df = postprocessing_funcs.add_scmr(combined_df)
        # Filter Add Ons
        combined_df = postprocessing_funcs.filter_add_ons(combined_df)
        # Clean MSR
        combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
        # Clean Default
        combined_df = postprocessing_funcs.clean_default_term(combined_df)
        # Clean Prov 2
        combined_df = postprocessing_funcs.clean_prov_2(combined_df)
        # Clean Lesser Of Rate
        combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
        # Clean LOB
        combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
        # Rename and reorder
        combined_df.rename(columns=valid.B_MAPPING, inplace=True)
        column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
        final_df = combined_df[column_order]
        return final_df
    else:
        return combined_df

def read_individual(filepath, filename):
    df_list = []
    for file in os.listdir(filepath):
        try:
            full_path = os.path.join(filepath, file)
            df = pd.read_csv(os.path.join(full_path, filename))
            df_list.append(df)
        except:
            pass
    return pd.concat(df_list, ignore_index=True)


all_column_mappings = valid.B_MAPPING.copy()
all_column_mappings.update(valid.AC_MAPPING)


# ############################## READ - Clean AC1 + B + 3 New B ##############################
abc = read_individual('output_individual/cnc_batch2_b', 'b_output.csv')
abc.drop(['Exclusions'], axis=1, inplace=True)
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
abc.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
abc.rename(columns={'If rate is % of Payer or MCR [STANDARD]' : 'RATE_STANDARD', 'If rate is % of Payer or MCR [STANDARD]_Short' : 'RATE_SHORT',
'Lesser of Logic Language, included (Y/N)' : 'LESSER', 'Flat Fee' : 'FLAT_FEE_STANDARD', 'Reimb. Methodology_short' : 'SHORT_METHODOLOGY'}, inplace=True)
abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False)
print("\n\nABC + New B: ")
print(abc.shape)
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")


# ############################## READ - New AC2 + Smart Chunked ##############################
ac2 = read_individual('output_individual/cnc_batch2_ac', 'ac_output.csv')
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
ac2.drop(['TERM_CLAUSE'], axis=1, inplace=True)
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
print("\n\nNew AC2: ")
print(ac2.shape)
print(list(ac2.columns))
print(f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}")

# ############################## READ - New AC1 (3 fields) ##############################
ac1 = read_individual('output_individual/cnc_batch2_ac1', 'ac_output.csv')
# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)']
ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
print("\n\nNew AC1: ")
print(ac1.shape)
print(list(ac1.columns))

# ############################## READ - Original AC1 ##############################
ac = pd.read_excel('reference_files/CNC-2.0-AC.xlsx')
ac.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
ac.rename(columns={'Evergreen, Fixed or Hard Term' : 'CONTRACT_AUTO_RENEWAL_IND', 
                   'Sequestration Reductions, included [Medicare only] (Y/N)' : 'SEQUESTRATION_REDUCTIONS_IND'}, inplace=True)
ac['Filename'] = ac['Filename'].str.replace('.txt', '', regex=False)
ac = ac[ac['Filename'].isin(abc['Filename'])]

print("\n\nOriginal AC1: ")
print(ac.shape)
print(list(ac.columns))
print(f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}")


############################## READ - Missing ABC ##############################
base_path = 'output_individual/cnc_batch2_b_missing'
ac_frames = []
b_frames = []
for subdir in os.listdir(base_path):
    subdir_path = os.path.join(base_path, subdir)
    
    # Construct file paths
    ac_file_path = os.path.join(subdir_path, 'test_batch-AC.csv')
    b_file_path = os.path.join(subdir_path, 'test_batch-B.csv')
    
    # Check if files exist and then read them
    if os.path.exists(ac_file_path):
        ac_df = pd.read_csv(ac_file_path)
        ac_df.rename(columns={v : k for k, v in valid.AC_MAPPING.items()}, inplace=True)
        ac_frames.append(ac_df)
    if os.path.exists(b_file_path):
        b_df = pd.read_csv(b_file_path)
        b_df.rename(columns={v : k for k, v in valid.B_MAPPING.items()}, inplace=True)
        b_frames.append(b_df)

ac_missing = pd.concat(ac_frames, ignore_index=True)
b_missing = pd.concat(b_frames, ignore_index=True)
b_missing = b_missing[[col for col in b_missing.columns if '.1' not in col]]

abc_missing = pd.merge(ac_missing, b_missing, on='Filename', how='left')
abc_missing.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
abc_missing['Filename'] = abc_missing['Filename'].str.replace('.txt', '', regex=False)

print("\n\nABC missing: ")
print(abc_missing.shape)
print(list(abc_missing.columns))
print(f"Invalid Cols: {[col for col in abc_missing.columns if col not in all_column_mappings.keys()]}")


# ############################## MERGE - Original AC + New AC1 ##############################
ac.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
ac = pd.merge(ac, ac1, on='Filename', how='right') # .reset_index(drop=True)
# ac.rename(columns=all_column_mappings, inplace=True)
ac = ac[[col for col in ac.columns if 'Unnamed' not in col]]
print("\n\nFinal AC1 (with 3 replaced fields)")
print(ac.shape)
print(list(ac.columns))

############################## ADD - New AC to abc ##############################
# Only ones that don't have B fields
abc = abc.reset_index(drop=True)
ac = ac.reset_index(drop=True)
abc = pd.concat([abc, ac], axis=0)
print("\n\nUpdated ABC (With non-B AC1 Fields added)")
print(abc.shape)
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")


############################## MERGE - AC2 to abc ##############################
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]
abc = pd.merge(abc, ac2, on='Filename', how='left')
print("\n\nUpdated ABC (with Non-B AC1 fields, new AC2 field, new B fields)")
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
print(abc.shape)

print(abc.TERM_CLAUSE.value_counts())
# abc.rename(columns=all_column_mappings, inplace=True)
# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv')

############################## ADD - ABC missing to abc ##############################
abc = pd.concat([abc, abc_missing], axis=0)
print("\n\nFinal ABC (with Non-B AC1 fields, new AC2 fields, new B fields, missing ABC files)")
print(abc.shape)
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")


############################## POSTPROCESS - Final Postprocess Step ##############################
def postprocess_ad_hoc(combined_df):
    # B Steps
    print(combined_df.shape)
    combined_df = postprocessing_funcs.add_scmr(combined_df)
    print(combined_df.shape)
    # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows
    # print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
    print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_default_term(combined_df)
    print(combined_df.shape)
    
    # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
    combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_lob(combined_df, "")

    # AC Steps
    combined_df = postprocessing_funcs.clean_ac_fields(combined_df)
    combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1)
    print(combined_df.shape)
    return combined_df

abc_final = postprocess_ad_hoc(abc)
abc_final.rename(columns=all_column_mappings, inplace=True)
print("\n\nFinal ABC")
print(abc_final.shape)
print(f'Unique files: {abc_final['Contract Name'].unique()}')
print(list(abc_final.columns))
print(f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}")

abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]]
print("\n\nFinal ABC")
print(abc_final.shape)
print(list(abc_final.columns))
print(f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}")


abc_final.to_csv('output_consolidated/CNC-2-RERUN-DRAFT4.csv')