doczyai-pipelines/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_1.py

import os
import pandas as pd
import logging
import concurrent.futures
logging.getLogger().setLevel("ERROR")

import warnings
warnings.filterwarnings("ignore")

import postprocessing_funcs
import valid
import config

def b_postprocess(filename, combined_df, pages):
    if combined_df.shape[0] > 0:
        # Metadata fields
        # combined_df['Contract Name'] = filename
        # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename)
        # combined_df['Pages'] = pages

        # Add Single Code, Multiple Rate
        combined_df = postprocessing_funcs.add_scmr(combined_df)
        # Filter Add Ons
        combined_df = postprocessing_funcs.filter_add_ons(combined_df)
        # Clean MSR
        combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
        # Clean Default
        combined_df = postprocessing_funcs.clean_default_term(combined_df)
        # Clean Prov 2
        combined_df = postprocessing_funcs.clean_prov_2(combined_df)
        # Clean Lesser Of Rate
        combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
        # Clean LOB
        combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
        # Rename and reorder
        combined_df.rename(columns=valid.B_MAPPING, inplace=True)
        column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
        final_df = combined_df[column_order]
        return final_df
    else:
        return combined_df

def read_individual(filepath, filename):
    df_list = []
    for file in os.listdir(filepath):
        full_path = os.path.join(filepath, file)
        df = pd.read_csv(os.path.join(full_path, filename))
        df_list.append(df)
    return pd.concat(df_list, ignore_index=True)

all_column_mappings = valid.B_MAPPING.copy()
all_column_mappings.update(valid.AC_MAPPING)

# For Original ABC - 
# For Duplicates - 
# For Non-B: All AC1 and AC2 Fields

# TERM_CLAUSE, CONTRACT_EFFECTIVE_DATE, PROV_GROUP_TIN, PROV_GROUP_NPI, DEFAULT_TERM + [ALL PART 2 FIELDS]


# # ############################## READ - Clean ABC Part 1 + 3 New B Fields ##############################
abc = read_individual('output_individual/cnc_batch1_b', 'b_output.csv')

abc.drop(['Exclusions'], axis=1, inplace=True)
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
abc.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
abc.rename(columns={'If rate is % of Payer or MCR [STANDARD]' : 'RATE_STANDARD', 'If rate is % of Payer or MCR [STANDARD]_Short' : 'RATE_SHORT',
'Lesser of Logic Language, included (Y/N)' : 'LESSER', 'Flat Fee' : 'FLAT_FEE_STANDARD', 'Reimb. Methodology_short' : 'SHORT_METHODOLOGY'}, inplace=True)
abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False)

print("Clean ABC Part 1 + 3 New B Fields: ")
print(f"Filenames: {len(abc.Filename.unique())}", abc.Filename[0])
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
print(abc.shape)

# ############################## READ - AC2 For All Contracts ##############################
ac2 = read_individual('output_individual/cnc_batch1_ac', 'ac_output.csv')
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
print("AC2 For All Contracts: ")
print(f"Filenames: {len(ac2.Filename.unique())}", ac2.Filename[0])
print(list(ac2.columns))
print(f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}")
print(ac2.shape)

# ############################## READ - New AC1 (3 fields) For All Contracts ##############################
ac1 = read_individual('output_individual/cnc_batch1_ac1', 'ac_output.csv')
# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)']
ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
ac1['Filename'] = ac1['Filename'].str.replace('.txt', '', regex=False)
print("New AC1 For All Contracts: ")
print(f"Filenames: {len(ac1.Filename.unique())}", ac1.Filename[0])
print(list(ac1.columns))
print(ac1.shape)

# ############################## READ - Original AC1 For All Contracts ##############################
ac = pd.read_csv('reference_files/CNC-1-AC.csv')
ac.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
ac.rename(columns={'Evergreen, Fixed or Hard Term' : 'CONTRACT_AUTO_RENEWAL_IND', 
                   'Sequestration Reductions, included [Medicare only] (Y/N)' : 'SEQUESTRATION_REDUCTIONS_IND'}, inplace=True)
print("Original AC1 For Non-B Contracts: ")
print(f"Filenames: {len(ac.Filename.unique())}", ac.Filename[0])
print(list(ac.columns))
print(f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}")
print(ac.shape)

############################## MERGE - Original AC1 + 3 New AC1 Fields ##############################
ac.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
ac = pd.merge(ac, ac1, on='Filename', how='right') # .reset_index(drop=True)
# ac.rename(columns=all_column_mappings, inplace=True)
ac = ac[[col for col in ac.columns if 'Unnamed' not in col]]
print("Final AC1 (with 3 replaced fields)")
print(f"Filenames: {len(ac.Filename.unique())}")
print(list(ac.columns))
print(ac.shape)

############################## FILTER - Only AC1 that are not already in Clean ABC ##############################
ac_new_only = ac[~ac['Filename'].isin(abc.Filename.unique())]
print("AC New Only (with 3 replaced fields)")
print(f"Filenames: {len(ac_new_only.Filename.unique())}")
print(list(ac_new_only.columns))
print(ac_new_only.shape)

############################## ADD - Only New AC1 to abc ##############################
abc = abc.reset_index(drop=True)
ac = ac.reset_index(drop=True)
abc = pd.concat([abc, ac_new_only], axis=0)
print("Updated ABC (With non-B AC1 Fields added)")
print(f"Filenames: {len(abc.Filename.unique())}")
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
print(abc.shape)

############################## MERGE - AC2 to abc ##############################
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]

# Merge ac2 to abc, keeping the columns from ac2 when there is a conflict. Except for CREDENTIALING_APP_IND
# abc = pd.merge(abc, ac2, on='Filename', how='left')
abc = pd.merge(abc, ac2, on='Filename', how='left', suffixes=('', '_from_ac2'))
for column in abc.columns:
    if column.endswith('_from_ac2'):
        orig_column = column[:-10]  # Remove the '_from_ac2' suffix
        if orig_column != 'CREDENTIALING_APP_IND':
            abc[orig_column] = abc[column]
        abc.drop(column, axis=1, inplace=True)

print("Updated ABC (with Non-B AC1 fields, new AC2 field, new B fields)")
print(f"Filenames: {len(abc.Filename.unique())}")
print(list(abc.columns))
print(f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}")
print(abc.shape)
# abc.rename(columns=all_column_mappings, inplace=True)
# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv')

############################## POSTPROCESS - Final Postprocess Step ##############################
def postprocess_ad_hoc(combined_df):
    # B Steps
    print(combined_df.shape)
    combined_df = postprocessing_funcs.add_scmr(combined_df)
    print(combined_df.shape)
    # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows
    # print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
    print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_default_term(combined_df)
    print(combined_df.shape)
    
    # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_lob(combined_df, "")

    # AC Steps
    combined_df = postprocessing_funcs.clean_ac_fields(combined_df)
    combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1)
    print(combined_df.shape)
    return combined_df

abc_final = postprocess_ad_hoc(abc)
abc_final.rename(columns=all_column_mappings, inplace=True)
abc_final.to_csv('output_consolidated/CNC-1-RERUN-DRAFT4.csv')
print("Final ABC")
print(f"Filenames: {len(abc.Filename.unique())}")
print(list(abc_final.columns))
print(abc_final.shape)