doczyai-pipelines/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_3.py

import os
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import logging
import concurrent.futures
logging.getLogger().setLevel("ERROR")
from concurrent.futures import ThreadPoolExecutor

import warnings
warnings.filterwarnings("ignore")

import postprocessing_funcs
import valid
import config
import prompts

def b_postprocess(filename, combined_df, pages):
    if combined_df.shape[0] > 0:
        # Metadata fields
        # combined_df['Contract Name'] = filename
        # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename)
        # combined_df['Pages'] = pages

        # Add Single Code, Multiple Rate
        combined_df = postprocessing_funcs.add_scmr(combined_df)
        # Filter Add Ons
        combined_df = postprocessing_funcs.filter_add_ons(combined_df)
        # Clean MSR
        combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
        # Clean Default
        combined_df = postprocessing_funcs.clean_default_term(combined_df)
        # Clean Prov 2
        combined_df = postprocessing_funcs.clean_prov_2(combined_df)
        # Clean Lesser Of Rate
        combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
        # Clean LOB
        combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
        # Rename and reorder
        combined_df.rename(columns=valid.B_MAPPING, inplace=True)
        column_order = [col for col in valid.B_MAPPING.values() if col in combined_df.columns]
        final_df = combined_df[column_order]
        return final_df
    else:
        return combined_df

def read_csv_file(full_path, filename):
    """Helper function to read CSV file."""
    try:
        df = pd.read_csv(os.path.join(full_path, filename))
        return df
    except Exception as e:
        # print(f"Failed to read {os.path.join(full_path, filename)}: {e}")
        return None

def read_individual(filepath, filename):
    df_list = []
    # Use ThreadPoolExecutor to handle file reading in parallel
    with ThreadPoolExecutor(max_workers=1000) as executor:
        # Create a list of future tasks
        futures = []
        for file in os.listdir(filepath):
            full_path = os.path.join(filepath, file)
            # Schedule the read_csv_file function to be executed and store the future object
            futures.append(executor.submit(read_csv_file, full_path, filename))
        
        # As each future completes, get the result (a DataFrame or None)
        for future in futures:
            result = future.result()
            if result is not None:
                df_list.append(result)
    
    # Concatenate all the DataFrames in the list
    if df_list:
        return pd.concat(df_list, ignore_index=True)


all_column_mappings = valid.B_MAPPING.copy()
all_column_mappings.update(valid.AC_MAPPING)

abc_path = 'output_individual/cnc_3_b'
ac1_original_path = 'output_consolidated/CNC-3.0-A-AC.csv'
ac1_new_path = 'output_individual/cnc_3_ac1'
ac2_path = 'output_individual/cnc_3_ac2'

# Read New AC2
# ac2 = read_individual(ac2_path, 'ac_output.csv')
# ac2.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
# ac2.to_csv('reference_files/ac2_new.csv')
ac2 = pd.read_csv('reference_files/ac2_new.csv')
ac2 = ac2[[col for col in ac2.columns if 'Unnamed' not in col]]
ac2['Filename'] = ac2['Filename'].str.replace('.txt', '', regex=False)
print("\n\nNew AC2")
print(f"Unique Filenames: {len(ac2['Filename'].unique())}")
print(ac2.shape)
print(list(ac2.columns))

# Read Original ABC + 3 new B
# abc = read_individual(abc_path, 'b_output.csv')
# abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
# abc.drop(['MEDICAL_NECESSITY_LANGUAGE_IND', 'MEDICAL_NECESSITY_LANGUAGE'], axis=1, inplace=True)
# abc.rename(columns={
#     'Lesser of Logic Language, included (Y/N)' : 'LESSER',
#     'Reimb. Methodology_short' : 'SHORT_METHODOLOGY',
# }, inplace=True)
# abc['Filename'] = abc['Filename'].str.replace('.txt', '', regex=False)
# overlapping_columns = [col for col in abc.columns if col in ac2.columns and col != 'Filename']
# abc.drop(overlapping_columns, axis=1, inplace=True)
# abc.to_csv('reference_files/abc_original.csv')
abc = pd.read_csv('reference_files/abc_original.csv')
abc = abc[[col for col in abc.columns if 'Unnamed' not in col]]
print("\n\nOriginal A1B + 3 New B")
print(f"Unique Filenames: {len(abc.Filename.unique())}")
print(abc.shape)
print(list(abc.columns))

# Read Original AC1
# ac1 = pd.read_csv(ac1_original_path)
# ac1.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
# ac1.rename(columns={'Sequestration Reductions, included [Medicare only] (Y/N)' : "SEQUESTRATION_REDUCTIONS_IND",
#                     'Evergreen, Fixed or Hard Term' : "CONTRACT_AUTO_RENEWAL_IND"}, inplace=True)
# ac1['Filename'] = ac1['Filename'].str.replace('.txt', '', regex=False)
# ac1 = ac1[~ac1['Filename'].isin(abc['Filename'])]
# overlapping_columns = [col for col in ac1.columns if col in ac2.columns and col != 'Filename']
# ac1.drop(overlapping_columns, axis=1, inplace=True)
# ac1.to_csv('reference_files/ac1_original.csv')
ac1 = pd.read_csv('reference_files/ac1_original.csv')
ac1 = ac1[[col for col in ac1.columns if 'Unnamed' not in col]]
print("\n\nOriginal AC1")
print(f"Unique Filenames: {len(ac1['Filename'].unique())}")
print(ac1.shape)
print(list(ac1.columns))

# Read 3 new AC1
# ac1_new = read_individual(ac1_new_path, 'ac_output.csv')
# ac1_new.rename(columns={v : k for k, v in all_column_mappings.items()}, inplace=True)
# ac1_new['Filename'] = ac1_new['Filename'].str.replace('.txt', '', regex=False)
# ac1_new.to_csv('reference_files/ac1_new.csv')
ac1_new = pd.read_csv('reference_files/ac1_new.csv')
ac1_new = ac1_new[[col for col in ac1_new.columns if 'Unnamed' not in col]]
print("\n\n3 New AC1")
print(f"Unique Filenames: {len(ac1_new.Filename.unique())}")
print(ac1_new.shape)
print(list(ac1_new.columns))

# Merge 3 new AC1 to original AC1
ac1.drop(['CONTRACT_EFFECTIVE_DT', 'PROV_GROUP_TIN', 'PROV_GROUP_NPI'], axis=1, inplace=True)
ac1_full = pd.merge(ac1, ac1_new, on='Filename', how='left')
print("\n\nAC 1 Full")
print(f"Unique Filenames: {len(ac1_full.Filename.unique())}")
print(ac1_full.shape)
print(list(ac1_full.columns))

# Append Full AC1 to Original ABC
abc = pd.concat([abc, ac1_full], axis=0)
abc = abc[[col for col in abc.columns if col not in [c for c in ac2.columns if c != 'Filename']]]
print("\n\nFull ABC")
print(f"Unique Filenames: {len(abc['Filename'].unique())}")
print(abc.shape)
print(list(abc.columns))

# Merge AC2
# Filenames in abc not in ac2
print(abc[~abc['Filename'].isin(ac2['Filename'])]['Filename'].unique())

# Filenames in ac2 not in abc
print(ac2[~ac2['Filename'].isin(abc['Filename'])]['Filename'].unique())

abc = pd.merge(abc, ac2, on='Filename', how='left')
print("\n\nABC Full Final (before postprocessing)")
print(f"Unique Filenames: {len(abc.Filename.unique())}")
print(abc.shape)
print(list(abc.columns))

#### Postprocess ####
def postprocess_ad_hoc(combined_df):
    # B Steps
    combined_df = postprocessing_funcs.add_scmr(combined_df)
    print(combined_df.shape)
    # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows
    # print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
    print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_default_term(combined_df)
    print(combined_df.shape)
    
    # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_prov_2(combined_df) # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_lob(combined_df, "")

    # AC Steps
    combined_df = postprocessing_funcs.clean_ac_fields(combined_df)
    combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1)
    print(combined_df.shape)
    return combined_df

abc_final = postprocess_ad_hoc(abc)
abc_final.rename(columns=all_column_mappings, inplace=True)
print("ABC Final (after postprocessing)")
print(f"Unique Filenames: {len(abc_final['Contract Name'].unique())}")
print(abc_final.shape)
print(list(abc_final.columns))

abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]]
print("ABC Full Final (after column renaming)")
print(f"Unique Filenames: {len(abc_final['Contract Name'].unique())}")
print(abc_final.shape)
print(list(abc_final.columns))

abc_final.to_csv('output_consolidated/CNC-3-RERUN-DRAFT6.csv')