doczyai-pipelines/fieldExtraction/scripts/adhoc/cnc_adhoc_stitching_2.py

import os
import pandas as pd
import logging
import concurrent.futures

logging.getLogger().setLevel("ERROR")

import warnings

warnings.filterwarnings("ignore")

import postprocessing_funcs
import valid
import config


def b_postprocess(filename, combined_df, pages):
    if combined_df.shape[0] > 0:
        # Metadata fields
        # combined_df['Contract Name'] = filename
        # combined_df['Parent Agreement Code'] = postprocessing_funcs.get_parent_agreement_code(filename)
        # combined_df['Pages'] = pages

        # Add Single Code, Multiple Rate
        combined_df = postprocessing_funcs.add_scmr(combined_df)
        # Filter Add Ons
        combined_df = postprocessing_funcs.filter_add_ons(combined_df)
        # Clean MSR
        combined_df = postprocessing_funcs.clean_msr_lesser(combined_df)
        # Clean Default
        combined_df = postprocessing_funcs.clean_default_term(combined_df)
        # Clean Prov 2
        combined_df = postprocessing_funcs.clean_prov_2(combined_df)
        # Clean Lesser Of Rate
        combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
        # Clean LOB
        combined_df = postprocessing_funcs.clean_lob(combined_df, filename)
        # Rename and reorder
        combined_df.rename(columns=valid.B_MAPPING, inplace=True)
        column_order = [
            col for col in valid.B_MAPPING.values() if col in combined_df.columns
        ]
        final_df = combined_df[column_order]
        return final_df
    else:
        return combined_df


def read_individual(filepath, filename):
    df_list = []
    for file in os.listdir(filepath):
        try:
            full_path = os.path.join(filepath, file)
            df = pd.read_csv(os.path.join(full_path, filename))
            df_list.append(df)
        except:
            pass
    return pd.concat(df_list, ignore_index=True)


all_column_mappings = valid.B_MAPPING.copy()
all_column_mappings.update(valid.AC_MAPPING)


# ############################## READ - Clean AC1 + B + 3 New B ##############################
abc = read_individual("output_individual/cnc_batch2_b", "b_output.csv")
abc.drop(["Exclusions"], axis=1, inplace=True)
abc = abc[[col for col in abc.columns if "Unnamed" not in col]]
abc.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
abc.rename(
    columns={
        "If rate is % of Payer or MCR [STANDARD]": "RATE_STANDARD",
        "If rate is % of Payer or MCR [STANDARD]_Short": "RATE_SHORT",
        "Lesser of Logic Language, included (Y/N)": "LESSER",
        "Flat Fee": "FLAT_FEE_STANDARD",
        "Reimb. Methodology_short": "SHORT_METHODOLOGY",
    },
    inplace=True,
)
abc["Filename"] = abc["Filename"].str.replace(".txt", "", regex=False)
print("\n\nABC + New B: ")
print(abc.shape)
print(list(abc.columns))
print(
    f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
)


# ############################## READ - New AC2 + Smart Chunked ##############################
ac2 = read_individual("output_individual/cnc_batch2_ac", "ac_output.csv")
ac2 = ac2[[col for col in ac2.columns if "Unnamed" not in col]]
ac2.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
ac2.drop(["TERM_CLAUSE"], axis=1, inplace=True)
ac2["Filename"] = ac2["Filename"].str.replace(".txt", "", regex=False)
print("\n\nNew AC2: ")
print(ac2.shape)
print(list(ac2.columns))
print(
    f"Invalid Cols: {[col for col in ac2.columns if col not in all_column_mappings.keys()]}"
)

# ############################## READ - New AC1 (3 fields) ##############################
ac1 = read_individual("output_individual/cnc_batch2_ac1", "ac_output.csv")
# ac1.columns = ['Filename', 'Contract Effective Date', 'IRS #', 'NPI (10-digits)']
ac1.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
print("\n\nNew AC1: ")
print(ac1.shape)
print(list(ac1.columns))

# ############################## READ - Original AC1 ##############################
ac = pd.read_excel("reference_files/CNC-2.0-AC.xlsx")
ac.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
ac.rename(
    columns={
        "Evergreen, Fixed or Hard Term": "CONTRACT_AUTO_RENEWAL_IND",
        "Sequestration Reductions, included [Medicare only] (Y/N)": "SEQUESTRATION_REDUCTIONS_IND",
    },
    inplace=True,
)
ac["Filename"] = ac["Filename"].str.replace(".txt", "", regex=False)
ac = ac[ac["Filename"].isin(abc["Filename"])]

print("\n\nOriginal AC1: ")
print(ac.shape)
print(list(ac.columns))
print(
    f"Invalid Cols: {[col for col in ac.columns if col not in all_column_mappings.keys()]}"
)


############################## READ - Missing ABC ##############################
base_path = "output_individual/cnc_batch2_b_missing"
ac_frames = []
b_frames = []
for subdir in os.listdir(base_path):
    subdir_path = os.path.join(base_path, subdir)

    # Construct file paths
    ac_file_path = os.path.join(subdir_path, "test_batch-AC.csv")
    b_file_path = os.path.join(subdir_path, "test_batch-B.csv")

    # Check if files exist and then read them
    if os.path.exists(ac_file_path):
        ac_df = pd.read_csv(ac_file_path)
        ac_df.rename(columns={v: k for k, v in valid.AC_MAPPING.items()}, inplace=True)
        ac_frames.append(ac_df)
    if os.path.exists(b_file_path):
        b_df = pd.read_csv(b_file_path)
        b_df.rename(columns={v: k for k, v in valid.B_MAPPING.items()}, inplace=True)
        b_frames.append(b_df)

ac_missing = pd.concat(ac_frames, ignore_index=True)
b_missing = pd.concat(b_frames, ignore_index=True)
b_missing = b_missing[[col for col in b_missing.columns if ".1" not in col]]

abc_missing = pd.merge(ac_missing, b_missing, on="Filename", how="left")
abc_missing.rename(columns={v: k for k, v in all_column_mappings.items()}, inplace=True)
abc_missing["Filename"] = abc_missing["Filename"].str.replace(".txt", "", regex=False)

print("\n\nABC missing: ")
print(abc_missing.shape)
print(list(abc_missing.columns))
print(
    f"Invalid Cols: {[col for col in abc_missing.columns if col not in all_column_mappings.keys()]}"
)


# ############################## MERGE - Original AC + New AC1 ##############################
ac.drop(
    ["CONTRACT_EFFECTIVE_DT", "PROV_GROUP_TIN", "PROV_GROUP_NPI"], axis=1, inplace=True
)
ac = pd.merge(ac, ac1, on="Filename", how="right")  # .reset_index(drop=True)
# ac.rename(columns=all_column_mappings, inplace=True)
ac = ac[[col for col in ac.columns if "Unnamed" not in col]]
print("\n\nFinal AC1 (with 3 replaced fields)")
print(ac.shape)
print(list(ac.columns))

############################## ADD - New AC to abc ##############################
# Only ones that don't have B fields
abc = abc.reset_index(drop=True)
ac = ac.reset_index(drop=True)
abc = pd.concat([abc, ac], axis=0)
print("\n\nUpdated ABC (With non-B AC1 Fields added)")
print(abc.shape)
print(list(abc.columns))
print(
    f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
)


############################## MERGE - AC2 to abc ##############################
abc = abc[
    [
        col
        for col in abc.columns
        if col not in [c for c in ac2.columns if c != "Filename"]
    ]
]
abc = pd.merge(abc, ac2, on="Filename", how="left")
print("\n\nUpdated ABC (with Non-B AC1 fields, new AC2 field, new B fields)")
print(list(abc.columns))
print(
    f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
)
print(abc.shape)

print(abc.TERM_CLAUSE.value_counts())
# abc.rename(columns=all_column_mappings, inplace=True)
# abc.to_csv('output_consolidated/CNC-Batch1-ABC-BeforePostprocessing.csv')

############################## ADD - ABC missing to abc ##############################
abc = pd.concat([abc, abc_missing], axis=0)
print(
    "\n\nFinal ABC (with Non-B AC1 fields, new AC2 fields, new B fields, missing ABC files)"
)
print(abc.shape)
print(list(abc.columns))
print(
    f"Invalid Cols: {[col for col in abc.columns if col not in all_column_mappings.keys()]}"
)


############################## POSTPROCESS - Final Postprocess Step ##############################
def postprocess_ad_hoc(combined_df):
    # B Steps
    print(combined_df.shape)
    combined_df = postprocessing_funcs.add_scmr(combined_df)
    print(combined_df.shape)
    # combined_df = postprocessing_funcs.filter_add_ons(combined_df) # Removing rows
    # print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_lesser_rate(combined_df)
    print(combined_df.shape)
    combined_df = postprocessing_funcs.clean_default_term(combined_df)
    print(combined_df.shape)

    # combined_df = postprocessing_funcs.clean_msr_lesser(combined_df) # Modify for ad hoc
    combined_df = postprocessing_funcs.clean_prov_2(combined_df)  # Modify for ad hoc
    # combined_df = postprocessing_funcs.clean_lob(combined_df, "")

    # AC Steps
    combined_df = postprocessing_funcs.clean_ac_fields(combined_df)
    combined_df = combined_df.apply(postprocessing_funcs.derive_indicators, axis=1)
    print(combined_df.shape)
    return combined_df


abc_final = postprocess_ad_hoc(abc)
abc_final.rename(columns=all_column_mappings, inplace=True)
print("\n\nFinal ABC")
print(abc_final.shape)
print(f"Unique files: {abc_final['Contract Name'].unique()}")
print(list(abc_final.columns))
print(
    f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}"
)

abc_final = abc_final[[col for col in valid.ABC_COLUMNS if col in abc_final.columns]]
print("\n\nFinal ABC")
print(abc_final.shape)
print(list(abc_final.columns))
print(
    f"Invalid Cols: {[col for col in abc_final.columns if col not in valid.ABC_COLUMNS]}"
)


abc_final.to_csv("output_consolidated/CNC-2-RERUN-DRAFT4.csv")