doczyai-pipelines/fieldExtraction/src/postprocess.py

import pandas as pd
import re
import os
import numpy as np

import postprocessing_funcs
import valid
import prompts
import claude_funcs
import config
import utils


def b_postprocess(filename, df, pages):

    if df.shape[0] > 0:
        # Metadata fields
        df.drop("Filename", axis=1, inplace=True)

        df["Contract Name"] = "Filename: " + filename
        df["Parent Agreement Code"] = postprocessing_funcs.get_parent_agreement_code(
            filename
        )
        df["Pages"] = pages

        # Add Single Code, Multiple Rate
        df = postprocessing_funcs.add_scmr(df)

        # Filter Add Ons
        df = postprocessing_funcs.filter_add_ons(df)

        # Clean MSR
        df = postprocessing_funcs.clean_msr_lesser(df)

        # Clean Default
        df = postprocessing_funcs.clean_default_term(df)

        # Clean Prov 2
        df = postprocessing_funcs.clean_prov_2(df)

        # Clean Lesser Of Rate
        df = postprocessing_funcs.clean_lesser_rate(df)

        # Clean LOB
        df = postprocessing_funcs.clean_lob(df, filename)

        df = postprocessing_funcs.methodology_short_fix(df)

        df = postprocessing_funcs.yn_fixes(df)

        # Rename and reorder
        df.rename(columns=valid.B_MAPPING, inplace=True)

        column_order = [col for col in valid.B_MAPPING.values() if col in df.columns]
        final_df = df[column_order]

        return final_df
    else:
        return df


def ac_postprocess(df, text_dict, filename, pages):

    if df.shape[0] > 0:

        df["Contract Name"] = "Filename: " + filename

        df["Pages"] = pages

        df["Parent Agreement Code"] = postprocessing_funcs.get_parent_agreement_code(
            filename
        )

        df = df.fillna("")

        df = postprocessing_funcs.clean_term_clause(df)

        df = postprocessing_funcs.clean_auto_renewal_ind(df)

        df = postprocessing_funcs.get_tin_from_filename(df)

        df = postprocessing_funcs.clean_tin(df)

        df = postprocessing_funcs.clean_npi(df)

        df = postprocessing_funcs.clean_network_access_fees(df)

        df = postprocessing_funcs.clean_health_plan_state(df)

        df = postprocessing_funcs.clean_health_plan_state_from_hotfix(df,filename,text_dict)
        
        df = postprocessing_funcs.clean_notice_provider_name_and_address(df)

        df = postprocessing_funcs.clean_policies_and_procedures(df)

        df = postprocessing_funcs.clean_tin_npi_other(df)

        df = postprocessing_funcs.icm_number_fix(df)

        df = postprocessing_funcs.yn_fixes(df)

        df = df.apply(lambda x: x.map(postprocessing_funcs.replace_null_terms))

        df = df.apply(lambda x: x.map(postprocessing_funcs.replace_quotes))

        # Derive Indicators
        df = df.apply(postprocessing_funcs.derive_indicators, axis=1)

        # Rename and Reorder
        df = df.rename(columns=valid.AC_MAPPING)
        column_order = [col for col in valid.ABC_COLUMNS if col in df.columns] + [
            col for col in df.columns if col not in valid.ABC_COLUMNS
        ]

        final_df = df[column_order]

        return final_df
    else:
        return df