doczyai-pipelines/on_demand_scripts/training_data_pre_processing.py

import pandas as pd
from datetime import datetime


def export_column_config(column_names: list):
    """
    This function exports the column names and datatypes to a csv file
    This will then be ingested to the training data column config
    """
    # Create a data frame from the 2 lists and export as csv with the current date and time as filename 

    # Create a datatypes list that is all VARCHAR strings equal to the length of the column_names list
    try:
        column_datatypes = ["VARCHAR" for i in range(len(column_names))]
        df = pd.DataFrame(list(zip(column_names, column_datatypes)), columns=["Column_Name", "Data_Type"])
        date = datetime.now()
        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
        df.to_csv(f"column_config_{timestamp}.csv", index=False)
        return "Column config created successfully"
    except Exception as e:
        return str(e)
                        

def process_xls(file_name: str):
    """
    This function processes the master_doczy_db.xlsx file and creates a csv file with the processed data
    This will then be ingested to the training data raw table
    """
    try:
        xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=4)  # Passing header as 4 to use sf_col as header
        datatypes = xl_df.iloc[0].values.tolist()  # grab the datatypes
        xl_df2 = xl_df[26:]  # Trim the df to remove the first 26 rows where the data is not useful
        xl_df2 = xl_df2.reset_index(drop=True)
        xl_df2.columns.values[7] = "DOCUMENT_NAME" # works
        xl_df2 = xl_df2.iloc[:, 7:] # Drop columns before DOCUMENT_NAME

        start_idx = xl_df2.columns.get_loc('DOCUMENT_NAME') + 1  # +1 because we don't want to drop 'DOCUMENT_NAME'

        # Get index of 'CONTRACT_TITLE' column
        end_idx = xl_df2.columns.get_loc('CONTRACT_TITLE')

        # Create a list of column names to drop, which are between 'DOCUMENT_NAME' and 'CONTRACT_TITLE'
        cols_to_drop = xl_df2.columns[start_idx:end_idx]

        # Drop the columns
        xl_df2.drop(columns=cols_to_drop, inplace=True)

        xl_df2.dropna(axis=1, how='all')
        date = datetime.now()
        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
        
        xl_df2 = xl_df2.loc[:, ~xl_df2.columns.str.startswith('Unnamed')] # Dropping any unnamed columns (Question cols without SF_COL_NAME)
        
        date = datetime.now()
        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))

        xl_df2.columns = xl_df2.columns.str.replace('.', '_', regex=False) # Replace '.' with '_' in column names so that snowflake can ingest 

        xl_df2.to_csv(f"processed_training_data-{timestamp}.csv", index=False)
        print("Processed training data created successfully")
        return xl_df2
    
    except Exception as e:
        return str(e)
    
    
def create_business_config_table(file_name: str):
    """
    This function creates a business config table from the Business excel file
    Where we extact the sf_columns, interrogation question, priority, group_no and theme
    """
    try:
        xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=2)  # Passing header as 4 to use sf_col as header
        xl_df = xl_df.iloc[:,12:] # Drop columns before DOCUMENT_NAME
        
        questions = xl_df.columns.tolist() # Grab the questions that are in the header row
        field_name = xl_df.iloc[0].tolist() # Grab the field_name
        sf_cols = xl_df.iloc[1].tolist() # Grab the sf_cols
        priority = xl_df.iloc[3].tolist() # Grab the priority
        group_no = xl_df.iloc[4].tolist() # Grab the group_no
        theme = xl_df.iloc[5].tolist() # Grab the theme

        # Create a dataframe from the lists 
        df_internal = pd.DataFrame({'field_name':field_name,'Column_name': sf_cols, 'Question': questions, 'priority': priority, 'group_no': group_no, 'theme': theme})

        # Drop rows where the question is 'Unnamed' and the column_name is NaN (Pandas automatically fills NaN with 'Unnamed' when reading excel files depending on the formatting)
        df_cleaned = df_internal[~df_internal['Question'].str.contains('Unnamed', na=False) & ~df_internal['Column_name'].isna()]
        
        date = datetime.now()
        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))

        df_cleaned.to_csv(f'biz_config-{timestamp}.csv', index=False)
        return "Business config created successfully"
    except Exception as e:
        return str(e)


    
def main():
    """
    For this script to work, ensure all rows in the excel file are expanded between the header and actual values.
    We need the sf_column name, group no, priority etc to be accessible for ingestion
    """
    file_name = "master_doczy_db.xlsx"
    xl_df = process_xls(file_name)
    status = export_column_config(xl_df.columns.tolist())
    print(status)
    status = create_business_config_table(file_name)
    print(status)


if __name__ == "__main__":
    main()