Added training data new script, 1st draft of logging lambda and some updates to SF objs

on_demand_scripts/training_data_pre_processing.py

@@ -0,0 +1,111 @@
+import pandas as pd
+from datetime import datetime
+
+
+def export_column_config(column_names: list):
+    """
+    This function exports the column names and datatypes to a csv file
+    This will then be ingested to the training data column config
+    """
+    # Create a data frame from the 2 lists and export as csv with the current date and time as filename 
+
+    # Create a datatypes list that is all VARCHAR strings equal to the length of the column_names list
+    try:
+        column_datatypes = ["VARCHAR" for i in range(len(column_names))]
+        df = pd.DataFrame(list(zip(column_names, column_datatypes)), columns=["Column_Name", "Data_Type"])
+        date = datetime.now()
+        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
+        df.to_csv(f"column_config_{timestamp}.csv", index=False)
+        return "Column config created successfully"
+    except Exception as e:
+        return str(e)
+                        
+
+def process_xls(file_name: str):
+    """
+    This function processes the master_doczy_db.xlsx file and creates a csv file with the processed data
+    This will then be ingested to the training data raw table
+    """
+    try:
+        xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=4)  # Passing header as 4 to use sf_col as header
+        datatypes = xl_df.iloc[0].values.tolist()  # grab the datatypes
+        xl_df2 = xl_df[26:]  # Trim the df to remove the first 26 rows where the data is not useful
+        xl_df2 = xl_df2.reset_index(drop=True)
+        xl_df2.columns.values[7] = "DOCUMENT_NAME" # works
+        xl_df2 = xl_df2.iloc[:, 7:] # Drop columns before DOCUMENT_NAME
+
+        start_idx = xl_df2.columns.get_loc('DOCUMENT_NAME') + 1  # +1 because we don't want to drop 'DOCUMENT_NAME'
+
+        # Get index of 'CONTRACT_TITLE' column
+        end_idx = xl_df2.columns.get_loc('CONTRACT_TITLE')
+
+        # Create a list of column names to drop, which are between 'DOCUMENT_NAME' and 'CONTRACT_TITLE'
+        cols_to_drop = xl_df2.columns[start_idx:end_idx]
+
+        # Drop the columns
+        xl_df2.drop(columns=cols_to_drop, inplace=True)
+
+        xl_df2.dropna(axis=1, how='all')
+        date = datetime.now()
+        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
+        
+        xl_df2 = xl_df2.loc[:, ~xl_df2.columns.str.startswith('Unnamed')] # Dropping any unnamed columns (Question cols without SF_COL_NAME)
+        
+        date = datetime.now()
+        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
+
+        xl_df2.columns = xl_df2.columns.str.replace('.', '_', regex=False) # Replace '.' with '_' in column names so that snowflake can ingest 
+
+        xl_df2.to_csv(f"processed_training_data-{timestamp}.csv", index=False)
+        print("Processed training data created successfully")
+        return xl_df2
+    
+    except Exception as e:
+        return str(e)
+    
+    
+def create_business_config_table(file_name: str):
+    """
+    This function creates a business config table from the Business excel file
+    Where we extact the sf_columns, interrogation question, priority, group_no and theme
+    """
+    try:
+        xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=2)  # Passing header as 4 to use sf_col as header
+        xl_df = xl_df.iloc[:,12:] # Drop columns before DOCUMENT_NAME
+        
+        questions = xl_df.columns.tolist() # Grab the questions that are in the header row
+        sf_cols = xl_df.iloc[1].tolist() # Grab the sf_cols
+        priority = xl_df.iloc[3].tolist() # Grab the priority
+        group_no = xl_df.iloc[4].tolist() # Grab the group_no
+        theme = xl_df.iloc[5].tolist() # Grab the theme
+
+        # Create a dataframe from the lists 
+        df_internal = pd.DataFrame({'Column_name': sf_cols, 'Question': questions, 'priority': priority, 'group_no': group_no, 'theme': theme})
+
+        # Drop rows where the question is 'Unnamed' and the column_name is NaN (Pandas automatically fills NaN with 'Unnamed' when reading excel files depending on the formatting)
+        df_cleaned = df_internal[~df_internal['Question'].str.contains('Unnamed', na=False) & ~df_internal['Column_name'].isna()]
+        
+        date = datetime.now()
+        timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
+
+        df_cleaned.to_csv(f'biz_config-{timestamp}.csv', index=False)
+    except Exception as e:
+        return str(e)
+
+
+    
+def main():
+    """
+    For this script to work, ensure all rows in the excel file are expanded between the header and actual values.
+    We need the sf_column name, group no, priority etc to be accessible for ingestion
+    """
+    file_name = "master_doczy_db.xlsx"
+    xl_df = process_xls(file_name)
+    status = export_column_config(xl_df.columns.tolist())
+    print(status)
+    status = create_business_config_table(file_name)
+    print(status)
+
+
+if __name__ == "__main__":
+    main()