Umang Mistry
114 lines
4.9 KiB
Python
114 lines
4.9 KiB
Python
import pandas as pd
|
|
from datetime import datetime
|
|
|
|
|
|
def export_column_config(column_names: list):
|
|
"""
|
|
This function exports the column names and datatypes to a csv file
|
|
This will then be ingested to the training data column config
|
|
"""
|
|
# Create a data frame from the 2 lists and export as csv with the current date and time as filename
|
|
|
|
# Create a datatypes list that is all VARCHAR strings equal to the length of the column_names list
|
|
try:
|
|
column_datatypes = ["VARCHAR" for i in range(len(column_names))]
|
|
df = pd.DataFrame(list(zip(column_names, column_datatypes)), columns=["Column_Name", "Data_Type"])
|
|
date = datetime.now()
|
|
timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
|
|
df.to_csv(f"column_config_{timestamp}.csv", index=False)
|
|
return "Column config created successfully"
|
|
except Exception as e:
|
|
return str(e)
|
|
|
|
|
|
def process_xls(file_name: str):
|
|
"""
|
|
This function processes the master_doczy_db.xlsx file and creates a csv file with the processed data
|
|
This will then be ingested to the training data raw table
|
|
"""
|
|
try:
|
|
xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=4) # Passing header as 4 to use sf_col as header
|
|
datatypes = xl_df.iloc[0].values.tolist() # grab the datatypes
|
|
xl_df2 = xl_df[26:] # Trim the df to remove the first 26 rows where the data is not useful
|
|
xl_df2 = xl_df2.reset_index(drop=True)
|
|
xl_df2.columns.values[7] = "DOCUMENT_NAME" # works
|
|
xl_df2 = xl_df2.iloc[:, 7:] # Drop columns before DOCUMENT_NAME
|
|
|
|
start_idx = xl_df2.columns.get_loc('DOCUMENT_NAME') + 1 # +1 because we don't want to drop 'DOCUMENT_NAME'
|
|
|
|
# Get index of 'CONTRACT_TITLE' column
|
|
end_idx = xl_df2.columns.get_loc('CONTRACT_TITLE')
|
|
|
|
# Create a list of column names to drop, which are between 'DOCUMENT_NAME' and 'CONTRACT_TITLE'
|
|
cols_to_drop = xl_df2.columns[start_idx:end_idx]
|
|
|
|
# Drop the columns
|
|
xl_df2.drop(columns=cols_to_drop, inplace=True)
|
|
|
|
xl_df2.dropna(axis=1, how='all')
|
|
date = datetime.now()
|
|
timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
|
|
|
|
xl_df2 = xl_df2.loc[:, ~xl_df2.columns.str.startswith('Unnamed')] # Dropping any unnamed columns (Question cols without SF_COL_NAME)
|
|
|
|
date = datetime.now()
|
|
timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
|
|
|
|
xl_df2.columns = xl_df2.columns.str.replace('.', '_', regex=False) # Replace '.' with '_' in column names so that snowflake can ingest
|
|
|
|
xl_df2.to_csv(f"processed_training_data-{timestamp}.csv", index=False)
|
|
print("Processed training data created successfully")
|
|
return xl_df2
|
|
|
|
except Exception as e:
|
|
return str(e)
|
|
|
|
|
|
def create_business_config_table(file_name: str):
|
|
"""
|
|
This function creates a business config table from the Business excel file
|
|
Where we extact the sf_columns, interrogation question, priority, group_no and theme
|
|
"""
|
|
try:
|
|
xl_df = pd.read_excel(file_name, sheet_name="Data Base", header=2) # Passing header as 4 to use sf_col as header
|
|
xl_df = xl_df.iloc[:,12:] # Drop columns before DOCUMENT_NAME
|
|
|
|
questions = xl_df.columns.tolist() # Grab the questions that are in the header row
|
|
field_name = xl_df.iloc[0].tolist() # Grab the field_name
|
|
sf_cols = xl_df.iloc[1].tolist() # Grab the sf_cols
|
|
priority = xl_df.iloc[3].tolist() # Grab the priority
|
|
group_no = xl_df.iloc[4].tolist() # Grab the group_no
|
|
theme = xl_df.iloc[5].tolist() # Grab the theme
|
|
|
|
# Create a dataframe from the lists
|
|
df_internal = pd.DataFrame({'field_name':field_name,'Column_name': sf_cols, 'Question': questions, 'priority': priority, 'group_no': group_no, 'theme': theme})
|
|
|
|
# Drop rows where the question is 'Unnamed' and the column_name is NaN (Pandas automatically fills NaN with 'Unnamed' when reading excel files depending on the formatting)
|
|
df_cleaned = df_internal[~df_internal['Question'].str.contains('Unnamed', na=False) & ~df_internal['Column_name'].isna()]
|
|
|
|
date = datetime.now()
|
|
timestamp = str(date.strftime("%m%d%Y_%H%M%S"))
|
|
|
|
df_cleaned.to_csv(f'biz_config-{timestamp}.csv', index=False)
|
|
return "Business config created successfully"
|
|
except Exception as e:
|
|
return str(e)
|
|
|
|
|
|
|
|
def main():
|
|
"""
|
|
For this script to work, ensure all rows in the excel file are expanded between the header and actual values.
|
|
We need the sf_column name, group no, priority etc to be accessible for ingestion
|
|
"""
|
|
file_name = "master_doczy_db.xlsx"
|
|
xl_df = process_xls(file_name)
|
|
status = export_column_config(xl_df.columns.tolist())
|
|
print(status)
|
|
status = create_business_config_table(file_name)
|
|
print(status)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|