diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py index d4283d3..60b251d 100644 --- a/streamlit/interface_3.py +++ b/streamlit/interface_3.py @@ -44,7 +44,7 @@ with st.sidebar: # st.write("Doczy") try: - # util.setup_page(REDIRECT_URI) + util.setup_page(REDIRECT_URI) _,c1= st.columns([4,1]) c1.write(f"User: **{st.session_state.user_info['displayName']}**") user_mail = st.session_state.user_info['mail'] @@ -74,6 +74,7 @@ except: field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True) field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True) # field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True) + field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')] st.write("Local copy of TRAINING_DATA_RAW table loaded") try: @@ -82,7 +83,6 @@ try: fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description]) fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True) fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True) - fields.rename(columns={'PRIORITY': 'PRIORITY'}, inplace = True) fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True) fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)] fields['Field Name'] = fields['SF_DB_COL_NAME'] @@ -91,6 +91,7 @@ try: except: fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True) fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name') + fields['Field Name'] = fields['SF_DB_COL_NAME'] fields = fields[~fields['Field Name'].isnull()] st.write("Local copy of BUSINESS_CONFIG table loaded") @@ -199,9 +200,13 @@ if user_mail in user_list: contract_list = sorted(file_list) # contract_list = sorted(os.listdir(SOURCE_DIRECTORY)) - if mode == 'Single field - Only Non Empty values': - column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] - field_values = field_values[~field_values[column_name].isnull()] + # if mode == 'Single field - Only Non Empty values': + # column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] + # field_values = field_values[~field_values[column_name].isnull()] + # elif mode == 'Single field': + # column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0] + # else: + # to be deleted later # df = pd.DataFrame({'col':contract_list}) # st.write(df) @@ -479,17 +484,24 @@ if user_mail in user_list: df['Snippet'] = snippet_list df['New Page Number'] = page_no_list df['Revised Prompt'] = [prompt] * len(contract_list_f) - st.write(df) + df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how ='left', on ='Field Name') - field_values_2 = pd.DataFrame(columns=['Contract Name', 'SF_DB_COL_NAME', 'Actual Value Stored']) + field_values_2 = pd.DataFrame(columns=['Contract Name', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Original Page Number']) for file_name in contract_list: document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace(' MU','').replace( '_MU','').replace('.txt','') in x][0] field_values_1 = field_values[field_values['Contract Name'] == document_name].head(1).transpose().reset_index() field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored'] + field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')] + field_values_p2 = field_values_1[field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')] + field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number'] + field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "") + + field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME']) field_values_1['Contract Name'] = document_name field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True) + df = pd.merge(df, field_values_2, how ='left', on =['Contract Name', 'SF_DB_COL_NAME']) answer_list = list(df['New Extracted value']) @@ -501,8 +513,8 @@ if user_mail in user_list: result_list = [i==j for i, j in zip(actual_value_list, answer_list)] df['Result'] = [str(x) for x in result_list] df = df[~df['Contract ID'].isnull()] - if 'Original Page Number' not in df.columns: - df['Original Page Number'] = ' ' + # if 'Original Page Number' not in df.columns: + # df['Original Page Number'] = ' ' df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level' ,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] @@ -538,7 +550,7 @@ if user_mail in user_list: csv_buf.seek(0) s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv') df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level' - ,'Snippet', 'New Page Number', 'Revised Prompt', 'Result']] + ,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] # df = pd.read_csv('results.csv') # df['Result'] = df['Result'].astype('str')