jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

original page no. added in UI3

Browse Source
This commit is contained in:
Mayank Aamseek
2024-03-28 23:31:12 +05:30
parent cdffe3d4f9
commit 5f7614b98f
1 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_3.py
+22 -10
View File
@@ -44,7 +44,7 @@ with st.sidebar:
# st.write("Doczy")
try:
# util.setup_page(REDIRECT_URI)
util.setup_page(REDIRECT_URI)
_,c1= st.columns([4,1])
c1.write(f"User: **{st.session_state.user_info['displayName']}**")
user_mail = st.session_state.user_info['mail']
@@ -74,6 +74,7 @@ except:
field_values = pd.read_csv('contract_field_values.csv', encoding='unicode_escape', skipinitialspace=True)
field_values.rename(columns={'(internal) Document Name': 'Document_Name'}, inplace = True)
# field_values.rename(columns={'(Internal) Carveout ID': 'Contract ID'}, inplace = True)
field_values = field_values.loc[:, ~field_values.columns.str.contains('Unnamed:')]
st.write("Local copy of TRAINING_DATA_RAW table loaded")
try:
@@ -82,7 +83,6 @@ try:
fields = pd.DataFrame.from_records(iter(cur), columns=[x[0] for x in cur.description])
fields.rename(columns={'FIELD_NAME': 'Field Name'}, inplace = True)
fields.rename(columns={'QUESTION': 'Interrogation Question?'}, inplace = True)
fields.rename(columns={'PRIORITY': 'PRIORITY'}, inplace = True)
fields.rename(columns={'SF_COL_NAME': 'SF_DB_COL_NAME'}, inplace = True)
fields = fields[~fields['SF_DB_COL_NAME'].str.endswith('_PG', na=None)]
fields['Field Name'] = fields['SF_DB_COL_NAME']
@@ -91,6 +91,7 @@ try:
except:
fields = pd.read_csv('contract_fields.csv', encoding='unicode_escape', skipinitialspace=True)
fields = fields.drop_duplicates(subset='Field Name', keep="first").sort_values('Field Name')
fields['Field Name'] = fields['SF_DB_COL_NAME']
fields = fields[~fields['Field Name'].isnull()]
st.write("Local copy of BUSINESS_CONFIG table loaded")
@@ -199,9 +200,13 @@ if user_mail in user_list:
contract_list = sorted(file_list)
# contract_list = sorted(os.listdir(SOURCE_DIRECTORY))
if mode == 'Single field - Only Non Empty values':
column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
field_values = field_values[~field_values[column_name].isnull()]
# if mode == 'Single field - Only Non Empty values':
# column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
# field_values = field_values[~field_values[column_name].isnull()]
# elif mode == 'Single field':
# column_name = fields.loc[fields['Field Name'] == field, 'SF_DB_COL_NAME'].iloc[0]
# else:
# to be deleted later
# df = pd.DataFrame({'col':contract_list})
# st.write(df)
@@ -479,17 +484,24 @@ if user_mail in user_list:
df['Snippet'] = snippet_list
df['New Page Number'] = page_no_list
df['Revised Prompt'] = [prompt] * len(contract_list_f)
st.write(df)
df = pd.merge(df, fields[['Field Name', 'SF_DB_COL_NAME']], how ='left', on ='Field Name')
field_values_2 = pd.DataFrame(columns=['Contract Name', 'SF_DB_COL_NAME', 'Actual Value Stored'])
field_values_2 = pd.DataFrame(columns=['Contract Name', 'SF_DB_COL_NAME', 'Actual Value Stored', 'Original Page Number'])
for file_name in contract_list:
document_name = [x for x in list(field_values['Contract Name']) if not pd.isna(x) and file_name.rsplit('/',1)[1].replace(' MU','').replace(
'_MU','').replace('.txt','') in x][0]
field_values_1 = field_values[field_values['Contract Name'] == document_name].head(1).transpose().reset_index()
field_values_1.columns = ['SF_DB_COL_NAME', 'Actual Value Stored']
field_values_p1 = field_values_1[~field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
field_values_p2 = field_values_1[field_values_1['SF_DB_COL_NAME'].str.endswith('_PG')]
field_values_p2.columns = ['SF_DB_COL_NAME', 'Original Page Number']
field_values_p2["SF_DB_COL_NAME"] = field_values_p2["SF_DB_COL_NAME"].str.replace("_PG", "")
field_values_1 = pd.merge(field_values_p1, field_values_p2, how ='left', on =['SF_DB_COL_NAME'])
field_values_1['Contract Name'] = document_name
field_values_2 = pd.concat([field_values_2, field_values_1], ignore_index = True)
df = pd.merge(df, field_values_2, how ='left', on =['Contract Name', 'SF_DB_COL_NAME'])
answer_list = list(df['New Extracted value'])
@@ -501,8 +513,8 @@ if user_mail in user_list:
result_list = [i==j for i, j in zip(actual_value_list, answer_list)]
df['Result'] = [str(x) for x in result_list]
df = df[~df['Contract ID'].isnull()]
if 'Original Page Number' not in df.columns:
df['Original Page Number'] = ' '
# if 'Original Page Number' not in df.columns:
# df['Original Page Number'] = ' '
df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level'
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
@@ -538,7 +550,7 @@ if user_mail in user_list:
csv_buf.seek(0)
s3_client.put_object(Bucket='doczy-dev-infra-raw-data-ingestion', Body=csv_buf.getvalue(), Key='training_interface/history.csv')
df = df[['Contract Name','Contract ID', 'SF_DB_COL_NAME', 'Actual Value Stored','New Extracted value','Confidence Level'
,'Snippet', 'New Page Number', 'Revised Prompt', 'Result']]
,'Snippet','Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']]
# df = pd.read_csv('results.csv')
# df['Result'] = df['Result'].astype('str')