From 3544ac82098eb33b703ffdba165e86896e94862c Mon Sep 17 00:00:00 2001 From: Mayank Aamseek Date: Tue, 5 Mar 2024 22:28:13 +0530 Subject: [PATCH] edited postprocessing --- streamlit/history.csv | 7 +++++++ streamlit/interface_3.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/streamlit/history.csv b/streamlit/history.csv index dc8bede..2506474 100644 --- a/streamlit/history.csv +++ b/streamlit/history.csv @@ -4,3 +4,10 @@ Associated Base Contract Effective Date,10,,2024-03-05 20:09:20,,1 Associated Base Contract Effective Date,10,,2024-03-05 20:10:33,,1 Associated Base Contract Effective Date,10,,2024-03-05 20:11:30,0.0,1 Associated Base Contract Effective Date,10,,2024-03-05 20:17:56,0.0,1 +Associated Base Contract Effective Date,10,,2024-03-05 22:13:25,0.0,1 +Associated Base Contract Effective Date,10,,2024-03-05 22:14:15,0.0,1 +Associated Base Contract Effective Date,10,,2024-03-05 22:16:45,0.0,1 +Associated Base Contract Effective Date,10,,2024-03-05 22:17:35,0.0,1 +Associated Base Contract Effective Date,20,,2024-03-05 22:18:38,0.0,1 +Associated Base Contract Effective Date,20,,2024-03-05 22:21:28,0.0,1 +Contract Auto Renewal (Y/N)?,20,,2024-03-05 22:22:33,0.0,1 diff --git a/streamlit/interface_3.py b/streamlit/interface_3.py index 26502eb..b21ec30 100644 --- a/streamlit/interface_3.py +++ b/streamlit/interface_3.py @@ -209,7 +209,7 @@ st.session_state["LLM"] = LLM # df = pd.DataFrame(columns=['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet', # 'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']) -df = pd.DataFrame(columns=['Contract Name','New Extracted value','Confidence Level','Snippet','New Page Number' +df = pd.DataFrame(columns=['Contract Name','Raw value','New Extracted value','Confidence Level','Snippet','New Page Number' , 'Revised Prompt', 'Result']) try: history = pd.read_csv('history.csv') @@ -217,6 +217,12 @@ except: history = pd.DataFrame(columns=['Field Name','# Contracts Tested', 'Username', 'Date/Time', 'Accuracy', 'Attempt #']) attempt = 0 +if llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']: + k_value = 14 +else: + k_value = 25 + + if st.button("Test Configuration"): answer_list = [] @@ -226,7 +232,7 @@ if st.button("Test Configuration"): attempt = attempt + 1 for page_list in page_list_all: - RETRIEVER = st.session_state.DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": 14}) + RETRIEVER = st.session_state.DB.as_retriever(search_kwargs={"filter": {"$or": page_list}, "k": k_value}) QA = RetrievalQA.from_chain_type( llm=st.session_state["LLM"], chain_type="stuff", @@ -234,7 +240,7 @@ if st.button("Test Configuration"): return_source_documents=True, # chain_type_kwargs={"prompt": prompt, "memory": None}, ) - score = st.session_state.DB.similarity_search_with_relevance_scores(prompt, k=14, filter={"$or": page_list}) + score = st.session_state.DB.similarity_search_with_relevance_scores(prompt, k=4, filter={"$or": page_list}) score_list.append(max(d[1] for d in score)) response = QA(prompt) answer, docs = response["result"], response["source_documents"] @@ -242,6 +248,7 @@ if st.button("Test Configuration"): doc_list.append(docs) response_list.append(response) + df['Raw value'] = answer_list # post-processing if 'Date' in field: date_list = [] @@ -249,14 +256,24 @@ if st.button("Test Configuration"): try: extracted_date = dateutil.parser.parse(str(answer).replace('"',''), fuzzy=True).date() except: - extracted_date = "N/A" + extracted_date = " " date_list.append(extracted_date) answer_list = date_list + elif llm_selected in ['Llama 2 Chat 13B', 'Llama 2 Chat 70B']: + answer_list = [answer.rstrip(".") for answer in answer_list] + answer_list = [answer if "I don't know" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "N/A" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "does not contain" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "None" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "Not specified in the contract" not in str(answer) else " " for answer in answer_list] + answer_list = [answer if "Not applicable" not in str(answer) else " " for answer in answer_list] + elif llm_selected in ['Claude 2', 'Claude Instant']: + answer_list = [answer if "Unfortunately, I do not have enough context" not in str(answer) else " " for answer in answer_list] + answer_list = [answer.rstrip(".") for answer in answer_list] else: answer_list = [answer.rstrip(".") for answer in answer_list] - # I don't know; does not contain; "None"; Not specified in the contract; Not applicable; N/A # answer_list = [str(x).rsplit(':',1)[0] if len(str(x).rsplit(':',1)) < 2 else str(x).rsplit(':',1)[1] for x in answer_list] - + df['Contract Name'] = contract_list # to be deleted later df['Contract Name'] = [contract.replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list] @@ -275,10 +292,10 @@ if st.button("Test Configuration"): df = df[~df['Contract ID'].isnull()] if 'Original Page Number' in df.columns: - df = df[['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet' + df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet' ,'Original Page Number', 'New Page Number', 'Revised Prompt', 'Result']] else: - df = df[['Contract Name','Contract ID','Actual Value Stored','New Extracted value','Confidence Level','Snippet' + df = df[['Contract Name','Contract ID','Actual Value Stored','Raw value','New Extracted value','Confidence Level','Snippet' , 'New Page Number', 'Revised Prompt', 'Result']] try: