jacob-mathison/doczyai-pipelines
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

find page no. using llm

Browse Source
This commit is contained in:
Mayank Aamseek
2024-04-06 14:49:24 +05:30
parent 2f05f90d6a
commit b461a6d342
1 changed files with 47 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
streamlit/interface_3.py
+47 -23
View File
@@ -279,7 +279,16 @@ if st.session_state.user_info['mail'] in user_list:
region_name="us-east-1",
)
question = prompt
# question = prompt
if mode == 'Multiple fields':
prompt_dict = json.loads(prompt)
prompt_dict_pg = prompt_dict | {str(k)+'_PG': "On which page can I find answer to the question - "+str(
v) for k, v in prompt_dict.items()}
question = json.dumps(dict(sorted(prompt_dict_pg.items())))
else:
question = json.dumps({field: prompt, str(field)+'_PG': "On which page can I find answer to the question - "+str(prompt)})
# st.write(question)
question_with_schema = question
attempt = 0
@@ -303,6 +312,7 @@ if st.session_state.user_info['mail'] in user_list:
data = s3_client.get_object(Bucket=bucket, Key=contract)
contents = data['Body'].read()
context = contents.decode("utf-8")
# st.write(question_with_schema)
# Add "You must answer in correct JSON format."
# Add Answer in JSON format: {{
@@ -316,7 +326,7 @@ if st.session_state.user_info['mail'] in user_list:
Question: {question}
Answer: Answer in JSON format: {{"""
parameters = {
"maxTokenCount":1024,
"maxTokenCount":1280,
"stopSequences":[],
"temperature":0,
"topP":0.9
@@ -336,7 +346,7 @@ if st.session_state.user_info['mail'] in user_list:
Answer: Answer in JSON format: {{"""
payload={
"prompt":"[INST]"+ prompt_data +"[/INST]",
"max_gen_len":1024,
"max_gen_len":1280,
"temperature":0.0,
"top_p":0.9
}
@@ -358,7 +368,7 @@ if st.session_state.user_info['mail'] in user_list:
Assistant: Answer in JSON format: {{"""
body = json.dumps(
{"prompt": anthropic.HUMAN_PROMPT + prompt_data + anthropic.AI_PROMPT,
"max_tokens_to_sample": 1024,
"max_tokens_to_sample": 1280,
"temperature":0.0,
"top_p":1,
"top_k":250,
@@ -414,19 +424,28 @@ if st.session_state.user_info['mail'] in user_list:
else:
response_dict = {field:response_text.strip("{").strip("}")}
if mode == 'Multiple fields':
field_l = list(response_dict.keys())
answer_l = list(response_dict.values())
else:
field_l = [field]
# answer = response_dict.get(field, " ")
try:
if isinstance(response_dict, dict):
answer_l = list(response_dict.values())[:1]
else:
answer_l = list(response_dict)[:1]
except:
answer_l = [response_dict]
# if mode == 'Multiple fields':
# field_l = list(response_dict.keys())
# answer_l = list(response_dict.values())
# else:
# field_l = [field]
# # answer = response_dict.get(field, " ")
# try:
# if isinstance(response_dict, dict):
# answer_l = list(response_dict.values())[:1]
# else:
# answer_l = list(response_dict)[:1]
# except:
# answer_l = [response_dict]
field_l = list(response_dict.keys())
answer_l = list(response_dict.values())
field_dict = {k: v for k, v in response_dict.items() if not k.endswith('_PG')}
page_dict = {k: v for k, v in response_dict.items() if k.endswith('_PG')}
page_dict = {k[:-3]: v for k, v in response_dict.items()}
field_l = list(field_dict.keys())
answer_l = list(field_dict.values())
page_no_l = [page_dict.get(x, "") for x in field_l]
field_list.extend(field_l)
answer_list.extend(answer_l)
@@ -436,13 +455,18 @@ if st.session_state.user_info['mail'] in user_list:
# page_no = " " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit(
# "Start of Page No. = ", 1)) > 1 else context[:location].rsplit("Start of Page No. = ", 1)[0]
# page_no = re.search(r'\d+', page_no).group() if page_no != " " and re.search(r'\d+', page_no) is not None else ""
location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in answer_l]
try:
location_l = [context.find(a, context.find("Start of Page No. = "+str(p))) if isinstance(
a, str) and a != "" else -1 for a, p in zip(answer_l, page_no_l)]
except:
location_l = [context.find(answer) if isinstance(answer, str) and answer != "" else -1 for answer in answer_l]
snippet_l = [' '.join(context[:location].split('.')[-4:]) + ' ' + ' '.join(context[location:].split('. ')[:5]
) if location != -1 else ' ' for location in location_l]
page_no_l = [" " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit(
"Start of Page No. = ", 1)) > 1 else context[:location].rsplit("Start of Page No. = ", 1)[0] for location in location_l]
# st.write(location_list)
page_no_l = [re.search(r'\d+', page).group() if page != " " and re.search(r'\d+', page) is not None else "" for page in page_no_l]
# page_no_l = [" " if location == -1 else context[:location].rsplit("Start of Page No. = ", 1)[1] if len(context[:location].rsplit(
# "Start of Page No. = ", 1)) > 1 else context[:location].rsplit("Start of Page No. = ", 1)[0] for location in location_l]
# # st.write(location_list)
# page_no_l = [re.search(r'\d+', page).group() if page != " " and re.search(r'\d+', page) is not None else "" for page in page_no_l]
snippet_list.extend(snippet_l)
page_no_list.extend(page_no_l)
@@ -484,7 +508,7 @@ if st.session_state.user_info['mail'] in user_list:
# to be deleted later
# contract_list_f = [contract.rsplit('/',1)[1].replace(' MU','').replace('_MU','').replace('.txt','') for contract in contract_list_f]
contract_list_f = [contract.rsplit('/',1)[1].replace('.txt','pdf') for contract in contract_list_f]
contract_list_f = [contract.rsplit('/',1)[1].replace('.txt','.pdf') for contract in contract_list_f]
df['Contract Name'] = contract_list_f
df['New Extracted value'] = answer_list