Katon Minhas
64e754d40f
Refactor/hotfix refactor * Initial hotfix refactor * Fix references * Remove test Approved-by: Alex Galarce
139 lines
5.2 KiB
Python
139 lines
5.2 KiB
Python
import re
|
|
|
|
import smart_chunking_funcs
|
|
|
|
regex_backticks = r"\|([^|]+)\|"
|
|
|
|
|
|
# TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently.
|
|
def get_effective_date_meridian_prompt(context):
|
|
prompt_template = f""""
|
|
Please analyze the following contract and extract ONLY the Signature Date. Follow these precise rules in order:
|
|
|
|
1. Look for Signature Date
|
|
|
|
2. If multiple matches are found:
|
|
- Prioritize the latest full date
|
|
|
|
3. If only partial dates are found, return N/A.
|
|
|
|
4. Return the date in YYYY-MM-DD format or N/A if date not found. Enclose just your final answer in |pipes|.
|
|
|
|
Here is the contract to analyze:
|
|
|
|
## START CONTRACT TEXT ##
|
|
|
|
{context}
|
|
|
|
## END CONTRACT TEXT ##
|
|
|
|
Before returning the output, VERIFY the date format is YYYY-MM-DD. If the date is not found, return N/A.
|
|
|
|
Finally state "Answer: |YYYY-MM-DD| or |N/A|"
|
|
"""
|
|
|
|
return prompt_template
|
|
|
|
|
|
def chunk_with_include_exclude_keywords(
|
|
text_dict: dict[str, str],
|
|
include_keywords: list[str],
|
|
exclude_keywords: list[str],
|
|
case_sensitive: bool = False,
|
|
apply_date_filtering: bool = True,
|
|
):
|
|
"""Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords.
|
|
|
|
Args:
|
|
text_dict (dict[str, str]): A dictionary keyed by string page number and valued
|
|
by string page contents
|
|
include_keywords (list[str]): A list of string keywords to include
|
|
exclude_keywords (list[str]): A list of string keywords to exclude
|
|
case_sensitive (bool): A flag for converting all keywords and text to lower case,
|
|
in order to ignore case altogether.
|
|
apply_date_filtering (bool): A flag to filter out the pages not containing a date.
|
|
|
|
Returns:
|
|
list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords
|
|
"""
|
|
|
|
if not case_sensitive:
|
|
include_keywords = [keyword.lower() for keyword in include_keywords]
|
|
exclude_keywords = [keyword.lower() for keyword in exclude_keywords]
|
|
text_dict = {key: value.lower() for key, value in text_dict.items()}
|
|
|
|
if apply_date_filtering:
|
|
pages_containing_date = set()
|
|
|
|
for page_num, page_content in text_dict.items():
|
|
dates_list = re.findall(smart_chunking_funcs.DATE_PATTERN, page_content)
|
|
|
|
if len(dates_list) > 0:
|
|
pages_containing_date.add(page_num)
|
|
|
|
pages_containing_date = sorted(pages_containing_date, key=int)
|
|
|
|
# debug lines for inclusion and exclusion keywords found on each page
|
|
|
|
# print(f"Pages containing date: {pages_containing_date}")
|
|
|
|
d = list() # to record details
|
|
|
|
for page_num, page_content in text_dict.items():
|
|
if page_num in pages_containing_date:
|
|
inclusion_keywords = set()
|
|
exclusion_keywords = set()
|
|
|
|
contains_inclusion_keyword = False
|
|
|
|
for keyword in include_keywords:
|
|
contains_inclusion_keyword = True
|
|
if keyword in page_content:
|
|
inclusion_keywords.add(keyword)
|
|
|
|
for keyword in exclude_keywords:
|
|
if keyword in page_content:
|
|
exclusion_keywords.add(keyword)
|
|
|
|
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords} | Inclusion Flag: {contains_inclusion_keyword}")
|
|
|
|
if contains_inclusion_keyword:
|
|
d.append(
|
|
{
|
|
"Page Number": page_num,
|
|
"Inclusion Keywords": inclusion_keywords,
|
|
"Exclusion Keywords": exclusion_keywords,
|
|
}
|
|
)
|
|
|
|
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}")
|
|
|
|
page_list = []
|
|
|
|
for page in text_dict.keys():
|
|
if not apply_date_filtering or (
|
|
apply_date_filtering and page in pages_containing_date
|
|
):
|
|
if any(
|
|
keyword in text_dict[page] for keyword in include_keywords
|
|
) and not any(keyword in text_dict[page] for keyword in exclude_keywords):
|
|
# if str(int(page) - 1) in text_dict.keys():
|
|
# page_list.append(str(int(page) - 1))
|
|
# print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)")
|
|
|
|
page_list.append(page)
|
|
# print(f"Taking page #: {page}")
|
|
|
|
# if str(int(page) + 1) in text_dict.keys():
|
|
# page_list.append(str(int(page) + 1))
|
|
# # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)")
|
|
|
|
page_list_sorted = sorted([int(page_str) for page_str in set(page_list)])
|
|
page_list_sorted_str = list(map(str, page_list_sorted))
|
|
|
|
# print(", ".join([page for page in page_list_sorted_str]))
|
|
|
|
# print(f"Obtained total pages: {len(page_list_sorted_str)}")
|
|
|
|
return page_list_sorted_str, d
|