Merged in feature/effective_date (pull request #323)
log information logic added to hotfix for contract effective date * log information logic added to hotfix for contract effective date Approved-by: Katon Minhas
This commit is contained in:
Siddhant Medar
committed by
Katon Minhas
Katon Minhas
parent
141c62c025
commit
d76c24211a
@@ -20,7 +20,24 @@ YYYY Mon DD (e.g., 2024 Dec 1)
|
||||
Accounts for month names in lowercase and uppercase
|
||||
"""
|
||||
|
||||
DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b'
|
||||
# original pattern
|
||||
# DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b'
|
||||
|
||||
# new pattern to include "day of" date format
|
||||
base_months = "January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec"
|
||||
|
||||
parts = [
|
||||
r'(?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])[-/](?:\d{4}|\d{2})', # MM/DD/YYYY
|
||||
r'\d{4}[-/](?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])', # YYYY/MM/DD
|
||||
r'(?:0?[1-9]|[12][0-9]|3[01])[-/](?:0?[1-9]|1[0-2])[-/](?:\d{4}|\d{2})', # DD/MM/YYYY
|
||||
f'(?:{base_months})\s+\d{{1,2}},\s*\d{{4}}', # Month DD, YYYY
|
||||
f'\d{{4}}\s+(?:{base_months})\s+\d{{1,2}}', # YYYY Month DD
|
||||
f'\d{{1,2}}\s+(?:{base_months})\s+\d{{4}}', # DD Month YYYY
|
||||
f'\d{{1,2}}(?:st|nd|rd|th)?[\s\n]+[dD]ay[\s\n]+[oO]f[\s\n]+(?:{base_months})[\s\n]*,[\s\n]*\d{{4}}' # D(st|nd|rd|th) day of Month, YYYY
|
||||
]
|
||||
|
||||
# Combine all parts with the word boundary and alternation
|
||||
DATE_PATTERN = r'\b(?:' + '|'.join(parts) + r')\b'
|
||||
|
||||
def chunk_hierarchical(
|
||||
text_dict: dict[str, str], keywords: list[str], case_sensitive: bool = False
|
||||
@@ -255,8 +272,7 @@ def chunk_with_include_exclude_keywords(
|
||||
dates_list = re.findall(DATE_PATTERN, page_content)
|
||||
|
||||
if len(dates_list) > 0:
|
||||
pages_containing_date.add(page_num)
|
||||
|
||||
pages_containing_date.add(page_num)
|
||||
pages_containing_date = sorted(pages_containing_date,key=int)
|
||||
|
||||
# debug lines for inclusion and exclusion keywords found on each page
|
||||
|
||||
@@ -449,51 +449,54 @@ def clean_irs(abc_df, filename, text_dict ):
|
||||
|
||||
|
||||
##### CONTRACT EFFECTIVE DATE #####
|
||||
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str:
|
||||
def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False):
|
||||
# perform smart chunking
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
page_list,d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"],
|
||||
exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"],
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
# meridian special case
|
||||
if is_meridian:
|
||||
if claude_answer_extracted == "N/A":
|
||||
# perform smart chunking with signature date pages included
|
||||
|
||||
included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
|
||||
|
||||
included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
|
||||
|
||||
excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
|
||||
|
||||
page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = included_keywords,
|
||||
exclude_keywords = excluded_keywords
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
|
||||
if len(page_list) > 0:
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
except Exception as e:
|
||||
raise
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
# meridian special case
|
||||
if is_meridian:
|
||||
if claude_answer_extracted == "N/A":
|
||||
# perform smart chunking with signature date pages included
|
||||
|
||||
included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"]
|
||||
|
||||
included_keywords = list(itertools.chain(included_keywords, ["Signature"]))
|
||||
|
||||
excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"]
|
||||
|
||||
page_list, d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict,
|
||||
include_keywords = included_keywords,
|
||||
exclude_keywords = excluded_keywords
|
||||
)
|
||||
|
||||
context = "\n".join(text_dict[page] for page in page_list)
|
||||
prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context)
|
||||
|
||||
try:
|
||||
claude_answer_raw = claude_funcs.invoke_claude(
|
||||
prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192)
|
||||
claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1]
|
||||
except Exception as e:
|
||||
raise
|
||||
|
||||
return claude_answer_extracted, d, page_list, claude_answer_raw
|
||||
|
||||
return claude_answer_extracted
|
||||
else:
|
||||
return "N/A", d, page_list, ""
|
||||
|
||||
def clean_contract_effective_date(
|
||||
df: pd.DataFrame,
|
||||
@@ -539,22 +542,24 @@ def clean_contract_effective_date(
|
||||
|
||||
if utils.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing
|
||||
try:
|
||||
answer = contract_effective_date_fix(file_name, text_dict,is_meridian)
|
||||
answer,d,page_list,claude_answer_raw = contract_effective_date_fix(file_name,text_dict,is_meridian)
|
||||
except Exception as e:
|
||||
print(f"Error processing {file_name}, got error: {e}")
|
||||
# print(f"Error processing {file_name}, got error: {e}")
|
||||
answer,d,page_list,claude_answer_raw = "N/A",{},[],""
|
||||
else:
|
||||
answer = contract_effective_date
|
||||
|
||||
if answer == "N/A":
|
||||
df["Contract Effective Date_corrected"] = answer
|
||||
else:
|
||||
try:
|
||||
corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix
|
||||
except utils.InvalidDateException as e:
|
||||
corrected_date = "N/A"
|
||||
try:
|
||||
corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix
|
||||
except utils.InvalidDateException as e:
|
||||
corrected_date = "N/A"
|
||||
# print(f"Effective Date for {file_name} --> {corrected_date}")
|
||||
|
||||
df["Contract Effective Date_corrected"] = corrected_date
|
||||
|
||||
df["Contract Effective Date_corrected"] = corrected_date
|
||||
df['Log Information'] = str(d)
|
||||
df['Pages selected'] = str(page_list)
|
||||
df["LLM Justification"] = claude_answer_raw
|
||||
|
||||
return df
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import re
|
||||
import ac_smart_chunking
|
||||
|
||||
regex_backticks = r"\|([^|]+)\|"
|
||||
|
||||
#TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently.
|
||||
@@ -28,3 +31,99 @@ def get_effective_date_meridian_prompt(context):
|
||||
"""
|
||||
|
||||
return prompt_template
|
||||
|
||||
|
||||
def chunk_with_include_exclude_keywords(
|
||||
text_dict: dict[str, str],
|
||||
include_keywords: list[str],
|
||||
exclude_keywords: list[str],
|
||||
case_sensitive: bool = False,
|
||||
apply_date_filtering: bool = True
|
||||
):
|
||||
|
||||
"""Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords.
|
||||
|
||||
Args:
|
||||
text_dict (dict[str, str]): A dictionary keyed by string page number and valued
|
||||
by string page contents
|
||||
include_keywords (list[str]): A list of string keywords to include
|
||||
exclude_keywords (list[str]): A list of string keywords to exclude
|
||||
case_sensitive (bool): A flag for converting all keywords and text to lower case,
|
||||
in order to ignore case altogether.
|
||||
apply_date_filtering (bool): A flag to filter out the pages not containing a date.
|
||||
|
||||
Returns:
|
||||
list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords
|
||||
"""
|
||||
|
||||
if not case_sensitive:
|
||||
include_keywords = [keyword.lower() for keyword in include_keywords]
|
||||
exclude_keywords = [keyword.lower() for keyword in exclude_keywords]
|
||||
text_dict = {key: value.lower() for key, value in text_dict.items()}
|
||||
|
||||
if apply_date_filtering:
|
||||
pages_containing_date = set()
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
dates_list = re.findall(ac_smart_chunking.DATE_PATTERN, page_content)
|
||||
|
||||
if len(dates_list) > 0:
|
||||
pages_containing_date.add(page_num)
|
||||
|
||||
pages_containing_date = sorted(pages_containing_date,key=int)
|
||||
|
||||
# debug lines for inclusion and exclusion keywords found on each page
|
||||
|
||||
# print(f"Pages containing date: {pages_containing_date}")
|
||||
|
||||
d = list() # to record details
|
||||
|
||||
for page_num, page_content in text_dict.items():
|
||||
if page_num in pages_containing_date:
|
||||
inclusion_keywords = set()
|
||||
exclusion_keywords = set()
|
||||
|
||||
contains_inclusion_keyword = False
|
||||
|
||||
for keyword in include_keywords:
|
||||
contains_inclusion_keyword = True
|
||||
if keyword in page_content:
|
||||
inclusion_keywords.add(keyword)
|
||||
|
||||
for keyword in exclude_keywords:
|
||||
if keyword in page_content:
|
||||
exclusion_keywords.add(keyword)
|
||||
|
||||
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords} | Inclusion Flag: {contains_inclusion_keyword}")
|
||||
|
||||
if contains_inclusion_keyword:
|
||||
d.append({"Page Number": page_num, "Inclusion Keywords": inclusion_keywords, "Exclusion Keywords": exclusion_keywords})
|
||||
|
||||
# print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}")
|
||||
|
||||
page_list = []
|
||||
|
||||
for page in text_dict.keys():
|
||||
if not apply_date_filtering or (apply_date_filtering and page in pages_containing_date):
|
||||
if any(keyword in text_dict[page] for keyword in include_keywords) and not any(
|
||||
keyword in text_dict[page] for keyword in exclude_keywords
|
||||
):
|
||||
# if str(int(page) - 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) - 1))
|
||||
# print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)")
|
||||
|
||||
page_list.append(page)
|
||||
# print(f"Taking page #: {page}")
|
||||
|
||||
# if str(int(page) + 1) in text_dict.keys():
|
||||
# page_list.append(str(int(page) + 1))
|
||||
# # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)")
|
||||
|
||||
page_list_sorted = sorted([int(page_str) for page_str in set(page_list)])
|
||||
page_list_sorted_str = list(map(str, page_list_sorted))
|
||||
|
||||
# print(", ".join([page for page in page_list_sorted_str]))
|
||||
|
||||
# print(f"Obtained total pages: {len(page_list_sorted_str)}")
|
||||
|
||||
return page_list_sorted_str, d
|
||||
Reference in New Issue
Block a user