diff --git a/fieldExtraction/src/ac_smart_chunking.py b/fieldExtraction/src/ac_smart_chunking.py index c9b803e..cc5b717 100644 --- a/fieldExtraction/src/ac_smart_chunking.py +++ b/fieldExtraction/src/ac_smart_chunking.py @@ -20,7 +20,24 @@ YYYY Mon DD (e.g., 2024 Dec 1) Accounts for month names in lowercase and uppercase """ -DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b' +# original pattern +# DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b' + +# new pattern to include "day of" date format +base_months = "January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec" + +parts = [ + r'(?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])[-/](?:\d{4}|\d{2})', # MM/DD/YYYY + r'\d{4}[-/](?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])', # YYYY/MM/DD + r'(?:0?[1-9]|[12][0-9]|3[01])[-/](?:0?[1-9]|1[0-2])[-/](?:\d{4}|\d{2})', # DD/MM/YYYY + f'(?:{base_months})\s+\d{{1,2}},\s*\d{{4}}', # Month DD, YYYY + f'\d{{4}}\s+(?:{base_months})\s+\d{{1,2}}', # YYYY Month DD + f'\d{{1,2}}\s+(?:{base_months})\s+\d{{4}}', # DD Month YYYY + f'\d{{1,2}}(?:st|nd|rd|th)?[\s\n]+[dD]ay[\s\n]+[oO]f[\s\n]+(?:{base_months})[\s\n]*,[\s\n]*\d{{4}}' # D(st|nd|rd|th) day of Month, YYYY +] + +# Combine all parts with the word boundary and alternation +DATE_PATTERN = r'\b(?:' + '|'.join(parts) + r')\b' def chunk_hierarchical( text_dict: dict[str, str], keywords: list[str], case_sensitive: bool = False @@ -255,8 +272,7 @@ def chunk_with_include_exclude_keywords( dates_list = re.findall(DATE_PATTERN, page_content) if len(dates_list) > 0: - pages_containing_date.add(page_num) - + pages_containing_date.add(page_num) pages_containing_date = sorted(pages_containing_date,key=int) # debug lines for inclusion and exclusion keywords found on each page diff --git a/fieldExtraction/src/cnc_hotfix.py b/fieldExtraction/src/cnc_hotfix.py index 91c6428..6d33498 100644 --- a/fieldExtraction/src/cnc_hotfix.py +++ b/fieldExtraction/src/cnc_hotfix.py @@ -449,51 +449,54 @@ def clean_irs(abc_df, filename, text_dict ): ##### CONTRACT EFFECTIVE DATE ##### -def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False) -> str: +def contract_effective_date_fix(contract_name: str, text_dict: dict[str, str],is_meridian: bool=False): # perform smart chunking - page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict, + page_list,d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict, include_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"], exclude_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"], ) - - context = "\n".join(text_dict[page] for page in page_list) - prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian - - try: - claude_answer_raw = claude_funcs.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) - claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1] - - except Exception as e: - raise - - # meridian special case - if is_meridian: - if claude_answer_extracted == "N/A": - # perform smart chunking with signature date pages included - - included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"] - - included_keywords = list(itertools.chain(included_keywords, ["Signature"])) - - excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"] - - page_list = ac_smart_chunking.chunk_with_include_exclude_keywords(text_dict, - include_keywords = included_keywords, - exclude_keywords = excluded_keywords - ) - - context = "\n".join(text_dict[page] for page in page_list) - prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context) + if len(page_list) > 0: + context = "\n".join(text_dict[page] for page in page_list) + prompt = prompts.get_effective_date_prompt(context) # get effective date - this is for both meridian and non-meridian - try: - claude_answer_raw = claude_funcs.invoke_claude( - prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) - claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1] - except Exception as e: - raise + try: + claude_answer_raw = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) + claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1] + + except Exception as e: + raise + + # meridian special case + if is_meridian: + if claude_answer_extracted == "N/A": + # perform smart chunking with signature date pages included + + included_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["included_keywords"] + + included_keywords = list(itertools.chain(included_keywords, ["Signature"])) + + excluded_keywords = keywords.GROUPED_KEYWORD_MAPPINGS["CONTRACT_EFFECTIVE_DT"]["excluded_keywords"] + + page_list, d = cnc_hotfix_effective_date_utils.chunk_with_include_exclude_keywords(text_dict, + include_keywords = included_keywords, + exclude_keywords = excluded_keywords + ) + + context = "\n".join(text_dict[page] for page in page_list) + prompt = cnc_hotfix_effective_date_utils.get_effective_date_meridian_prompt(context) + + try: + claude_answer_raw = claude_funcs.invoke_claude( + prompt, config.MODEL_ID_CLAUDE35_SONNET, contract_name, 8192) + claude_answer_extracted = re.findall(pattern=cnc_hotfix_effective_date_utils.regex_backticks, string=claude_answer_raw)[-1] + except Exception as e: + raise + + return claude_answer_extracted, d, page_list, claude_answer_raw - return claude_answer_extracted + else: + return "N/A", d, page_list, "" def clean_contract_effective_date( df: pd.DataFrame, @@ -539,22 +542,24 @@ def clean_contract_effective_date( if utils.is_empty(contract_effective_date): # Try prompting (with smart chunking) when the contract effective date is missing try: - answer = contract_effective_date_fix(file_name, text_dict,is_meridian) + answer,d,page_list,claude_answer_raw = contract_effective_date_fix(file_name,text_dict,is_meridian) except Exception as e: - print(f"Error processing {file_name}, got error: {e}") + # print(f"Error processing {file_name}, got error: {e}") + answer,d,page_list,claude_answer_raw = "N/A",{},[],"" else: answer = contract_effective_date - if answer == "N/A": - df["Contract Effective Date_corrected"] = answer - else: - try: - corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix - except utils.InvalidDateException as e: - corrected_date = "N/A" + try: + corrected_date = utils.convert_to_us_date_format(answer) # apply date formatting fix + except utils.InvalidDateException as e: + corrected_date = "N/A" + # print(f"Effective Date for {file_name} --> {corrected_date}") - df["Contract Effective Date_corrected"] = corrected_date - + df["Contract Effective Date_corrected"] = corrected_date + df['Log Information'] = str(d) + df['Pages selected'] = str(page_list) + df["LLM Justification"] = claude_answer_raw + return df diff --git a/fieldExtraction/src/cnc_hotfix_effective_date_utils.py b/fieldExtraction/src/cnc_hotfix_effective_date_utils.py index c3678e9..e99350a 100644 --- a/fieldExtraction/src/cnc_hotfix_effective_date_utils.py +++ b/fieldExtraction/src/cnc_hotfix_effective_date_utils.py @@ -1,3 +1,6 @@ +import re +import ac_smart_chunking + regex_backticks = r"\|([^|]+)\|" #TODO: leave this one in here; don't move it to prompts until we get confirmation that we want to handle meridian differently. @@ -28,3 +31,99 @@ def get_effective_date_meridian_prompt(context): """ return prompt_template + + +def chunk_with_include_exclude_keywords( + text_dict: dict[str, str], + include_keywords: list[str], + exclude_keywords: list[str], + case_sensitive: bool = False, + apply_date_filtering: bool = True + ): + + """Look for pages that include any of the include_keywords and exclude any pages that contain any of the exclude_keywords. + + Args: + text_dict (dict[str, str]): A dictionary keyed by string page number and valued + by string page contents + include_keywords (list[str]): A list of string keywords to include + exclude_keywords (list[str]): A list of string keywords to exclude + case_sensitive (bool): A flag for converting all keywords and text to lower case, + in order to ignore case altogether. + apply_date_filtering (bool): A flag to filter out the pages not containing a date. + + Returns: + list[str]: The sorted list of string page numbers that include all include_keywords and exclude any exclude_keywords + """ + + if not case_sensitive: + include_keywords = [keyword.lower() for keyword in include_keywords] + exclude_keywords = [keyword.lower() for keyword in exclude_keywords] + text_dict = {key: value.lower() for key, value in text_dict.items()} + + if apply_date_filtering: + pages_containing_date = set() + + for page_num, page_content in text_dict.items(): + dates_list = re.findall(ac_smart_chunking.DATE_PATTERN, page_content) + + if len(dates_list) > 0: + pages_containing_date.add(page_num) + + pages_containing_date = sorted(pages_containing_date,key=int) + + # debug lines for inclusion and exclusion keywords found on each page + + # print(f"Pages containing date: {pages_containing_date}") + + d = list() # to record details + + for page_num, page_content in text_dict.items(): + if page_num in pages_containing_date: + inclusion_keywords = set() + exclusion_keywords = set() + + contains_inclusion_keyword = False + + for keyword in include_keywords: + contains_inclusion_keyword = True + if keyword in page_content: + inclusion_keywords.add(keyword) + + for keyword in exclude_keywords: + if keyword in page_content: + exclusion_keywords.add(keyword) + + # print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords} | Inclusion Flag: {contains_inclusion_keyword}") + + if contains_inclusion_keyword: + d.append({"Page Number": page_num, "Inclusion Keywords": inclusion_keywords, "Exclusion Keywords": exclusion_keywords}) + + # print(f"Page #: {page_num} contains inclusion keywords: {inclusion_keywords} and contain exclusion keywords: {exclusion_keywords}") + + page_list = [] + + for page in text_dict.keys(): + if not apply_date_filtering or (apply_date_filtering and page in pages_containing_date): + if any(keyword in text_dict[page] for keyword in include_keywords) and not any( + keyword in text_dict[page] for keyword in exclude_keywords + ): + # if str(int(page) - 1) in text_dict.keys(): + # page_list.append(str(int(page) - 1)) + # print(f"Taking page #: {str(int(page) - 1)} (as buffer for following page)") + + page_list.append(page) + # print(f"Taking page #: {page}") + + # if str(int(page) + 1) in text_dict.keys(): + # page_list.append(str(int(page) + 1)) + # # print(f"Taking page #: {str(int(page) + 1)} (as buffer for previous page)") + + page_list_sorted = sorted([int(page_str) for page_str in set(page_list)]) + page_list_sorted_str = list(map(str, page_list_sorted)) + + # print(", ".join([page for page in page_list_sorted_str])) + + # print(f"Obtained total pages: {len(page_list_sorted_str)}") + + return page_list_sorted_str, d \ No newline at end of file