diff --git a/fieldExtraction/src/ac_smart_chunking.py b/fieldExtraction/src/ac_smart_chunking.py index cc5b717..3ae2dd2 100644 --- a/fieldExtraction/src/ac_smart_chunking.py +++ b/fieldExtraction/src/ac_smart_chunking.py @@ -1,43 +1,25 @@ import re """ -Note: -The below pattern will match with the following date formats: -Numeric Formats -MM/DD/YYYY or MM-DD-YYYY (e.g., 11/18/2024, 11-18-2024) -MM/DD/YY or MM-DD-YY (e.g., 11/18/24, 11-18-24) -YYYY/MM/DD or YYYY-MM-DD (e.g., 2024/11/18, 2024-11-18) -DD/MM/YY or DD-MM-YY (e.g., 18/11/24, 18-11-24) -Textual Formats -Month DD, YYYY (e.g., December 1, 2024) -Month D, YYYY (e.g., December 1, 2024 — with a single-digit day) -YYYY Month DD (e.g., 2024 December 1) -Abbreviated Month Formats -Mon DD, YYYY (e.g., Dec 1, 2024) -DD Mon YYYY (e.g., 1 Dec 2024) -YYYY Mon DD (e.g., 2024 Dec 1) +Date pattern regex: +Matches for "day of", +any full month name (e.g., "January", "February", ...) +any month abbreviation (e.g., "Jan", "Feb", ...) +date formats: +MM/DD/YYYY (e.g., 12/25/2023) +MM.DD.YYYY (e.g., 12.25.2023) +MM-DD-YYYY (e.g., 12-25-2023) +YYYY/MM/DD (e.g., 2024/12/09) +YYYY.MM.DD (e.g., 2024.12.09) +YYYY-MM-DD (e.g., 2024-12-09) +YYYYMMDD (e.g., 20241209) +MMYY or YYMM format (e.g., 1224 for December 2024 or 2405 for May 2024) +YYYY.MM.DD (e.g., 2024.12.09) -Accounts for month names in lowercase and uppercase +Case insensitive """ -# original pattern -# DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b' - -# new pattern to include "day of" date format -base_months = "January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec" - -parts = [ - r'(?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])[-/](?:\d{4}|\d{2})', # MM/DD/YYYY - r'\d{4}[-/](?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])', # YYYY/MM/DD - r'(?:0?[1-9]|[12][0-9]|3[01])[-/](?:0?[1-9]|1[0-2])[-/](?:\d{4}|\d{2})', # DD/MM/YYYY - f'(?:{base_months})\s+\d{{1,2}},\s*\d{{4}}', # Month DD, YYYY - f'\d{{4}}\s+(?:{base_months})\s+\d{{1,2}}', # YYYY Month DD - f'\d{{1,2}}\s+(?:{base_months})\s+\d{{4}}', # DD Month YYYY - f'\d{{1,2}}(?:st|nd|rd|th)?[\s\n]+[dD]ay[\s\n]+[oO]f[\s\n]+(?:{base_months})[\s\n]*,[\s\n]*\d{{4}}' # D(st|nd|rd|th) day of Month, YYYY -] - -# Combine all parts with the word boundary and alternation -DATE_PATTERN = r'\b(?:' + '|'.join(parts) + r')\b' +DATE_PATTERN = r"(?i)\bday of\b|\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\b|\b(?:\d{1,2}[\/\.-]\d{1,2}[\/\.-](?:\d{2}|\d{4})|\d{4}[\/\.-]\d{1,2}[\/\.-]\d{1,2}|\d{8}|\d{2}\d{2}|\d{4}[\/\.-]\d{1,2}[\/\.-]\d{1,2}|\d{4}[\.]\d{1,2}[\.]\d{1,2})\b" def chunk_hierarchical( text_dict: dict[str, str], keywords: list[str], case_sensitive: bool = False diff --git a/fieldExtraction/tests/ac_smart_chunking_test.py b/fieldExtraction/tests/ac_smart_chunking_test.py new file mode 100644 index 0000000..c3bb08b --- /dev/null +++ b/fieldExtraction/tests/ac_smart_chunking_test.py @@ -0,0 +1,112 @@ +import pytest +import re + +from ac_smart_chunking import DATE_PATTERN + +legal_traditional_dates = [ + "This 12th day of December, 2024", + "The 23RD DAY of JANUARY, 2026", + "The 0th dAy of JaNuArY, 1000", + "12th of December, Two Thousand and Twenty-Four", + "Thirteenth of DeCeMbEr, Two Thousand and Sixteen", + "Thirteenth of DeCeMbEr, Nineteen Hundred and Sixty Seven", + "Thirteenth of DeCeMbEr, One Thousand, Nine Hundred, and Sixty Seven", + "The 12th of December, 2024", + "The 145th of January, 2023", + "12th day of December, 2024", + "1st day of January, 2025", + "2nd day of JaNuArY, 2025", + "3rd day of JANUARY, 2025", + "21st day of January, 2025", + "22nd day of January, 2025", + "On 12 December 2025", +] + +numeric_dates = [ + "20.12.2024", + "01.01.1000", + "05.12.06", + "12.18.2024", + "20241212", + "11/18/2014", + "11-18-2024", + "11/18/24", + "11-18-24", + "2024/11/18", + "2024-11-18", + "18/11/24", + "1224", + "2024.12.09", +] + +abbreviated_dates = [ + "18-Nov-2024", + "Nov-18-2024", + "Dec/12/2024", + "Dec.12.2024", + "Dec/2024", +] + +contextual_dates = [ + "Friday, December 18, 2024", + "Fri, Dec 18, 2024", +] + +compact_dates = [ + "12-December-2024", + "December 2024", + "Dec-12-2024", +] + +case_variations = [ + "friday, december 18, 2024", + "FRI, DEC 18, 2024", + "DECEMBER 2024", + # Multiple spaces + "Friday, December 18, 2024", + "Dec 12, 2024", + # Multiple newlines + "Friday,\nDecember\n18,\n2024", + "Dec\n12,\n2024", + # Original formats with newlines + "1 day\nof December\n,\n2006", + "1\nday\nof\nDecember\n,\n2006", + "1st day\nof\nDecember,\n2006", +] + +ordinal_dates = [ + "1st day of December, 2006", + "1st Day of December,2006", + "2nd day of January, 2024", + "2nd Day of Jan,2024", + "3rd day of November, 1999", + "3rd Day of Nov,1999", + "4th day of March, 2023", + "4th Day of Mar,2023", + "21st day of April, 2022", + "22nd day of May,2022", + "23rd day of June, 2021", + "24th Day of July,2021", +] + +invalid_dates = [ + "Invalid date", + "Not a date", + "Decide", # make sure "dec" doesn't accidentally match to "decide" or other words + "decided", + "Novel", # make sure "nov" doesn't accidentally match to other words + "novelty", +] + + +def test_valid_dates(): + for date_list in [legal_traditional_dates, numeric_dates, abbreviated_dates, + contextual_dates, compact_dates, case_variations, ordinal_dates]: + for date in date_list: + match = re.findall(DATE_PATTERN, date) + assert match + +def test_invalid_dates(): + for date in invalid_dates: + match = re.findall(DATE_PATTERN, date) + assert not match \ No newline at end of file