Merged in hotfix/cleaned-new-date-regex (pull request #325)
Hotfix/cleaned new date regex * new date pattern * add tests Approved-by: Katon Minhas
This commit is contained in:
Alex Galarce
committed by
Katon Minhas
Katon Minhas
parent
d62e477436
commit
c6a854b2b9
@@ -1,43 +1,25 @@
|
||||
import re
|
||||
|
||||
"""
|
||||
Note:
|
||||
The below pattern will match with the following date formats:
|
||||
Numeric Formats
|
||||
MM/DD/YYYY or MM-DD-YYYY (e.g., 11/18/2024, 11-18-2024)
|
||||
MM/DD/YY or MM-DD-YY (e.g., 11/18/24, 11-18-24)
|
||||
YYYY/MM/DD or YYYY-MM-DD (e.g., 2024/11/18, 2024-11-18)
|
||||
DD/MM/YY or DD-MM-YY (e.g., 18/11/24, 18-11-24)
|
||||
Textual Formats
|
||||
Month DD, YYYY (e.g., December 1, 2024)
|
||||
Month D, YYYY (e.g., December 1, 2024 — with a single-digit day)
|
||||
YYYY Month DD (e.g., 2024 December 1)
|
||||
Abbreviated Month Formats
|
||||
Mon DD, YYYY (e.g., Dec 1, 2024)
|
||||
DD Mon YYYY (e.g., 1 Dec 2024)
|
||||
YYYY Mon DD (e.g., 2024 Dec 1)
|
||||
Date pattern regex:
|
||||
Matches for "day of",
|
||||
any full month name (e.g., "January", "February", ...)
|
||||
any month abbreviation (e.g., "Jan", "Feb", ...)
|
||||
date formats:
|
||||
MM/DD/YYYY (e.g., 12/25/2023)
|
||||
MM.DD.YYYY (e.g., 12.25.2023)
|
||||
MM-DD-YYYY (e.g., 12-25-2023)
|
||||
YYYY/MM/DD (e.g., 2024/12/09)
|
||||
YYYY.MM.DD (e.g., 2024.12.09)
|
||||
YYYY-MM-DD (e.g., 2024-12-09)
|
||||
YYYYMMDD (e.g., 20241209)
|
||||
MMYY or YYMM format (e.g., 1224 for December 2024 or 2405 for May 2024)
|
||||
YYYY.MM.DD (e.g., 2024.12.09)
|
||||
|
||||
Accounts for month names in lowercase and uppercase
|
||||
Case insensitive
|
||||
"""
|
||||
|
||||
# original pattern
|
||||
# DATE_PATTERN = r'\b((0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01])[-/](\d{4}|\d{2})|(\d{4}[-/](0?[1-9]|1[0-2])[-/](0?[1-9]|[12][0-9]|3[01]))|((0?[1-9]|[12][0-9]|3[01])[-/](0?[1-9]|1[0-2])[-/](\d{4}|\d{2}))|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2},\s+\d{4}|(?:\d{4}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{1,2})|(?:\d{1,2}\s+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec)\s+\d{4}))\b'
|
||||
|
||||
# new pattern to include "day of" date format
|
||||
base_months = "January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|oct|nov|dec"
|
||||
|
||||
parts = [
|
||||
r'(?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])[-/](?:\d{4}|\d{2})', # MM/DD/YYYY
|
||||
r'\d{4}[-/](?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])', # YYYY/MM/DD
|
||||
r'(?:0?[1-9]|[12][0-9]|3[01])[-/](?:0?[1-9]|1[0-2])[-/](?:\d{4}|\d{2})', # DD/MM/YYYY
|
||||
f'(?:{base_months})\s+\d{{1,2}},\s*\d{{4}}', # Month DD, YYYY
|
||||
f'\d{{4}}\s+(?:{base_months})\s+\d{{1,2}}', # YYYY Month DD
|
||||
f'\d{{1,2}}\s+(?:{base_months})\s+\d{{4}}', # DD Month YYYY
|
||||
f'\d{{1,2}}(?:st|nd|rd|th)?[\s\n]+[dD]ay[\s\n]+[oO]f[\s\n]+(?:{base_months})[\s\n]*,[\s\n]*\d{{4}}' # D(st|nd|rd|th) day of Month, YYYY
|
||||
]
|
||||
|
||||
# Combine all parts with the word boundary and alternation
|
||||
DATE_PATTERN = r'\b(?:' + '|'.join(parts) + r')\b'
|
||||
DATE_PATTERN = r"(?i)\bday of\b|\b(?:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|jun(?:e)?|jul(?:y)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\b|\b(?:\d{1,2}[\/\.-]\d{1,2}[\/\.-](?:\d{2}|\d{4})|\d{4}[\/\.-]\d{1,2}[\/\.-]\d{1,2}|\d{8}|\d{2}\d{2}|\d{4}[\/\.-]\d{1,2}[\/\.-]\d{1,2}|\d{4}[\.]\d{1,2}[\.]\d{1,2})\b"
|
||||
|
||||
def chunk_hierarchical(
|
||||
text_dict: dict[str, str], keywords: list[str], case_sensitive: bool = False
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import pytest
|
||||
import re
|
||||
|
||||
from ac_smart_chunking import DATE_PATTERN
|
||||
|
||||
legal_traditional_dates = [
|
||||
"This 12th day of December, 2024",
|
||||
"The 23RD DAY of JANUARY, 2026",
|
||||
"The 0th dAy of JaNuArY, 1000",
|
||||
"12th of December, Two Thousand and Twenty-Four",
|
||||
"Thirteenth of DeCeMbEr, Two Thousand and Sixteen",
|
||||
"Thirteenth of DeCeMbEr, Nineteen Hundred and Sixty Seven",
|
||||
"Thirteenth of DeCeMbEr, One Thousand, Nine Hundred, and Sixty Seven",
|
||||
"The 12th of December, 2024",
|
||||
"The 145th of January, 2023",
|
||||
"12th day of December, 2024",
|
||||
"1st day of January, 2025",
|
||||
"2nd day of JaNuArY, 2025",
|
||||
"3rd day of JANUARY, 2025",
|
||||
"21st day of January, 2025",
|
||||
"22nd day of January, 2025",
|
||||
"On 12 December 2025",
|
||||
]
|
||||
|
||||
numeric_dates = [
|
||||
"20.12.2024",
|
||||
"01.01.1000",
|
||||
"05.12.06",
|
||||
"12.18.2024",
|
||||
"20241212",
|
||||
"11/18/2014",
|
||||
"11-18-2024",
|
||||
"11/18/24",
|
||||
"11-18-24",
|
||||
"2024/11/18",
|
||||
"2024-11-18",
|
||||
"18/11/24",
|
||||
"1224",
|
||||
"2024.12.09",
|
||||
]
|
||||
|
||||
abbreviated_dates = [
|
||||
"18-Nov-2024",
|
||||
"Nov-18-2024",
|
||||
"Dec/12/2024",
|
||||
"Dec.12.2024",
|
||||
"Dec/2024",
|
||||
]
|
||||
|
||||
contextual_dates = [
|
||||
"Friday, December 18, 2024",
|
||||
"Fri, Dec 18, 2024",
|
||||
]
|
||||
|
||||
compact_dates = [
|
||||
"12-December-2024",
|
||||
"December 2024",
|
||||
"Dec-12-2024",
|
||||
]
|
||||
|
||||
case_variations = [
|
||||
"friday, december 18, 2024",
|
||||
"FRI, DEC 18, 2024",
|
||||
"DECEMBER 2024",
|
||||
# Multiple spaces
|
||||
"Friday, December 18, 2024",
|
||||
"Dec 12, 2024",
|
||||
# Multiple newlines
|
||||
"Friday,\nDecember\n18,\n2024",
|
||||
"Dec\n12,\n2024",
|
||||
# Original formats with newlines
|
||||
"1 day\nof December\n,\n2006",
|
||||
"1\nday\nof\nDecember\n,\n2006",
|
||||
"1st day\nof\nDecember,\n2006",
|
||||
]
|
||||
|
||||
ordinal_dates = [
|
||||
"1st day of December, 2006",
|
||||
"1st Day of December,2006",
|
||||
"2nd day of January, 2024",
|
||||
"2nd Day of Jan,2024",
|
||||
"3rd day of November, 1999",
|
||||
"3rd Day of Nov,1999",
|
||||
"4th day of March, 2023",
|
||||
"4th Day of Mar,2023",
|
||||
"21st day of April, 2022",
|
||||
"22nd day of May,2022",
|
||||
"23rd day of June, 2021",
|
||||
"24th Day of July,2021",
|
||||
]
|
||||
|
||||
invalid_dates = [
|
||||
"Invalid date",
|
||||
"Not a date",
|
||||
"Decide", # make sure "dec" doesn't accidentally match to "decide" or other words
|
||||
"decided",
|
||||
"Novel", # make sure "nov" doesn't accidentally match to other words
|
||||
"novelty",
|
||||
]
|
||||
|
||||
|
||||
def test_valid_dates():
|
||||
for date_list in [legal_traditional_dates, numeric_dates, abbreviated_dates,
|
||||
contextual_dates, compact_dates, case_variations, ordinal_dates]:
|
||||
for date in date_list:
|
||||
match = re.findall(DATE_PATTERN, date)
|
||||
assert match
|
||||
|
||||
def test_invalid_dates():
|
||||
for date in invalid_dates:
|
||||
match = re.findall(DATE_PATTERN, date)
|
||||
assert not match
|
||||
Reference in New Issue
Block a user