Katon Minhas
063c95d6ac
feature/dynamic and generalized branch * included txt files for nltk_data * move nltk_data to src * Fix last upload count * Last upload count bugfix * Fixed B processing * Remove client-specific postprocessing * split consolidate_output * Run by file * Fix output * Reconfigure smart_chunk fields * Add Full Context * Fix merge conflicts * Regex, Smart-Chunked, and Full working - not adding smart-chunked-->full when necessary * Modernized run_full_context_fields() * Switched set to list in field_context * Move fields from smart_chunked to full_context as part of 'field_context' function * Working version with placeholders * Update poetry and pyproject * Update s3 output * Remove deprecated unit test * Updated error messages * Updated smart chunk ac name to one to one * Update dependencies - end-to-end test for write s3 functional * Add basic multithreading * Send individual output to s3/local Approved-by: Alex Galarce
113 lines
2.7 KiB
Python
113 lines
2.7 KiB
Python
import re
|
|
|
|
import pytest
|
|
|
|
from src.client.smart_chunking_funcs import DATE_PATTERN
|
|
|
|
legal_traditional_dates = [
|
|
"This 12th day of December, 2024",
|
|
"The 23RD DAY of JANUARY, 2026",
|
|
"The 0th dAy of JaNuArY, 1000",
|
|
"12th of December, Two Thousand and Twenty-Four",
|
|
"Thirteenth of DeCeMbEr, Two Thousand and Sixteen",
|
|
"Thirteenth of DeCeMbEr, Nineteen Hundred and Sixty Seven",
|
|
"Thirteenth of DeCeMbEr, One Thousand, Nine Hundred, and Sixty Seven",
|
|
"The 12th of December, 2024",
|
|
"The 145th of January, 2023",
|
|
"12th day of December, 2024",
|
|
"1st day of January, 2025",
|
|
"2nd day of JaNuArY, 2025",
|
|
"3rd day of JANUARY, 2025",
|
|
"21st day of January, 2025",
|
|
"22nd day of January, 2025",
|
|
"On 12 December 2025",
|
|
]
|
|
|
|
numeric_dates = [
|
|
"20.12.2024",
|
|
"01.01.1000",
|
|
"05.12.06",
|
|
"12.18.2024",
|
|
"20241212",
|
|
"11/18/2014",
|
|
"11-18-2024",
|
|
"11/18/24",
|
|
"11-18-24",
|
|
"2024/11/18",
|
|
"2024-11-18",
|
|
"18/11/24",
|
|
"1224",
|
|
"2024.12.09",
|
|
]
|
|
|
|
abbreviated_dates = [
|
|
"18-Nov-2024",
|
|
"Nov-18-2024",
|
|
"Dec/12/2024",
|
|
"Dec.12.2024",
|
|
"Dec/2024",
|
|
]
|
|
|
|
contextual_dates = [
|
|
"Friday, December 18, 2024",
|
|
"Fri, Dec 18, 2024",
|
|
]
|
|
|
|
compact_dates = [
|
|
"12-December-2024",
|
|
"December 2024",
|
|
"Dec-12-2024",
|
|
]
|
|
|
|
case_variations = [
|
|
"friday, december 18, 2024",
|
|
"FRI, DEC 18, 2024",
|
|
"DECEMBER 2024",
|
|
# Multiple spaces
|
|
"Friday, December 18, 2024",
|
|
"Dec 12, 2024",
|
|
# Multiple newlines
|
|
"Friday,\nDecember\n18,\n2024",
|
|
"Dec\n12,\n2024",
|
|
# Original formats with newlines
|
|
"1 day\nof December\n,\n2006",
|
|
"1\nday\nof\nDecember\n,\n2006",
|
|
"1st day\nof\nDecember,\n2006",
|
|
]
|
|
|
|
ordinal_dates = [
|
|
"1st day of December, 2006",
|
|
"1st Day of December,2006",
|
|
"2nd day of January, 2024",
|
|
"2nd Day of Jan,2024",
|
|
"3rd day of November, 1999",
|
|
"3rd Day of Nov,1999",
|
|
"4th day of March, 2023",
|
|
"4th Day of Mar,2023",
|
|
"21st day of April, 2022",
|
|
"22nd day of May,2022",
|
|
"23rd day of June, 2021",
|
|
"24th Day of July,2021",
|
|
]
|
|
|
|
invalid_dates = [
|
|
"Invalid date",
|
|
"Not a date",
|
|
"Decide", # make sure "dec" doesn't accidentally match to "decide" or other words
|
|
"decided",
|
|
"Novel", # make sure "nov" doesn't accidentally match to other words
|
|
"novelty",
|
|
]
|
|
|
|
|
|
def test_valid_dates():
|
|
for date_list in [legal_traditional_dates, numeric_dates, abbreviated_dates,
|
|
contextual_dates, compact_dates, case_variations, ordinal_dates]:
|
|
for date in date_list:
|
|
match = re.findall(DATE_PATTERN, date)
|
|
assert match
|
|
|
|
def test_invalid_dates():
|
|
for date in invalid_dates:
|
|
match = re.findall(DATE_PATTERN, date)
|
|
assert not match |