From 5e9de5021d6ac3ea981bfaf4513ad442cf233f43 Mon Sep 17 00:00:00 2001 From: Katon Minhas Date: Fri, 21 Nov 2025 19:21:56 +0000 Subject: [PATCH] Merged in hotfix/nltk-bug (pull request #784) backup - tempfix * backup - tempfix Approved-by: Karan Desai --- .../parent_child_preprocessing.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/fieldExtraction/src/parent_child/parent_child_preprocessing.py b/fieldExtraction/src/parent_child/parent_child_preprocessing.py index 8259d02..92dd0ab 100644 --- a/fieldExtraction/src/parent_child/parent_child_preprocessing.py +++ b/fieldExtraction/src/parent_child/parent_child_preprocessing.py @@ -3,17 +3,40 @@ import warnings warnings.filterwarnings("ignore") import pandas as pd import nltk +import ssl -nltk.download("stopwords") -from nltk.corpus import stopwords - -stop_words = stopwords.words("english") import re import string import src.config as config from datetime import datetime import logging +try: + _create_unverified_https_context = ssl._create_unverified_context +except AttributeError: + pass +else: + ssl._create_default_https_context = _create_unverified_https_context + +# Download stopwords if not already present +try: + from nltk.corpus import stopwords + stop_words = stopwords.words("english") +except LookupError as e: + # Fallback: use a hardcoded list of common English stopwords + logging.warning(f"Could not download NLTK stopwords: {e}. Using fallback list.") + stop_words = [ + 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", + "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', + 'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", + 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', + 'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', + 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', + 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', + 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', + 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', + 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once' + ] def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_NAME all_fields_df["contract_name_cleaned"] = (