Merged in hotfix/nltk-bug (pull request #784)

backup - tempfix

* backup - tempfix


Approved-by: Karan Desai
This commit is contained in:
Katon Minhas
2025-11-21 19:21:56 +00:00
parent 8ea69063b1
commit 5e9de5021d
@@ -3,17 +3,40 @@ import warnings
warnings.filterwarnings("ignore")
import pandas as pd
import nltk
import ssl
nltk.download("stopwords")
from nltk.corpus import stopwords
stop_words = stopwords.words("english")
import re
import string
import src.config as config
from datetime import datetime
import logging
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
pass
else:
ssl._create_default_https_context = _create_unverified_https_context
# Download stopwords if not already present
try:
from nltk.corpus import stopwords
stop_words = stopwords.words("english")
except LookupError as e:
# Fallback: use a hardcoded list of common English stopwords
logging.warning(f"Could not download NLTK stopwords: {e}. Using fallback list.")
stop_words = [
'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're",
"you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he',
'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's",
'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which',
'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are',
'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does',
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until',
'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into',
'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down',
'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once'
]
def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_NAME
all_fields_df["contract_name_cleaned"] = (