Merged in hotfix/nltk-bug (pull request #784)
backup - tempfix * backup - tempfix Approved-by: Karan Desai
This commit is contained in:
@@ -3,17 +3,40 @@ import warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
import pandas as pd
|
||||
import nltk
|
||||
import ssl
|
||||
|
||||
nltk.download("stopwords")
|
||||
from nltk.corpus import stopwords
|
||||
|
||||
stop_words = stopwords.words("english")
|
||||
import re
|
||||
import string
|
||||
import src.config as config
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
try:
|
||||
_create_unverified_https_context = ssl._create_unverified_context
|
||||
except AttributeError:
|
||||
pass
|
||||
else:
|
||||
ssl._create_default_https_context = _create_unverified_https_context
|
||||
|
||||
# Download stopwords if not already present
|
||||
try:
|
||||
from nltk.corpus import stopwords
|
||||
stop_words = stopwords.words("english")
|
||||
except LookupError as e:
|
||||
# Fallback: use a hardcoded list of common English stopwords
|
||||
logging.warning(f"Could not download NLTK stopwords: {e}. Using fallback list.")
|
||||
stop_words = [
|
||||
'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're",
|
||||
"you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he',
|
||||
'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's",
|
||||
'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which',
|
||||
'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are',
|
||||
'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does',
|
||||
'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until',
|
||||
'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into',
|
||||
'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down',
|
||||
'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once'
|
||||
]
|
||||
|
||||
def clean_filename(all_fields_df, contact_name_col="FILE_NAME"): # Fixing FILE_NAME
|
||||
all_fields_df["contract_name_cleaned"] = (
|
||||
|
||||
Reference in New Issue
Block a user