From 4334800fa357446b6c416a52b7892a7b05ff6c46 Mon Sep 17 00:00:00 2001
From: Karan Desai <kdesai@aarete.com>
Date: Tue, 25 Nov 2025 23:03:05 +0000
Subject: [PATCH] Merged in feature/pc_code_optimization (pull request #788)

added stopwords json

* added stopwords json


Approved-by: Katon Minhas
---
 .../mappings/stop_words_for_pc_cleanup.json   | 202 ++++++++++++++++++
 .../parent_child_preprocessing.py             |  30 +--
 2 files changed, 207 insertions(+), 25 deletions(-)
 create mode 100644 fieldExtraction/constants/mappings/stop_words_for_pc_cleanup.json

diff --git a/fieldExtraction/constants/mappings/stop_words_for_pc_cleanup.json b/fieldExtraction/constants/mappings/stop_words_for_pc_cleanup.json
new file mode 100644
index 0000000..d86cd3b
--- /dev/null
+++ b/fieldExtraction/constants/mappings/stop_words_for_pc_cleanup.json
@@ -0,0 +1,202 @@
+{
+    "stop_words": [
+        "a",
+        "about",
+        "above",
+        "after",
+        "again",
+        "against",
+        "ain",
+        "all",
+        "am",
+        "an",
+        "and",
+        "any",
+        "are",
+        "aren",
+        "aren't",
+        "as",
+        "at",
+        "be",
+        "because",
+        "been",
+        "before",
+        "being",
+        "below",
+        "between",
+        "both",
+        "but",
+        "by",
+        "can",
+        "couldn",
+        "couldn't",
+        "d",
+        "did",
+        "didn",
+        "didn't",
+        "do",
+        "does",
+        "doesn",
+        "doesn't",
+        "doing",
+        "don",
+        "don't",
+        "down",
+        "during",
+        "each",
+        "few",
+        "for",
+        "from",
+        "further",
+        "had",
+        "hadn",
+        "hadn't",
+        "has",
+        "hasn",
+        "hasn't",
+        "have",
+        "haven",
+        "haven't",
+        "having",
+        "he",
+        "he'd",
+        "he'll",
+        "her",
+        "here",
+        "hers",
+        "herself",
+        "he's",
+        "him",
+        "himself",
+        "his",
+        "how",
+        "i",
+        "i'd",
+        "if",
+        "i'll",
+        "i'm",
+        "in",
+        "into",
+        "is",
+        "isn",
+        "isn't",
+        "it",
+        "it'd",
+        "it'll",
+        "it's",
+        "its",
+        "itself",
+        "i've",
+        "just",
+        "ll",
+        "m",
+        "ma",
+        "me",
+        "mightn",
+        "mightn't",
+        "more",
+        "most",
+        "mustn",
+        "mustn't",
+        "my",
+        "myself",
+        "needn",
+        "needn't",
+        "no",
+        "nor",
+        "not",
+        "now",
+        "o",
+        "of",
+        "off",
+        "on",
+        "once",
+        "only",
+        "or",
+        "other",
+        "our",
+        "ours",
+        "ourselves",
+        "out",
+        "over",
+        "own",
+        "re",
+        "s",
+        "same",
+        "shan",
+        "shan't",
+        "she",
+        "she'd",
+        "she'll",
+        "she's",
+        "should",
+        "shouldn",
+        "shouldn't",
+        "should've",
+        "so",
+        "some",
+        "such",
+        "t",
+        "than",
+        "that",
+        "that'll",
+        "the",
+        "their",
+        "theirs",
+        "them",
+        "themselves",
+        "then",
+        "there",
+        "these",
+        "they",
+        "they'd",
+        "they'll",
+        "they're",
+        "they've",
+        "this",
+        "those",
+        "through",
+        "to",
+        "too",
+        "under",
+        "until",
+        "up",
+        "ve",
+        "very",
+        "was",
+        "wasn",
+        "wasn't",
+        "we",
+        "we'd",
+        "we'll",
+        "we're",
+        "were",
+        "weren",
+        "weren't",
+        "we've",
+        "what",
+        "when",
+        "where",
+        "which",
+        "while",
+        "who",
+        "whom",
+        "why",
+        "will",
+        "with",
+        "won",
+        "won't",
+        "wouldn",
+        "wouldn't",
+        "y",
+        "you",
+        "you'd",
+        "you'll",
+        "your",
+        "you're",
+        "yours",
+        "yourself",
+        "yourselves",
+        "you've"
+    ]
+}
diff --git a/fieldExtraction/src/parent_child/parent_child_preprocessing.py b/fieldExtraction/src/parent_child/parent_child_preprocessing.py
index 92dd0ab..c81265e 100644
--- a/fieldExtraction/src/parent_child/parent_child_preprocessing.py
+++ b/fieldExtraction/src/parent_child/parent_child_preprocessing.py
@@ -10,33 +10,13 @@ import string
 import src.config as config
 from datetime import datetime
 import logging
+from src.utils import io_utils
 
-try:
-    _create_unverified_https_context = ssl._create_unverified_context
-except AttributeError:
-    pass
-else:
-    ssl._create_default_https_context = _create_unverified_https_context
+stop_words_dict = io_utils.convert_json_to_dict(
+    "constants/mappings/stop_words_for_pc_cleanup.json"
+)
+stop_words = stop_words_dict["stop_words"]
 
-# Download stopwords if not already present
-try:
-    from nltk.corpus import stopwords
-    stop_words = stopwords.words("english")
-except LookupError as e:
-    # Fallback: use a hardcoded list of common English stopwords
-    logging.warning(f"Could not download NLTK stopwords: {e}. Using fallback list.")
-    stop_words = [
-        'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', "you're", 
-        "you've", "you'll", "you'd", 'your', 'yours', 'yourself', 'yourselves', 'he', 
-        'him', 'his', 'himself', 'she', "she's", 'her', 'hers', 'herself', 'it', "it's", 
-        'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 
-        'who', 'whom', 'this', 'that', "that'll", 'these', 'those', 'am', 'is', 'are', 
-        'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 
-        'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 
-        'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 
-        'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 
-        'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once'
-    ]
 
 def clean_filename(all_fields_df, contact_name_col="FILE_NAME"):  # Fixing FILE_NAME
     all_fields_df["contract_name_cleaned"] = (