From 896386e86d8d921d00fd29ad26da8c9529b602aa Mon Sep 17 00:00:00 2001 From: Alex Galarce Date: Tue, 28 Jan 2025 22:38:05 +0000 Subject: [PATCH] Merged in bugfix/fix-nltk-to-new-path (pull request #367) Bugfix/fix nltk to new path * point NLTK to right path finally * take out unreferenced function Approved-by: Katon Minhas --- .../src/investment/smart_chunking_funcs.py | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/fieldExtraction/src/investment/smart_chunking_funcs.py b/fieldExtraction/src/investment/smart_chunking_funcs.py index 29e8dd2..e2a43bb 100644 --- a/fieldExtraction/src/investment/smart_chunking_funcs.py +++ b/fieldExtraction/src/investment/smart_chunking_funcs.py @@ -19,7 +19,7 @@ import json import nltk import os # Append path of NLTK data directory (fieldExtraction/nltk_data) -nltk.data.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "nltk_data")) +nltk.data.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "nltk_data")) from nltk.tokenize import word_tokenize from itertools import combinations @@ -117,33 +117,6 @@ def group_fields(field_set: FieldSet, threshold=3): return groups -def get_full_context_fields( - ac_answers_dict: dict[str, str], no_keyword_matches: list -) -> list: - - smartly_chunked = [] - - for field_groups in keywords.GROUPED_KEYWORD_MAPPINGS.keys(): - for fields in keywords.GROUPED_KEYWORD_MAPPINGS[field_groups]["fields"]: - smartly_chunked.append(fields) - - # Instantiate full context fields from those that weren't smart chunked - full_context_fields = [ - field for field in prompts.AC_DICT if field not in smartly_chunked - ] - - # Add fields with no keyword matches to full context list - if len(no_keyword_matches) > 0: - full_context_fields.extend(no_keyword_matches) - - # Filter out fields that already have answers - # e.g., some have already been filled out by regex chunking - full_context_fields = [ - field for field in full_context_fields if field not in ac_answers_dict.keys() - ] - return full_context_fields - - def chunk_hierarchical(