diff --git a/fieldExtraction/src/investment/file_processing.py b/fieldExtraction/src/investment/file_processing.py index 07ac491..599a017 100644 --- a/fieldExtraction/src/investment/file_processing.py +++ b/fieldExtraction/src/investment/file_processing.py @@ -3,6 +3,7 @@ import os import pandas as pd +from src.utils.string_utils import datetime_str import src.investment.one_to_n_funcs as one_to_n_funcs import src.investment.one_to_one_funcs as one_to_one_funcs import src.investment.postprocess as postprocess @@ -15,11 +16,6 @@ import src.utils.string_utils as string_utils import src.utils.io_utils as io_utils from src.config import FIELD_JSON_PATH from src.investment.one_to_n_funcs import dynamic_and_exhibit_level, reimbursement_level, combine_one_to_n_answers, reimbursement_tin_npi -from datetime import datetime - -def datetime_str(): - return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") - def process_file(file_object, all_dataset, run_timestamp): filename, contract_text = file_object diff --git a/fieldExtraction/src/utils/io_utils.py b/fieldExtraction/src/utils/io_utils.py index c1e76f4..04bc8d7 100644 --- a/fieldExtraction/src/utils/io_utils.py +++ b/fieldExtraction/src/utils/io_utils.py @@ -9,6 +9,7 @@ import src.tracking.tracking_utils as tracking_utils from src import config from src.investment.code_funcs import crosswalk_levels, get_mappings import src.constants.investment_values as investment_values +from src.utils.string_utils import datetime_str def read_local(file_path): # io_utils.py @@ -275,7 +276,7 @@ def load_all_dataset(): return all_dataset def load_embeddings(): - """_summary_: loads all embedding files required for processing indirect codes e.g. proc codes + """loads all embedding files required for processing indirect codes (e.g. proc codes) """ # list all embeddings present in s3 @@ -297,5 +298,17 @@ def load_embeddings(): # download embeddings not present locally for file in file_list: - if file not in local_files: - s3_client.download_file("doczy-investment",file,file) + local_path = file + + # create directory structure for this file + directory = os.path.dirname(local_path) + if not os.path.exists(directory): + os.makedirs(directory) + + # check if file already exists + if not os.path.exists(local_path): + try: + print(f"{datetime_str()} Downloading {file} to {local_path}...") + s3_client.download_file("doczy-investment", file, local_path) + except Exception as e: + print(f"{datetime_str()} Error downloading {file}: {e}") diff --git a/fieldExtraction/src/utils/string_utils.py b/fieldExtraction/src/utils/string_utils.py index acaa521..e6dbf41 100644 --- a/fieldExtraction/src/utils/string_utils.py +++ b/fieldExtraction/src/utils/string_utils.py @@ -1,3 +1,4 @@ +from datetime import datetime import json import re import warnings @@ -323,3 +324,7 @@ def get_exhibit_chunk(text_dict: dict, - It is important that `exhibit_page` exists as a key in the `exhibit_chunk_mapping` dictionary and corresponds to the starting page of an exhibit. """ return '\n'.join([page_text for page_num, page_text in text_dict.items() if exhibit_chunk_mapping[page_num] == exhibit_page]) + + +def datetime_str(): + return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")