Merged in feature/reimbursement-primary-optimizization (pull request #428)

Fix load_embeddings()

* move datetime_str function to string_utils module

* enhance load_embeddings to create directory structure and handle download errors

* Merged main into feature/reimbursement-primary-optimizization


Approved-by: Katon Minhas

fieldExtraction/src/investment/file_processing.py

@@ -3,6 +3,7 @@ import os
 
 import pandas as pd
 
+from src.utils.string_utils import datetime_str
 import src.investment.one_to_n_funcs as one_to_n_funcs
 import src.investment.one_to_one_funcs as one_to_one_funcs
 import src.investment.postprocess as postprocess
@@ -15,11 +16,6 @@ import src.utils.string_utils as string_utils
 import src.utils.io_utils as io_utils
 from src.config import FIELD_JSON_PATH
 from src.investment.one_to_n_funcs import dynamic_and_exhibit_level, reimbursement_level, combine_one_to_n_answers, reimbursement_tin_npi
-from datetime import datetime
-
-def datetime_str():
-    return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")
-
 
 def process_file(file_object, all_dataset, run_timestamp):
     filename, contract_text = file_object

fieldExtraction/src/utils/io_utils.py

@@ -9,6 +9,7 @@ import src.tracking.tracking_utils as tracking_utils
 from src import config
 from src.investment.code_funcs import crosswalk_levels, get_mappings
 import src.constants.investment_values as investment_values
+from src.utils.string_utils import datetime_str
 
 
 def read_local(file_path): # io_utils.py
@@ -275,7 +276,7 @@ def load_all_dataset():
     return all_dataset
 
 def load_embeddings():
-    """_summary_: loads all embedding files required for processing indirect codes e.g. proc codes
+    """loads all embedding files required for processing indirect codes (e.g. proc codes)
     """
 
     # list all embeddings present in s3
@@ -297,5 +298,17 @@ def load_embeddings():
 
     # download embeddings not present locally
     for file in file_list:
-        if file not in local_files:
-            s3_client.download_file("doczy-investment",file,file)
+        local_path = file
+
+        # create directory structure for this file
+        directory = os.path.dirname(local_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        
+        # check if file already exists
+        if not os.path.exists(local_path):
+            try:
+                print(f"{datetime_str()} Downloading {file} to {local_path}...")
+                s3_client.download_file("doczy-investment", file, local_path)
+            except Exception as e:
+                print(f"{datetime_str()} Error downloading {file}: {e}")

fieldExtraction/src/utils/string_utils.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import json
 import re
 import warnings
@@ -323,3 +324,7 @@ def get_exhibit_chunk(text_dict: dict,
         - It is important that `exhibit_page` exists as a key in the `exhibit_chunk_mapping` dictionary and corresponds to the starting page of an exhibit.
     """
     return '\n'.join([page_text for page_num, page_text in text_dict.items() if exhibit_chunk_mapping[page_num] == exhibit_page])
+
+
+def datetime_str():
+    return datetime.now().strftime("[%Y-%m-%d %H:%M:%S]")