Merged in feature/add-dotenv (pull request #266)

Feature/add dotenv

* transferred sensitive information to .env file from src/config.py

* replace model IDs, AWS account info

* Merge remote-tracking branch 'origin/main' into feature/add-dotenv

* add explanatory comment

* Merge remote-tracking branch 'origin/main' into feature/add-dotenv

* remove tests that were just for local


Approved-by: Katon Minhas

.gitignore

@@ -93,4 +93,5 @@ docs/
 .pytest_cache/
 __pycache__/
 
+.env
 new/

fieldExtraction/poetry.lock

@@ -1006,6 +1006,20 @@ files = [
 [package.dependencies]
 six = ">=1.5"
 
+[[package]]
+name = "python-dotenv"
+version = "1.0.1"
+description = "Read key-value pairs from a .env file and set them as environment variables"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"},
+    {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"},
+]
+
+[package.extras]
+cli = ["click (>=5.0)"]
+
 [[package]]
 name = "pytz"
 version = "2024.2"
@@ -1318,4 +1332,4 @@ zstd = ["zstandard (>=0.18.0)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "195ab38a76cbe281ddfaf343f1e5da592ac8b3e6e323a3f03f4ff640fc3f7398"
+content-hash = "ea503ef05494f0fb5392971c2d1cf4ddfb97bbd5ac7a419c6f592115c9860595"

fieldExtraction/pyproject.toml

@@ -11,6 +11,7 @@ python = "^3.12"
 pandas = "^2.2.3"
 boto3 = "^1.35.40"
 anthropic = "^0.36.0"
+python-dotenv = "^1.0.1"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"

fieldExtraction/src/config.py

@@ -11,6 +11,13 @@ import requests
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
+import os
+from dotenv import load_dotenv
+
+# Please see the following confluence page for instructions on loading keys into your
+# `.env`:
+# https://aarete.atlassian.net/wiki/spaces/DoczyAI/pages/1283751954/Using+.env+for+AWS+keys
+load_dotenv()
 
 ######################################## UTILS ###################################################
 def check_s3_bucket_exists(s3_client, bucket_name: str):
@@ -39,6 +46,13 @@ TODAY = datetime.now().strftime("%Y%m%d")
 ######################################## SYSTEM ARGUMENTS ########################################
 # read_mode, run_mode, input_dir, output_dir, consolidated_output_dir
 
+def get_value_from_env(key: str, default_value: str=None) -> str:
+    value = os.getenv(key,default_value)
+
+    if value is not None:
+        return value
+    
+    raise KeyError(f'{key} not found in .env file')
 
 def get_arg_value(arg_name, default):
     args = [item for item in sys.argv if arg_name == item.split("=")[0]]
@@ -116,9 +130,11 @@ TABLE_ANALYSIS_NAME = f"{BATCH_ID}-Table-Analysis.csv"
 
 # Bedrock
 if RUN_MODE == "local":
-    AWS_ACCESS_KEY_ID="ASIA6GBMBVWOC2KZPWDJ"
-    AWS_SECRET_ACCESS_KEY="Ee0mAnRHA03wWxFaM4NQEy1PgdXotF9OV7b5D41Y"
-    AWS_SESSION_TOKEN="IQoJb3JpZ2luX2VjEFAaCXVzLWVhc3QtMiJHMEUCIQDfNitr9GFdwaTb5tOSyNWsU9uuButk54yNlHGxt8FxogIgCq1IqDKHRdm1YUR+ddAFOt7RBAw2KnZZ7EOEZxgsu34qigMI2f//////////ARAAGgw5NzUwNDk5NjA4NjAiDMn0VL/snLYOd/XIPSreAlJkUy5fBZKgnhx/mmD6oC+wlCebfHCEhzmdrBJkuxPVBNgV9ebKVLUIP2fyRE+MK+/P//+RbjZzxolcLYoM4HIs6EVNO2fa/NSjTfv7IVnUAGDbS+jVUbr8ICNLz2eo8mDCm8J0SS5kTpVWhuZ6kJhxvOmN9y2U/RLCCbhxAMaIrNC1/bS8JQODSBsP25aDtdJflBazk6jSwiNry0ka6gTkmdk0R4RYXccCv7mUdmIc5V+6eH1a7qslbVdDpS+hsrU/42Jsg57CyR+Tz7a0zNixUhwXj1s8Ma8ygzkTJKevXF2W3AlJ2XK1WIwegbD6CDroFKd7USr+RrY+2Ftr+8dWqYhvt00UAHGSBpNpffkqJFcOTnUiD+6pipymwL8OYk1Pc+LB2QfNYTZgQYWkofqAx4A6NEtk4cmO0qSHpQoiBrD+sv/3TE1pShlhFnRo1Ij99byJrqSBoIUNXNS2MKyj07kGOqYBoPi0JfB2A3gmYsTmYLGMLbnfET8vr2E8WVUSr5NMrbm4fo3Yc4W3gISOVToncH254EqO7Q+gakUAi65w8JgOjQlPkM+u9Knpetw27Ym38qkxkEQ7Htk1zbEXgYkm4iQyaPXBFtD6d3Nsv2gOGQRhXgC4uLuZvqdrVPF+msYU4Cwi2rLpmsvBlbI11DIPvAapJsNz7sK+GNhXdJefP49RK/pYAH10+w=="
+
+    AWS_ACCESS_KEY_ID = get_value_from_env('AWS_ACCESS_KEY_ID')
+    AWS_SECRET_ACCESS_KEY = get_value_from_env('AWS_SECRET_ACCESS_KEY')
+    AWS_SESSION_TOKEN = get_value_from_env('AWS_SESSION_TOKEN')
+
     config = Config(read_timeout=2000)
     BEDROCK_RUNTIME = boto3.client(
         service_name="bedrock-runtime",
@@ -231,7 +247,6 @@ def assume_role_and_get_runtime(account_id, role_name):
     )
     return runtime
 
-
 dev_account_id = "660131068782"
 dev_role_name = "doczy-dev-bedrock-lambda-role"