Michael McGuinness
35 lines
992 B
Python
35 lines
992 B
Python
import boto3
|
|
from botocore.exceptions import ClientError
|
|
import snowflake.connector
|
|
|
|
|
|
def get_files(s3_client, path, client_bucket, batch_id):
|
|
batch_objects = s3_client.list_objects_v2(
|
|
Bucket=client_bucket, Prefix=path + "/" + batch_id, Delimiter="/"
|
|
)
|
|
file_list = []
|
|
for prefix in batch_objects["CommonPrefixes"]:
|
|
file_list.append(prefix["Prefix"][:-1].split("/")[-1])
|
|
file_count = len(file_list)
|
|
return file_count, file_list
|
|
|
|
|
|
def get_s3_client():
|
|
region_name = "us-east-2"
|
|
s3_client = boto3.client("s3", region_name=region_name)
|
|
return s3_client
|
|
|
|
|
|
def get_snowflake_conn(schema):
|
|
secret = "" # get_secret()
|
|
secret_dict = eval(secret)
|
|
conn = snowflake.connector.connect(
|
|
user=secret_dict["user"],
|
|
password=secret_dict["password"],
|
|
account=secret_dict["account_alias"],
|
|
warehouse=secret_dict["warehouse"],
|
|
database=secret_dict["database"],
|
|
schema=schema,
|
|
)
|
|
return conn
|