Michael McGuinness
6c028ef892
DRAFT: Feature/adhoc ops scripts * added adhoc ops scripts for pre-doczy work * Added first draft of cost automation script * Fixed dir name * Added Scheduler lambda * Added TX adhoc script for rerun + Updated cost automation script * Added some misc scripts * Merged main into feature/adhoc-ops-scripts * Aryan Ad Hoc Scripts Pushed * De Duplication Script added * Merged main into feature/adhoc-ops-scripts Approved-by: Umang Shailesh Mistry
61 lines
2.0 KiB
Python
61 lines
2.0 KiB
Python
import boto3
|
|
import concurrent.futures
|
|
|
|
s3 = boto3.Session(profile_name='temp_cred').client('s3')
|
|
|
|
def move_file(bucket_name, src_folder, dest_folder, file_key):
|
|
src_file_path = f"{src_folder}/{file_key}"
|
|
dest_file_path = f"{dest_folder}/{file_key}"
|
|
|
|
try:
|
|
s3.copy_object(
|
|
Bucket=bucket_name,
|
|
CopySource={'Bucket': bucket_name, 'Key': src_file_path},
|
|
Key=dest_file_path
|
|
)
|
|
|
|
s3.delete_object(Bucket=bucket_name, Key=src_file_path)
|
|
print(f"Moved {src_file_path} to {dest_file_path}")
|
|
|
|
except Exception as e:
|
|
print(f"Error moving {src_file_path}: {e}")
|
|
|
|
def list_files(bucket_name, folder, extension=None):
|
|
files = []
|
|
paginator = s3.get_paginator('list_objects_v2')
|
|
|
|
try:
|
|
for page in paginator.paginate(Bucket=bucket_name, Prefix=folder):
|
|
contents = page.get('Contents', [])
|
|
for item in contents:
|
|
key = item['Key']
|
|
if not extension or key.endswith(extension):
|
|
files.append(key.split('/')[-1])
|
|
except Exception as e:
|
|
print(f"Error listing files in {bucket_name}/{folder}: {e}")
|
|
|
|
return files
|
|
|
|
def move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=10):
|
|
files = list_files(bucket_name, src_folder)
|
|
|
|
if not files:
|
|
print(f"No {extension} files found in the source folder.")
|
|
return
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
|
|
futures = [
|
|
executor.submit(move_file, bucket_name, src_folder, dest_folder, file_key)
|
|
for file_key in files
|
|
]
|
|
|
|
concurrent.futures.wait(futures)
|
|
print(f"All {extension} files have been moved.")
|
|
|
|
bucket_name = 'centene-national-contracting-files'
|
|
src_folder = 'batch6_16_file/redundant_files'
|
|
dest_folder = 'batch6_16_file/txt_files'
|
|
extension = '.txt'
|
|
|
|
move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=40)
|