Umang Shailesh Mistry
c210052952
Feature/ops scripts * Added comments to Aryan's script and added some more scripts * Search and Copy Script uploaded as a Python Notebook - with comments and markdown * Merged main into feature/ops_scripts * Merged main into feature/ops_scripts Approved-by: Michael McGuinness Approved-by: Chris Stobie
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
import boto3
|
|
import concurrent.futures
|
|
|
|
|
|
"""
|
|
This script moves files with a specific extension from one folder to another in an S3 bucket.
|
|
It uses multiple threads to move files in parallel.
|
|
|
|
The script uses a temporary AWS profile called 'temp_cred' to authenticate with AWS.
|
|
The profile should be configured in the ~/.aws/credentials file.
|
|
|
|
Mainly used to copy and delete (move) files in the same bucket. Typically used to move redundant dups back to the batch TXT folder.
|
|
Can be used to copy .txt / .pdf files from one folder to another in the same bucket.
|
|
|
|
"""
|
|
|
|
s3 = boto3.Session(profile_name='temp_cred').client('s3')
|
|
|
|
def move_file(bucket_name, src_folder, dest_folder, file_key):
|
|
src_file_path = f"{src_folder}/{file_key}"
|
|
dest_file_path = f"{dest_folder}/{file_key}"
|
|
|
|
try:
|
|
s3.copy_object(
|
|
Bucket=bucket_name,
|
|
CopySource={'Bucket': bucket_name, 'Key': src_file_path},
|
|
Key=dest_file_path
|
|
)
|
|
|
|
s3.delete_object(Bucket=bucket_name, Key=src_file_path)
|
|
print(f"Moved {src_file_path} to {dest_file_path}")
|
|
|
|
except Exception as e:
|
|
print(f"Error moving {src_file_path}: {e}")
|
|
|
|
def list_files(bucket_name, folder, extension=None):
|
|
files = []
|
|
paginator = s3.get_paginator('list_objects_v2')
|
|
|
|
try:
|
|
for page in paginator.paginate(Bucket=bucket_name, Prefix=folder):
|
|
contents = page.get('Contents', [])
|
|
for item in contents:
|
|
key = item['Key']
|
|
if not extension or key.endswith(extension):
|
|
files.append(key.split('/')[-1])
|
|
except Exception as e:
|
|
print(f"Error listing files in {bucket_name}/{folder}: {e}")
|
|
|
|
return files
|
|
|
|
def move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=10):
|
|
files = list_files(bucket_name, src_folder)
|
|
|
|
if not files:
|
|
print(f"No {extension} files found in the source folder.")
|
|
return
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
|
|
futures = [
|
|
executor.submit(move_file, bucket_name, src_folder, dest_folder, file_key)
|
|
for file_key in files
|
|
]
|
|
|
|
concurrent.futures.wait(futures)
|
|
print(f"All {extension} files have been moved.")
|
|
|
|
bucket_name = 'centene-national-contracting-files'
|
|
src_folder = 'batch6_16_file/redundant_files'
|
|
dest_folder = 'batch6_16_file/txt_files'
|
|
extension = '.txt'
|
|
|
|
move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=40)
|