doczyai-pipelines/archive/ops_scripts/CNC/s3_move_files_with_extension.py

import boto3
import concurrent.futures


"""
This script moves files with a specific extension from one folder to another in an S3 bucket.
It uses multiple threads to move files in parallel.

The script uses a temporary AWS profile called 'temp_cred' to authenticate with AWS.
The profile should be configured in the ~/.aws/credentials file.

Mainly used to copy and delete (move) files in the same bucket. Typically used to move redundant dups back to the batch TXT folder.
Can be used to copy .txt / .pdf files from one folder to another in the same bucket.

"""

s3 = boto3.Session(profile_name='temp_cred').client('s3')

def move_file(bucket_name, src_folder, dest_folder, file_key):
    src_file_path = f"{src_folder}/{file_key}"
    dest_file_path = f"{dest_folder}/{file_key}"
    
    try:
        s3.copy_object(
            Bucket=bucket_name,
            CopySource={'Bucket': bucket_name, 'Key': src_file_path},
            Key=dest_file_path
        )

        s3.delete_object(Bucket=bucket_name, Key=src_file_path)
        print(f"Moved {src_file_path} to {dest_file_path}")
    
    except Exception as e:
        print(f"Error moving {src_file_path}: {e}")

def list_files(bucket_name, folder, extension=None):
    files = []
    paginator = s3.get_paginator('list_objects_v2')
    
    try:
        for page in paginator.paginate(Bucket=bucket_name, Prefix=folder):
            contents = page.get('Contents', [])
            for item in contents:
                key = item['Key']
                if not extension or key.endswith(extension):
                    files.append(key.split('/')[-1])
    except Exception as e:
        print(f"Error listing files in {bucket_name}/{folder}: {e}")
    
    return files

def move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=10):
    files = list_files(bucket_name, src_folder)
    
    if not files:
        print(f"No {extension} files found in the source folder.")
        return

    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
        futures = [
            executor.submit(move_file, bucket_name, src_folder, dest_folder, file_key)
            for file_key in files
        ]

        concurrent.futures.wait(futures)
        print(f"All {extension} files have been moved.")

bucket_name = 'centene-national-contracting-files'
src_folder = 'batch6_16_file/redundant_files'
dest_folder = 'batch6_16_file/txt_files'
extension = '.txt'

move_files_multi_thread(bucket_name, src_folder, dest_folder, num_threads=40)