{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "This is an adhoc script that is used to ease the process of archiving. \n", "Initially there was a scirpt that would run the process of archiving but there were edge cases where that script would fail due to the file names having nuances. \n", "\n", "This script simplifies that process. \n", "It will take input of S3 URIs of folders that need to be archived, \n", "e.g. \n", "s3://centene-national-contracting-files/batch_3_priority_files/3b_missing_files_103024/\n", "\n", "And move it to \n", "\n", "s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/3b_missing_files_103024/\n", "\n", "This archived folders within the batch folder. \n", "\n", "There is an option for dryrun which can be set as true which only outputs the \"aws s3 mv --recursive\" command. \n", " This can be used to run it manually to have more control over the folders that get archived \n", "\n", "Or we can set dry_run as False and let the subprocess run the aws cli command.\n", "\n", "This is setup as ipynb to ensure some more control over variables and paths and iterating accordingly " ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "import subprocess\n", "import logging\n", "import posixpath\n", "import sys\n", "\n", "# ----------------------------- Configuration ----------------------------- #\n", "\n", "# List of S3 URIs to archive\n", "S3_URIS = [\n", " 's3://centene-national-contracting-files/batch_3_priority_files/3b_missing_files_103024/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/dups_rerun_outputs/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/dups-for-textract/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/for-textract/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/hotfix_rerun_112724/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/redundant_dups/',\n", " 's3://centene-national-contracting-files/batch_3_priority_files/tables-for-textract/'\n", " 's3://centene-national-contracting-files/batch_3_priority_files/throttled_files/'\n", " # Add more S3 URIs as needed\n", "]\n", "\n", "# AWS Profile to use\n", "AWS_PROFILE = 'doczy_uat' # Replace with your AWS profile name\n", "\n", "# Dry Run Configuration\n", "DRY_RUN = False # Set to False to execute the commands\n", "\n", "# Logging Configuration\n", "LOG_FILE = 's3_mv_commands.log' # Log file name\n", "\n", "# ---------------------------- End Configuration --------------------------- #\n", "\n", "# Initialize logging\n", "logging.basicConfig(\n", " filename=LOG_FILE,\n", " filemode='a',\n", " format='%(asctime)s - %(levelname)s - %(message)s',\n", " level=logging.INFO\n", ")\n", "\n", "# Initialize a thread-safe print function\n", "from threading import Lock\n", "print_lock = Lock()\n", "\n", "def log(message, level='info'):\n", " \"\"\"\n", " Logs messages to both the console and a log file.\n", "\n", " :param message: The message to log.\n", " :param level: The logging level ('info', 'error', 'warning', 'debug').\n", " \"\"\"\n", " with print_lock:\n", " if level == 'info':\n", " logging.info(message)\n", " print(message)\n", " elif level == 'error':\n", " logging.error(message)\n", " print(f\"ERROR: {message}\")\n", " elif level == 'warning':\n", " logging.warning(message)\n", " print(f\"WARNING: {message}\")\n", " elif level == 'debug':\n", " logging.debug(message)\n", " print(f\"DEBUG: {message}\")\n", "\n", "def parse_s3_uri(s3_uri):\n", " \"\"\"\n", " Parses an S3 URI and returns the bucket and key.\n", "\n", " :param s3_uri: The S3 URI (e.g., s3://bucket/key)\n", " :return: Tuple of (bucket, key)\n", " \"\"\"\n", " if not s3_uri.startswith('s3://'):\n", " raise ValueError(f\"Invalid S3 URI: {s3_uri}\")\n", " parts = s3_uri[5:].split('/', 1)\n", " if len(parts) != 2:\n", " raise ValueError(f\"Invalid S3 URI: {s3_uri}\")\n", " bucket, key = parts\n", " return bucket, key\n", "\n", "def construct_destination_key(bucket, key):\n", " \"\"\"\n", " Constructs the destination key by inserting 'ARCHIVE/' after the batch prefix.\n", "\n", " :param bucket: The S3 bucket name.\n", " :param key: The original S3 object key.\n", " :return: The destination S3 object key.\n", " \"\"\"\n", " # Split the key into parts\n", " parts = key.split('/', 1)\n", " if len(parts) == 1:\n", " # If there's no subfolder, place ARCHIVE directly\n", " archive_key = posixpath.join('ARCHIVE', parts[0])\n", " else:\n", " batch_prefix, sub_key = parts\n", " archive_key = posixpath.join(batch_prefix, 'ARCHIVE', sub_key)\n", " return archive_key\n", "\n", "def generate_mv_command(source_uri, destination_uri):\n", " \"\"\"\n", " Generates the aws s3 mv command.\n", "\n", " :param source_uri: Source S3 URI.\n", " :param destination_uri: Destination S3 URI.\n", " :return: The command as a list suitable for subprocess.\n", " \"\"\"\n", " command = [\n", " 'aws',\n", " 's3',\n", " 'mv',\n", " source_uri,\n", " destination_uri,\n", " '--profile',\n", " AWS_PROFILE,\n", " '--recursive'\n", " ]\n", " return command\n", "\n", "def execute_command(command):\n", " \"\"\"\n", " Executes a command using subprocess.\n", "\n", " :param command: The command as a list.\n", " :return: Tuple of (returncode, stdout, stderr)\n", " \"\"\"\n", " try:\n", " result = subprocess.run(\n", " command,\n", " check=False, # We'll handle errors manually\n", " stdout=subprocess.PIPE,\n", " stderr=subprocess.PIPE,\n", " text=True\n", " )\n", " return result.returncode, result.stdout, result.stderr\n", " except Exception as e:\n", " return -1, '', str(e)\n", "\n", "def main():\n", " \"\"\"\n", " Main function to process the list of S3 URIs and perform mv operations.\n", " \"\"\"\n", " log(f\"Starting S3 mv operations with DRY_RUN={DRY_RUN}\", 'info')\n", " \n", " for s3_uri in S3_URIS:\n", " try:\n", " bucket, key = parse_s3_uri(s3_uri)\n", " destination_key = construct_destination_key(bucket, key)\n", " destination_uri = f\"s3://{bucket}/{destination_key}\"\n", " \n", " log(f\"Source URI: {s3_uri}\", 'info')\n", " log(f\"Destination URI: {destination_uri}\", 'info')\n", " \n", " command = generate_mv_command(s3_uri, destination_uri)\n", " command_str = ' '.join(command)\n", " log(f\"Generated Command: {command_str}\", 'debug')\n", " \n", " if DRY_RUN:\n", " log(f\"[Dry Run] Command to execute: {command_str}\", 'info')\n", " continue # Skip execution in dry run mode\n", " \n", " # Execute the command\n", " returncode, stdout, stderr = execute_command(command)\n", " if returncode == 0:\n", " log(f\"Successfully moved: {s3_uri} to {destination_uri}\", 'info')\n", " else:\n", " log(f\"Failed to move: {s3_uri} to {destination_uri}\", 'error')\n", " log(f\"Error: {stderr.strip()}\", 'error')\n", " \n", " except Exception as e:\n", " log(f\"Error processing URI '{s3_uri}': {e}\", 'error')\n", " \n", " log(\"S3 mv operations completed.\", 'info')\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting S3 mv operations with DRY_RUN=False\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/3b_missing_files_103024/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/3b_missing_files_103024/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/3b_missing_files_103024/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/3b_missing_files_103024/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/3b_missing_files_103024/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/3b_missing_files_103024/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/dups_rerun_outputs/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups_rerun_outputs/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/dups_rerun_outputs/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups_rerun_outputs/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/dups_rerun_outputs/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups_rerun_outputs/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/dups-for-textract/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups-for-textract/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/dups-for-textract/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups-for-textract/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/dups-for-textract/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/dups-for-textract/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/for-textract/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/for-textract/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/for-textract/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/for-textract/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/for-textract/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/for-textract/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/hotfix_rerun_112724/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/hotfix_rerun_112724/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/hotfix_rerun_112724/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/hotfix_rerun_112724/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/hotfix_rerun_112724/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/hotfix_rerun_112724/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/redundant_dups/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/redundant_dups/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/redundant_dups/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/redundant_dups/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/redundant_dups/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/redundant_dups/\n", "Source URI: s3://centene-national-contracting-files/batch_3_priority_files/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/\n", "Destination URI: s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/\n", "DEBUG: Generated Command: aws s3 mv s3://centene-national-contracting-files/batch_3_priority_files/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/ s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/ --profile doczy_uat --recursive\n", "Successfully moved: s3://centene-national-contracting-files/batch_3_priority_files/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/ to s3://centene-national-contracting-files/batch_3_priority_files/ARCHIVE/tables-for-textract/s3://centene-national-contracting-files/batch_3_priority_files/throttled_files/\n", "S3 mv operations completed.\n" ] } ], "source": [ "main()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.4" } }, "nbformat": 4, "nbformat_minor": 2 }