doczyai-pipelines/archive/ops_scripts/generic/excel_s3_diff.py

import boto3
import pandas as pd
from concurrent.futures import ThreadPoolExecutor
import os
import math
import csv


"""
This scirpt compares the file names in an Excel file (Batch excel files that MCS usually provides) with the file names in an S3 bucket.
The Excel file must contain a column with the file names to search for.

This is used mainly to find files that are missing in the batch staging files and cannot be located in the s3 bucket.
The output of this script can be then used with t_drive_search.py to find the missing files in the T drive.

Note: This script outputs both a CSV and an Excel file with the missing files and encoding issues.
We use the output excel file with the tdrive search script to find the missing files in the T drive.

"""

# Function to list S3 objects and normalize the filenames
def list_s3_files(bucket_name, prefix, s3_client):
    normalized_s3_files = []
    paginator = s3_client.get_paginator('list_objects_v2')
    for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix):
        if 'Contents' in page:
            for obj in page['Contents']:
                key = obj['Key']
                # Extract and normalize the filename (remove prefix and extension)
                # filename = key.split('/')[-1]  # Take the file name from the key
                filename = os.path.basename(key)  # Remove any directory path
                if len(filename) > 4:
                    normalized_filename = filename[:-4]  # Remove the last 4 characters (file extension)
                    normalized_s3_files.append(normalized_filename)
    return set(normalized_s3_files)

# Main function for processing
def find_missing_files(input_excel, bucket_name, prefix, output_csv, max_workers=10):
    # Initialize boto3 session and S3 client
    session = boto3.Session(profile_name='doczy_uat')
    s3_client = session.client('s3')

    # Read the Excel file with header=1
    excel_df = pd.read_excel(input_excel, header=1, sheet_name='Sheet1')
    excel_filenames = excel_df['File Name']  # Remove extensions if present
    excel_filenames_set = set(excel_filenames)

    # Get the normalized S3 filenames
    normalized_s3_files = list_s3_files(bucket_name, prefix, s3_client)

    # Find files that are in Excel but not in S3
    missing_files = excel_filenames_set - normalized_s3_files

    print(f"Type of missing_files: {type(missing_files)}")
    print(f"Type of normalized_files: {type(normalized_s3_files)}")
    print(f"Type of excel_filenames_set: {type(excel_filenames_set)}")

    # Identify files that may have encoding differences, ensuring only strings are processed
    missing_files_list = [file for file in missing_files if isinstance(file, str) and not (isinstance(file, float) and math.isnan(file))]
    non_encoded_issues = [file for file in missing_files_list if all(ord(char) < 128 for char in file)]

    output_df = pd.DataFrame({
    'Missing File Name': missing_files_list,
    'Encoding Check': ['No Encoding Issue' if file in non_encoded_issues else 'Encoding Issue' for file in missing_files_list]
    })  

    output_df.to_csv(output_csv, index=False, encoding='utf-8', quoting=csv.QUOTE_ALL)
    # Saving output as excel file
    output_df.to_excel(output_csv[:-4] + '.xlsx', index=False)

    # Write the missing files and non-encoded issue files to the output CSV
    # with open(output_csv, mode='w', newline='', encoding='utf-8') as csvfile:
    #     csvfile.write('Missing File Name,Encoding Check\n')
    #     for file in missing_files_list:
    #         encoding_status = 'No Encoding Issue' if file in non_encoded_issues else 'Encoding Issue'
    #         csvfile.write(f"{file},{encoding_status}\n")

if __name__ == "__main__":
    input_excel = 'C:\\Doczy\\National contracting\\Allbatches\\batch_files\\updated_batches_102424\\first_batch\\Batch 14_10222024.xlsx'     # Path to your input Excel file
    bucket_name = 'centene-national-contracting-files'     # Your S3 bucket name
    prefix = 'batch_14_priority_files/TXT_FILES/'              # Your S3 prefix
    output_csv = 'C:\\Doczy\\National contracting\\Allbatches\\diff_batch14_2.csv'     # This will also save as xls
    find_missing_files(input_excel, bucket_name, prefix, output_csv)