84 lines
4.1 KiB
Python
84 lines
4.1 KiB
Python
|
import boto3
|
||
|
|
import pandas as pd
|
||
|
|
from concurrent.futures import ThreadPoolExecutor
|
||
|
|
import os
|
||
|
|
import math
|
||
|
|
import csv
|
||
|
|
|
||
|
|
|
||
|
|
"""
|
||
|
|
This scirpt compares the file names in an Excel file (Batch excel files that MCS usually provides) with the file names in an S3 bucket.
|
||
|
|
The Excel file must contain a column with the file names to search for.
|
||
|
|
|
||
|
|
This is used mainly to find files that are missing in the batch staging files and cannot be located in the s3 bucket.
|
||
|
|
The output of this script can be then used with t_drive_search.py to find the missing files in the T drive.
|
||
|
|
|
||
|
|
Note: This script outputs both a CSV and an Excel file with the missing files and encoding issues.
|
||
|
|
We use the output excel file with the tdrive search script to find the missing files in the T drive.
|
||
|
|
|
||
|
|
"""
|
||
|
|
|
||
|
|
# Function to list S3 objects and normalize the filenames
|
||
|
|
def list_s3_files(bucket_name, prefix, s3_client):
|
||
|
|
normalized_s3_files = []
|
||
|
|
paginator = s3_client.get_paginator('list_objects_v2')
|
||
|
|
for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix):
|
||
|
|
if 'Contents' in page:
|
||
|
|
for obj in page['Contents']:
|
||
|
|
key = obj['Key']
|
||
|
|
# Extract and normalize the filename (remove prefix and extension)
|
||
|
|
# filename = key.split('/')[-1] # Take the file name from the key
|
||
|
|
filename = os.path.basename(key) # Remove any directory path
|
||
|
|
if len(filename) > 4:
|
||
|
|
normalized_filename = filename[:-4] # Remove the last 4 characters (file extension)
|
||
|
|
normalized_s3_files.append(normalized_filename)
|
||
|
|
return set(normalized_s3_files)
|
||
|
|
|
||
|
|
# Main function for processing
|
||
|
|
def find_missing_files(input_excel, bucket_name, prefix, output_csv, max_workers=10):
|
||
|
|
# Initialize boto3 session and S3 client
|
||
|
|
session = boto3.Session(profile_name='doczy_uat')
|
||
|
|
s3_client = session.client('s3')
|
||
|
|
|
||
|
|
# Read the Excel file with header=1
|
||
|
|
excel_df = pd.read_excel(input_excel, header=1, sheet_name='Sheet1')
|
||
|
|
excel_filenames = excel_df['File Name'] # Remove extensions if present
|
||
|
|
excel_filenames_set = set(excel_filenames)
|
||
|
|
|
||
|
|
# Get the normalized S3 filenames
|
||
|
|
normalized_s3_files = list_s3_files(bucket_name, prefix, s3_client)
|
||
|
|
|
||
|
|
# Find files that are in Excel but not in S3
|
||
|
|
missing_files = excel_filenames_set - normalized_s3_files
|
||
|
|
|
||
|
|
print(f"Type of missing_files: {type(missing_files)}")
|
||
|
|
print(f"Type of normalized_files: {type(normalized_s3_files)}")
|
||
|
|
print(f"Type of excel_filenames_set: {type(excel_filenames_set)}")
|
||
|
|
|
||
|
|
# Identify files that may have encoding differences, ensuring only strings are processed
|
||
|
|
missing_files_list = [file for file in missing_files if isinstance(file, str) and not (isinstance(file, float) and math.isnan(file))]
|
||
|
|
non_encoded_issues = [file for file in missing_files_list if all(ord(char) < 128 for char in file)]
|
||
|
|
|
||
|
|
output_df = pd.DataFrame({
|
||
|
|
'Missing File Name': missing_files_list,
|
||
|
|
'Encoding Check': ['No Encoding Issue' if file in non_encoded_issues else 'Encoding Issue' for file in missing_files_list]
|
||
|
|
})
|
||
|
|
|
||
|
|
output_df.to_csv(output_csv, index=False, encoding='utf-8', quoting=csv.QUOTE_ALL)
|
||
|
|
# Saving output as excel file
|
||
|
|
output_df.to_excel(output_csv[:-4] + '.xlsx', index=False)
|
||
|
|
|
||
|
|
# Write the missing files and non-encoded issue files to the output CSV
|
||
|
|
# with open(output_csv, mode='w', newline='', encoding='utf-8') as csvfile:
|
||
|
|
# csvfile.write('Missing File Name,Encoding Check\n')
|
||
|
|
# for file in missing_files_list:
|
||
|
|
# encoding_status = 'No Encoding Issue' if file in non_encoded_issues else 'Encoding Issue'
|
||
|
|
# csvfile.write(f"{file},{encoding_status}\n")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
input_excel = 'C:\\Doczy\\National contracting\\Allbatches\\batch_files\\updated_batches_102424\\first_batch\\Batch 14_10222024.xlsx' # Path to your input Excel file
|
||
|
|
bucket_name = 'centene-national-contracting-files' # Your S3 bucket name
|
||
|
|
prefix = 'batch_14_priority_files/TXT_FILES/' # Your S3 prefix
|
||
|
|
output_csv = 'C:\\Doczy\\National contracting\\Allbatches\\diff_batch14_2.csv' # This will also save as xls
|
||
|
|
find_missing_files(input_excel, bucket_name, prefix, output_csv)
|